[ { "path": ".Rbuildignore", "content": "^.*\\.Rproj$\n^.travis.yml\n^README.md\n^\\.Rprofile$\n^\\.Rproj\\.user$\n^appveyor\\.yml$\n^check$\n^doc$\n^gen$\n^libs$\n^inst/lib$\n^inst/libs$\n^revdep$\n^src/.*\\.o$\n^src/tbb/build$\n^tags$\n^tests/testthat/pkg/RcppParallelTest/src/.*\\.dll$\n^tests/testthat/pkg/RcppParallelTest/src/.*\\.s?o$\n^tools/tbb$\n^\\.github$\n^patches" }, { "path": ".gitattributes", "content": "Makevars text eol=lf\n" }, { "path": ".github/.gitignore", "content": "*.html\n" }, { "path": ".github/workflows/R-CMD-check.yaml", "content": "# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples\n# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help\non:\n push:\n branches: [main, master]\n pull_request:\n branches: [main, master]\n\nname: R-CMD-check\n\njobs:\n R-CMD-check:\n runs-on: ${{ matrix.config.os }}\n\n name: ${{ matrix.config.os }} (${{ matrix.config.r }})\n\n strategy:\n fail-fast: false\n matrix:\n config:\n - {os: macOS-latest, r: 'release'}\n - {os: ubuntu-latest, r: 'release'}\n - {os: windows-latest, r: 'release'}\n env:\n GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}\n R_KEEP_PKG_SOURCE: yes\n steps:\n - uses: actions/checkout@v3\n\n - uses: r-lib/actions/setup-r@v2\n with:\n use-public-rspm: true\n\n - uses: r-lib/actions/setup-r-dependencies@v2\n with:\n extra-packages: any::rcmdcheck\n needs: check\n\n - uses: r-lib/actions/check-r-package@v2\n" }, { "path": ".gitignore", "content": ".Rprofile\n.Rproj.user\n.Rhistory\n.RData\n.DS_Store\ncheck\ninst/doc\ninst/lib\ninst/libs\nlibs\nrevdep\nsrc-i386\nsrc-x64\ntbb.log\n\nsrc/tbb/build\nsrc/tbb/build-tbb\n\nR/tbb-autodetected.R\n\n" }, { "path": "DESCRIPTION", "content": "Package: RcppParallel\nType: Package\nTitle: Parallel Programming Tools for 'Rcpp'\nVersion: 5.1.10.9000\nAuthors@R: c(\n person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\",\n comment = c(ORCID = \"0000-0003-2880-7407\")),\n person(\"JJ\", \"Allaire\", role = c(\"aut\"), email = \"jj@rstudio.com\"),\n person(\"Romain\", \"Francois\", role = c(\"aut\", \"cph\")),\n person(\"Gregory\", \"Vandenbrouck\", role = \"aut\"),\n person(\"Marcus\", \"Geelnard\", role = c(\"aut\", \"cph\"),\n comment = \"TinyThread library, https://tinythreadpp.bitsnbites.eu/\"),\n person(\"Hamada S.\", \"Badr\", email = \"badr@jhu.edu\", role = c(\"ctb\"),\n comment = c(ORCID = \"0000-0002-9808-2344\")),\n person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\"), email = \"edd@debian.org\",\n comment = c(ORCID = \"0000-0001-6419-907X\")),\n person(family = \"Intel\", role = c(\"aut\", \"cph\"), comment = \"oneTBB library\"),\n person(family = \"UXL Foundation\", role = c(\"aut\", \"cph\"), comment = \"oneTBB library\"),\n person(family = \"Microsoft\", role = \"cph\"),\n person(family = \"Posit, PBC\", role = \"cph\")\n )\nDescription: High level functions for parallel programming with 'Rcpp'.\n For example, the 'parallelFor()' function can be used to convert the work of\n a standard serial \"for\" loop into a parallel one and the 'parallelReduce()'\n function can be used for accumulating aggregate or other values.\nDepends: R (>= 3.6.0)\nSuggests:\n Rcpp,\n RUnit,\n knitr,\n rmarkdown\nRoxygen: list(markdown = TRUE)\nSystemRequirements: CMake (>= 3.5)\nLicense: GPL (>= 3)\nURL: https://rcppcore.github.io/RcppParallel/, https://github.com/RcppCore/RcppParallel\nBugReports: https://github.com/RcppCore/RcppParallel/issues\nBiarch: TRUE\nRoxygenNote: 7.3.2\nEncoding: UTF-8\n" }, { "path": "NAMESPACE", "content": "# Generated by roxygen2: do not edit by hand\n\nexport(CxxFlags)\nexport(LdFlags)\nexport(RcppParallel.package.skeleton)\nexport(RcppParallelLibs)\nexport(defaultNumThreads)\nexport(setThreadOptions)\nexport(tbbLibraryPath)\n" }, { "path": "NEWS.md", "content": "\n## RcppParallel 6.0.0 (UNRELEASED)\n\n* RcppParallel no longer includes tbb headers as part of the RcppParallel/TBB.h\n header, and instead only exposes its TBB-specific APIs for parallel work.\n \n* RcppParallel now bundles oneTBB 2022.0.0. Note that the TBB ABI has changed;\n packages which depend on RcppParallel may need to be rebuilt.\n \n* On Windows, RcppParallel now uses the copy of TBB provided by Rtools.\n If TBB is not available, RcppParallel will use only the fallback 'tinythread'\n implementation. In practice, this implies that RcppParallel will now only\n provide a TBB backend with R (>= 4.2.0).\n\n## RcppParallel 5.1.11\n\n* Compatibility fixes for LLVM 21.\n\n## RcppParallel 5.1.10\n\n* Fixed an issue where packages linking to RcppParallel could inadverently\n depend on internals of the TBB library available during compilation, even\n if the package did not explicitly use TBB itself.\n\n## RcppParallel 5.1.9\n\n* RcppParallel no longer passes `-rpath` when building / linking on Windows.\n This fixes build issues when building RcppParallel when using the LLVM\n linker on Windows. (@kalibera)\n \n## RcppParallel 5.1.8\n\n* RcppParallel now explicitly links to the bundled copy of TBB on macOS. (#206; @jeroen)\n\n## RcppParallel 5.1.7\n\n* Remove deprecated `std::iterator`. (#192; @Enchufa2)\n\n## RcppParallel 5.1.6\n\n* Patch for TBB to allow compilation with gcc-13.\n\n* Fixed a memory leak that could occur when using TinyThread on POSIX systems.\n (#185; @dipertix and and @kevinushey)\n\n## RcppParallel 5.1.5\n\n* Patches to ensure compatibility with the R 4.2.0 UCRT toolchain on Windows,\n adapted from patches contributed by Tomas Kalibera.\n\n* Fixed an issue where setting `TBB_ROOT` (or `TBB_INC` / `TBB_LIB`) would\n copy rather than symlink the associated libraries. (#161)\n\n## RcppParallel 5.1.4\n\n* Fixed an issue causing client packages of RcppParallel to fail to compile\n on Solaris.\n\n## RcppParallel 5.1.3\n\n* Fixed an issue that prevented compilation of RcppParallel with R (< 4.0.0)\n of R on Windows.\n\n* The `RCPP_PARALLEL_USE_TBBMALLOC_PROXY` environment variable can now be used\n to control whether RcppParallel loads the `tbbmalloc_proxy` library on load.\n See https://www.threadingbuildingblocks.org/docs/help/tbb_userguide/Automically_Replacing_malloc.html\n for more information.\n\n## RcppParallel 5.1.2\n\n* `RcppParallel` gains the `tbbLibraryPath()` function, to be used when attempting\n to query the location of the TBB libraries that `RcppParallel` has been\n configured to use. This may be useful for R packages which wish to explicitly\n use, or link to, these libraries.\n\n## RcppParallel 5.1.1\n\n* Updated bundled version of TBB (Intel TBB 2019 Update 8).\n\n* RcppParallel can now be configured to use an external copy of TBB, via the\n `TBB_LIB` and `TBB_INC` environment variables. These should be set to the\n directories containing the TBB libraries and headers, respectively.\n \n* Added support for the latest versions of Intel oneAPI TBB / oneTBB.\n\n* Updated TBB functionality for the new interface.\n\n* Falling back to building TBB from local source code.\n\n* Backward TBB compatibility based on `__TBB_tbb_stddef_H`.\n\n* Resolved conflicts between system and local TBB headers.\n\n* Fixed URLs, used HTTPS, and minor cleanups.\n\n* Updated package DESCRIPTION and bumped version.\n\n* `setThreadOptions(...)` can again be called multiple times per session.\n The requested number of threads will be used for invocations to `parallelFor()`\n and `parallelReduce()` that don't explicitly request a specific number of threads.\n \n* The `parallelFor()` and `parallelReduce()` functions gain the `numThreads`\n argument, allowing one to limit the number of threads used for a\n particular computation.\n\n## RcppParallel 5.0.3\n\n* Fixed compilation on macOS M1 machines.\n\n## RcppParallel 5.0.2\n\n* `setThreadOptions(...)` can now only be called once per session, to avoid\n segfaults when compiling RcppParallel / TBB with gcc 10.1. Subsequent\n calls to `setThreadOptions(...)` are ignored.\n\n## RcppParallel 5.0.1\n\n* Fixed compilation issue on OpenSUSE Tumbleweed with -flto=auto\n\n* Fixed compilation when CPPFLAGS = -I/usr/local/include and a version\n of libtbb is installed there\n\n## RcppParallel 5.0.0\n\n* RcppParallel backend can now be customized with RCPP_PARALLEL_BACKEND\n environment variable (supported values are 'tbb' and 'tinythread')\n \n* Fixed issue when compiling RcppParallel on macOS Catalina\n\n* Fixed issue when compiling RcppParallel with Rtools40\n\n## RcppParallel 4.4.4\n\n* Fixed an issue when compiling RcppParallel with clang-9 on Fedora\n\n## RcppParallel 4.4.3\n\n* Suppress gcc-9 warnings related -Wclass-memaccess\n\n* Added TBB headers for serial TBB operations (#90, @mikejiang)\n\n* Fixed row iterator constructor (#87, @wtianyi)\n\n* Fixed compilation on FreeBSD\n\n## RcppParallel 4.4.2\n\n* Suppress gcc-8 warnings related to -Wclass-memaccess\n\n* Use PKG_CXXFLAGS rather than PKG_CPPFLAGS\n\n* Remove unused dependency on the BH package\n\n## RcppParallel 4.4.1\n\n* Ensure user-specified R configuration passed to TBB\n\n* Work around warnings emitted by gcc 8\n\n## RcppParallel 4.4.0\n\n* Respect user-defined compiler settings (e.g. from ~/.R/Makevars).\n\n* Remove TBB's attempts to suppress compiler diagnostics.\n\n* Allow setting the number of threads to use via RCPP_PARALLEL_NUM_THREADS\n environment variable.\n\n* Update to TBB 2018 Update 1.\n\n* Add native registration of compiled functions.\n\n## RcppParallel 4.3.20\n\n* Add support for Rtools 3.3 w/ GCC 4.9\n\n## RcppParallel 4.3.14\n\n* Add support for TBB on Solaris\n\n* Fix failure to compile on OS X Snow Leopard R toolchain\n\n* Add const and non-const operator[] for RMatrix class\n\n## RcppParallel 4.3.8\n\n* Add tbbmalloc library\n\n* Correctly pass clang to TBB configure when R is using clang\n\n## RcppParallel 4.3.6\n\n* Support for TBB on Windows\n\n## RcppParallel 4.3.3\n\n* Update to TBB 4.3 (fixes clang compilation error in platform.h)\n\n* Forward CXX to TBB Makefile\n\n## RcppParallel 4.2.5\n\n* Initial release\n" }, { "path": "R/RcppParallel-package.R", "content": "\n#' Parallel programming tools for Rcpp\n#' \n#' High level functions for doing parallel programming with Rcpp. For example,\n#' the `parallelFor()` function can be used to convert the work of a\n#' standard serial \"for\" loop into a parallel one, and the `parallelReduce()`\n#' function can be used for accumulating aggregate or other values.\n#' \n#' The high level interface enables safe and robust parallel programming\n#' without direct manipulation of operating system threads. On Windows, macOS,\n#' and Linux systems the underlying implementation is based on Intel TBB\n#' (Threading Building Blocks). On other platforms, a less-performant fallback\n#' implementation based on the TinyThread library is used.\n#' \n#' For additional documentation, see the package website at:\n#' \n#' \n#' \n#' \n#' @name RcppParallel-package\n#' @docType package\n#' @aliases RcppParallel RcppParallel-package\n#' @keywords package parallel\nNULL\n" }, { "path": "R/aaa.R", "content": "\n# stubs that get overridden via configure script\nTBB_ENABLED <- TRUE\nTBB_LIB <- \"\"\nTBB_INC <- \"\"\n\nTBB_NAME <- \"tbb\"\nTBB_MALLOC_NAME <- \"tbbmalloc\"" }, { "path": "R/flags.R", "content": "\n#' Compilation flags for RcppParallel\n#'\n#' Output the compiler or linker flags required to build against RcppParallel.\n#'\n#' These functions are typically called from `Makevars` as follows:\n#'\n#' ```\n#' PKG_LIBS += $(shell \"${R_HOME}/bin/Rscript\" -e \"RcppParallel::LdFlags()\")\n#' ```\n#'\n#' On Windows, the flags ensure that the package links with the built-in TBB\n#' library. On Linux and macOS, the output is empty, because TBB is loaded\n#' dynamically on load by `RcppParallel`.\n#'\n#' \\R packages using RcppParallel should also add the following to their\n#' `NAMESPACE` file:\n#'\n#' ```\n#' importFrom(RcppParallel, RcppParallelLibs)\n#' ```\n#'\n#' This is necessary to ensure that \\pkg{RcppParallel} (and so, TBB) is loaded\n#' and available.\n#'\n#' @name flags\n#' @rdname flags\n#' @aliases RcppParallelLibs LdFlags CxxFlags\n#'\n#' @return Returns \\code{NULL}, invisibly. These functions are called for\n#' their side effects (writing the associated flags to stdout).\n#'\nNULL\n\n\n#' @name flags\n#' @export\nCxxFlags <- function() {\n cat(tbbCxxFlags())\n}\n\n#' @name flags\n#' @export\nLdFlags <- function() {\n cat(tbbLdFlags())\n}\n\n#' @name flags\n#' @export\nRcppParallelLibs <- function() {\n LdFlags()\n}\n\n" }, { "path": "R/options.R", "content": "\n#' Thread options for RcppParallel\n#' \n#' Set thread options (number of threads to use for task scheduling and stack\n#' size per-thread) for RcppParallel.\n#' \n#' RcppParallel is automatically initialized with the default number of threads\n#' and thread stack size when it loads. You can call `setThreadOptions()` at\n#' any time to change the defaults.\n#' \n#' The `parallelFor()` and `parallelReduce()` also accept `numThreads` as\n#' an argument, if you'd like to control the number of threads specifically\n#' to be made available for a particular parallel function call. Note that\n#' this value is advisory, and TBB may choose a smaller number of threads\n#' if the number of requested threads cannot be honored on the system.\n#' \n#' @aliases setThreadOptions defaultNumThreads\n#' \n#' @param numThreads\n#' Number of threads to use for task scheduling. Call `defaultNumThreads()`\n#' to determine the the default value used for \"auto\".\n#' \n#' @param stackSize\n#' Stack size (in bytes) to use for worker threads. The\n#' default used for \"auto\" is 2MB on 32-bit systems and 4MB on 64-bit systems\n#' (note that this parameter has no effect on Windows).\n#' \n#' @return\n#' `defaultNumThreads()` returns the default number of threads used by\n#' RcppParallel, if another value isn't specified either via\n#' `setThreadOptions()` or explicitly in calls to `parallelFor()` and\n#' `parallelReduce()`.\n#'\n#' @examples\n#' \n#' \\dontrun{\n#' library(RcppParallel)\n#' setThreadOptions(numThreads = 4)\n#' defaultNumThreads()\n#' }\n#' \n#' @export setThreadOptions\nsetThreadOptions <- function(numThreads = \"auto\",\n stackSize = \"auto\")\n{\n # validate and resolve numThreads\n if (identical(numThreads, \"auto\"))\n numThreads <- -1L\n else if (!is.numeric(numThreads))\n stop(\"numThreads must be an integer\")\n else\n numThreads <- as.integer(numThreads)\n \n # validate and resolve stackSize\n if (identical(stackSize, \"auto\"))\n stackSize <- 0L\n else if (!is.numeric(stackSize))\n stop(\"stackSize must be an integer\")\n else\n stackSize <- as.integer(stackSize)\n \n # set RCPP_PARALLEL_NUM_THREADS\n if (numThreads == -1L)\n Sys.unsetenv(\"RCPP_PARALLEL_NUM_THREADS\")\n else\n Sys.setenv(RCPP_PARALLEL_NUM_THREADS = numThreads)\n \n # set RCPP_PARALLEL_STACK_SIZE\n if (stackSize == 0L)\n Sys.unsetenv(\"RCPP_PARALLEL_STACK_SIZE\")\n else\n Sys.setenv(RCPP_PARALLEL_STACK_SIZE = stackSize)\n}\n\n#' @rdname setThreadOptions\n#' @export\ndefaultNumThreads <- function() {\n .Call(\"defaultNumThreads\", PACKAGE = \"RcppParallel\")\n}\n\nisUsingTbb <- function() {\n backend <- Sys.getenv(\"RCPP_PARALLEL_BACKEND\", \"tbb\")\n identical(backend, \"tbb\")\n}\n\n" }, { "path": "R/platform.R", "content": "\nis_windows <- function() {\n .Platform$OS.type == \"windows\"\n}\n\nis_mac <- function() {\n Sys.info()[[\"sysname\"]] == \"Darwin\"\n}\n\nis_unix <- function() {\n .Platform$OS.type == \"unix\"\n}\n\nis_solaris <- function() {\n Sys.info()[[\"sysname\"]] == \"SunOS\"\n}\n\nis_sparc <- function() {\n info <- Sys.info()\n all(\n info[[\"sysname\"]] == \"SunOS\",\n info[[\"machine\"]] != \"i86pc\"\n )\n}\n\n\n" }, { "path": "R/plugin.R", "content": "\n# Inline plugin used by sourceCpp.\ninlineCxxPlugin <- function() {\n \n list(\n env = list(\n PKG_CXXFLAGS = tbbCxxFlags(),\n PKG_LIBS = tbbLdFlags()\n ),\n includes = \"#include \",\n LinkingTo = \"RcppParallel\",\n body = identity,\n Depends = \"RcppParallel\"\n )\n}\n" }, { "path": "R/skeleton.R", "content": "\n#' Create a skeleton for a new package depending on RcppParallel\n#' \n#' \\code{RcppParallel.package.skeleton} automates the creation of a new source\n#' package that intends to use features of RcppParallel.\n#' \n#' It is based on the \\link[utils]{package.skeleton} function which it executes\n#' first.\n#' \n#' In addition to \\link[Rcpp]{Rcpp.package.skeleton} :\n#' \n#' The \\samp{DESCRIPTION} file gains an Imports line requesting that the\n#' package depends on RcppParallel and a LinkingTo line so that the package\n#' finds RcppParallel header files.\n#' \n#' The \\samp{NAMESPACE} gains a \\code{useDynLib} directive as well as an\n#' \\code{importFrom(RcppParallel, evalCpp} to ensure instantiation of\n#' RcppParallel.\n#' \n#' The \\samp{src} directory is created if it does not exists and a\n#' \\samp{Makevars} file is added setting the environment variables\n#' \\samp{PKG_LIBS} to accomodate the necessary flags to link with the\n#' RcppParallel library.\n#' \n#' If the \\code{example_code} argument is set to \\code{TRUE}, example files\n#' \\samp{vector-sum.cpp} is created in the \\samp{src} directory.\n#' \\code{Rcpp::compileAttributes()} is then called to generate\n#' \\code{src/RcppExports.cpp} and \\code{R/RcppExports.R}. These files are given\n#' as an example and should eventually by removed from the generated package.\n#' \n#' @param name The name of your R package.\n#' @param example_code If \\code{TRUE}, example C++ code using RcppParallel is\n#' added to the package.\n#' @param ... Optional arguments passed to \\link[Rcpp]{Rcpp.package.skeleton}.\n#' @return Nothing, used for its side effects\n#' @seealso \\link[utils]{package.skeleton}\n#' @references Read the \\emph{Writing R Extensions} manual for more details.\n#' \n#' Once you have created a \\emph{source} package you need to install it: see\n#' the \\emph{R Installation and Administration} manual, \\code{\\link{INSTALL}}\n#' and \\code{\\link{install.packages}}.\n#' @keywords programming\n#' @examples\n#' \n#' \\dontrun{\n#' # simple package\n#' RcppParallel.package.skeleton(\"foobar\")\n#' }\n#' \n#' @export RcppParallel.package.skeleton\nRcppParallel.package.skeleton <- function(name = \"anRpackage\",\n example_code = TRUE,\n ...)\n{\n # call Rcpp.package.skeleton() -- provide 'list' explicitly\n # and clean up after\n env <- new.env(parent = emptyenv())\n env$dummy <- NULL\n Rcpp::Rcpp.package.skeleton(\n name = name,\n attributes = FALSE,\n module = FALSE,\n example_code = FALSE,\n environment = env,\n list = \"dummy\",\n ...\n )\n \n # move to generated package directory\n owd <- setwd(name)\n on.exit(setwd(owd), add = TRUE)\n \n # remove dummy stuff\n unlink(\"data\", recursive=TRUE)\n unlink(\"man/dummy.Rd\")\n unlink(\"Read-and-delete-me\")\n lns <- readLines(\"NAMESPACE\")\n writeLines(lns[!grepl(\"dummy\", lns)], \"NAMESPACE\")\n unlink(\"src/init.c\")\n \n message(\"\\nAdding RcppParallel settings\")\n \n # update DESCRIPTION file\n desc <- read.dcf(\"DESCRIPTION\", all = TRUE, keep.white = TRUE)\n version <- sprintf(\"RcppParallel (>= %s)\", utils::packageVersion(\"RcppParallel\"))\n \n desc$Imports <- paste0(desc$Imports, \", \", version)\n message(\" >> added Imports: \", desc$Imports)\n \n desc$LinkingTo <- paste0(desc$LinkingTo, \", RcppParallel\")\n message(\" >> added LinkingTo: \", desc$LinkingTo)\n \n desc$SystemRequirements <- \"GNU make\"\n message(\" >> added SystemRequirements: GNU make\")\n \n write.dcf(desc, file = \"DESCRIPTION\", keep.white = TRUE)\n \n # update NAMESPACE file\n message(\" >> added importFrom(RcppParallel,RcppParallelLibs) directive to NAMESPACE\")\n cat(\"importFrom(RcppParallel,RcppParallelLibs)\",\n file = \"NAMESPACE\",\n sep = \"\\n\",\n append = TRUE)\n \n # write Makevars files\n dir.create(\"src\", showWarnings = FALSE)\n \n # src/Makevars\n message(\" >> added src/Makevars\")\n cat(\n c(\n '# We also need importFrom(RcppParallel,RcppParallelLibs) in NAMESPACE',\n 'PKG_LIBS += $(shell ${R_HOME}/bin/Rscript -e \"RcppParallel::RcppParallelLibs()\")'\n ),\n file = \"src/Makevars\",\n sep = \"\\n\"\n )\n \n # src/Makevars.win\n message(\" >> added src/Makevars.win\")\n cat(\n c(\n 'PKG_CXXFLAGS += -DRCPP_PARALLEL_USE_TBB=1',\n 'PKG_LIBS += $(shell \"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe\" -e \"RcppParallel::RcppParallelLibs()\")'\n ),\n file = \"src/Makevars.win\",\n sep = \"\\n\"\n )\n \n # write an example script using RcppParallel\n if (example_code) {\n \n message(\" >> added example file src/vector-sum.cpp\")\n file.copy(\n system.file(\"skeleton/vector-sum.cpp\", package = \"RcppParallel\"),\n \"src/vector-sum.cpp\"\n )\n \n message(\" >> added example documentation man/vector-sum.Rd\")\n file.copy(\n system.file(\"skeleton/vector-sum.Rd\", package = \"RcppParallel\"),\n \"man/vector-sum.Rd\"\n )\n \n message(\" >> compiled Rcpp attributes\")\n Rcpp::compileAttributes()\n }\n \n TRUE\n}\n" }, { "path": "R/tbb-autodetected.R.in", "content": "\nTBB_ENABLED <- @TBB_ENABLED@\nTBB_LIB <- \"@TBB_LIB@\"\nTBB_INC <- \"@TBB_INC@\"\n\nTBB_NAME <- \"@TBB_NAME@\"\nTBB_MALLOC_NAME <- \"@TBB_MALLOC_NAME@\"" }, { "path": "R/tbb.R", "content": "\n#' Get the Path to a TBB Library\n#'\n#' Retrieve the path to a TBB library. This can be useful for \\R packages\n#' using RcppParallel that wish to use, or re-use, the version of TBB that\n#' RcppParallel has been configured to use.\n#'\n#' @param name\n#' The name of the TBB library to be resolved. Normally, this is one of\n#' `tbb`, `tbbmalloc`, or `tbbmalloc_proxy`. When `NULL`, the library\n#' path containing the TBB libraries is returned instead.\n#'\n#' @export\ntbbLibraryPath <- function(name = NULL) {\n\n # library paths for different OSes\n sysname <- Sys.info()[[\"sysname\"]]\n\n # find root for TBB install\n tbbRoot <- Sys.getenv(\"TBB_LIB\", unset = tbbRoot())\n if (is.null(name))\n return(tbbRoot)\n\n # form library names\n tbbLibNames <- list(\n \"Darwin\" = paste0(\"lib\", name, \".dylib\"),\n \"Windows\" = paste0(\"lib\", name, c(\"12\", \"\"), \".a\"),\n \"SunOS\" = paste0(\"lib\", name, \".so\"),\n \"Linux\" = paste0(\"lib\", name, c(\".so.2\", \".so\"))\n )\n\n # skip systems that we know not to be compatible\n isCompatible <- !is_sparc() && !is.null(tbbLibNames[[sysname]])\n if (!isCompatible)\n return(NULL)\n\n # find the request library (if any)\n libNames <- tbbLibNames[[sysname]]\n for (libName in libNames) {\n \n tbbName <- file.path(tbbRoot, libName)\n if (file.exists(tbbName))\n return(tbbName)\n \n arch <- if (nzchar(.Platform$r_arch)) .Platform$r_arch\n suffix <- paste(c(\"lib\", arch, libName), collapse = \"/\")\n tbbName <- system.file(suffix, package = \"RcppParallel\")\n if (file.exists(tbbName))\n return(tbbName)\n \n }\n\n}\n\ntbbCxxFlags <- function() {\n\n if (!TBB_ENABLED)\n return(\"-DRCPP_PARALLEL_USE_TBB=0\")\n \n flags <- c(\"-DRCPP_PARALLEL_USE_TBB=1\")\n\n # if TBB_INC is set, apply those library paths\n tbbInc <- Sys.getenv(\"TBB_INC\", unset = TBB_INC)\n if (!file.exists(tbbInc)) {\n tbbInc <- system.file(\"include\", package = \"RcppParallel\")\n }\n \n # add include path\n if (nzchar(tbbInc) && file.exists(tbbInc)) {\n \n # prefer new interface if version.h exists -- we keep this\n # for compatibility with packages like StanHeaders, rstan\n versionPath <- file.path(tbbInc, \"tbb/version.h\")\n if (file.exists(versionPath))\n flags <- c(flags, \"-DTBB_INTERFACE_NEW\")\n \n # now add the include path\n flags <- c(flags, paste0(\"-I\", asBuildPath(tbbInc)))\n \n }\n\n # return flags as string\n paste(flags, collapse = \" \")\n\n}\n\n# Return the linker flags required for TBB on this platform\ntbbLdFlags <- function() {\n \n # on Windows, we statically link to oneTBB\n if (is_windows()) {\n \n libPath <- system.file(\"libs\", package = \"RcppParallel\")\n if (nzchar(.Platform$r_arch))\n libPath <- file.path(libPath, .Platform$r_arch)\n \n ldFlags <- sprintf(\"-L%s -lRcppParallel\", asBuildPath(libPath))\n return(ldFlags)\n \n }\n \n # shortcut if TBB_LIB defined\n tbbLib <- Sys.getenv(\"TBB_LINK_LIB\", Sys.getenv(\"TBB_LIB\", unset = TBB_LIB))\n if (nzchar(tbbLib)) {\n if (R.version$os == \"emscripten\") {\n fmt <- \"-L%1$s -l%2$s\"\n return(sprintf(fmt, asBuildPath(tbbLib), TBB_NAME))\n }\n fmt <- \"-L%1$s -Wl,-rpath,%1$s -l%2$s -l%3$s\"\n return(sprintf(fmt, asBuildPath(tbbLib), TBB_NAME, TBB_MALLOC_NAME))\n }\n \n # explicitly link on macOS\n # https://github.com/RcppCore/RcppParallel/issues/206\n if (is_mac()) {\n fmt <- \"-L%s -l%s -l%s\"\n return(sprintf(fmt, asBuildPath(tbbLibraryPath()), TBB_NAME, TBB_MALLOC_NAME))\n }\n\n # nothing required on other platforms\n \"\"\n\n}\n\ntbbRoot <- function() {\n\n if (nzchar(TBB_LIB))\n return(TBB_LIB)\n\n rArch <- .Platform$r_arch\n parts <- c(\"lib\", if (nzchar(rArch)) rArch)\n libDir <- paste(parts, collapse = \"/\")\n system.file(libDir, package = \"RcppParallel\")\n\n}\n" }, { "path": "R/utils.R", "content": "\n# generate paths consumable by the compilers and linkers\n# in particular, on Windows and Solaris, this means the path _cannot_ be quoted !!\nasBuildPath <- function(path) {\n\n # normalize paths using forward slashes\n path <- normalizePath(path, winslash = \"/\", mustWork = FALSE)\n\n # prefer short path names if the path has spaces\n if (is_windows() && grepl(\" \", path, fixed = TRUE))\n path <- utils::shortPathName(path)\n\n # if we still have spaces, and we're not Windows or Solaris, try quoting\n if (grepl(\" \", path, fixed = TRUE) && !is_solaris())\n path <- shQuote(path)\n\n # ensure we use forward slashes, even on Windows\n path <- chartr(\"\\\\\", \"/\", path)\n\n # return path\n path\n\n}\n\n" }, { "path": "R/zzz.R", "content": "\n# !diagnostics suppress=.dllInfo,.tbbDllInfo,.tbbMallocDllInfo,.tbbMallocProxyDllInfo\n\n# NOTE: we intentionally do _not_ load tbbmalloc_proxy by default, as its\n# intended use is to replace the default allocator, something that may be\n# dangerous to do by default. in addition, TBB's documentation recommends\n# only loading explicitly via e.g. LD_PRELOAD\n.dllInfo <- NULL\n.tbbDllInfo <- NULL\n.tbbMallocDllInfo <- NULL\n.tbbMallocProxyDllInfo <- NULL\n\nloadTbbLibrary <- function(name) {\n # TBB is statically linked on Windows\n if (is_windows()) {\n return(NULL)\n }\n path <- tbbLibraryPath(name)\n if (is.null(path))\n return(NULL)\n \n if (!file.exists(path)) {\n warning(\"TBB library \", shQuote(name), \" not found.\")\n return(NULL)\n }\n \n dyn.load(path, local = FALSE, now = TRUE)\n \n}\n\n.onLoad <- function(libname, pkgname) {\n \n # on Windows, load RcppParallel first\n if (.Platform$OS.type == \"windows\") {\n .dllInfo <<- library.dynam(\"RcppParallel\", pkgname, libname)\n }\n \n # load tbb, tbbmalloc\n .tbbDllInfo <<- loadTbbLibrary(\"tbb\")\n .tbbMallocDllInfo <<- loadTbbLibrary(\"tbbmalloc\")\n \n # load tbbmalloc_proxy, but only if requested\n useTbbMallocProxy <- Sys.getenv(\"RCPP_PARALLEL_USE_TBBMALLOC_PROXY\", unset = \"FALSE\")\n if (useTbbMallocProxy %in% c(\"TRUE\", \"True\", \"true\", \"1\"))\n .tbbMallocProxyDllInfo <<- loadTbbLibrary(\"tbbmalloc_proxy\")\n \n # load RcppParallel library if available\n if (.Platform$OS.type != \"windows\") {\n .dllInfo <<- library.dynam(\"RcppParallel\", pkgname, libname, local = FALSE)\n }\n \n}\n\n.onUnload <- function(libpath) {\n \n # unload the package library\n if (!is.null(.dllInfo))\n library.dynam.unload(\"RcppParallel\", libpath)\n \n # NOTE: we do not explicitly unload tbbmalloc_proxy as switching\n # the allocator at runtime can cause issues\n \n # unload tbbmalloc if we loaded it\n if (!is.null(.tbbMallocDllInfo))\n dyn.unload(.tbbMallocDllInfo[[\"path\"]])\n \n # unload tbb if we loaded it\n if (!is.null(.tbbDllInfo))\n dyn.unload(.tbbDllInfo[[\"path\"]])\n \n}\n" }, { "path": "README.md", "content": "\n## RcppParallel\n\n\n[![CRAN](https://www.r-pkg.org/badges/version/RcppParallel)](https://cran.r-project.org/package=RcppParallel)\n[![R-CMD-check](https://github.com/RcppCore/RcppParallel/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/RcppCore/RcppParallel/actions/workflows/R-CMD-check.yaml)\n\n\nHigh level functions for parallel programming with Rcpp. The `parallelFor()` function can be used to convert the work of a standard serial \"for\" loop into a parallel one, and the `parallelReduce()` function can be used for accumulating aggregate or other values.\n\nThe high level interface enables safe and robust parallel programming without direct manipulation of operating system threads. On Windows, macOS, and Linux systems, the underlying implementation is based on [Intel TBB](https://github.com/oneapi-src/oneTBB) (Threading Building Blocks). On other platforms, a less-performant fallback implementation based on the [TinyThread](https://tinythreadpp.bitsnbites.eu/) library is used.\n\nFor additional documentation on using RcppParallel see the package website at http://rcppcore.github.io/RcppParallel/.\n\n\n### Intel TBB\n\n`RcppParallel` supports the new interface of Intel TBB, and can be configured to use an external copy of TBB (e.g., with [`oneTBB`](https://github.com/oneapi-src/oneTBB) or the system TBB library), using the `TBB_LIB` and `TBB_INC` environment variables.\n\nTo build the development version of `RcppParallel` with [`oneTBB`](https://github.com/oneapi-src/oneTBB):\n\n- Install [`oneTBB`](https://github.com/oneapi-src/oneTBB).\n\nFor example, installing [`oneTBB`](https://github.com/oneapi-src/oneTBB) on Linux 64-bit (`x86_64`) to `$HOME` directory (change if needed!):\n\n```bash\nTBB_RELEASE=\"https://api.github.com/repos/oneapi-src/oneTBB/releases/latest\"\nTBB_TAG=$(curl --silent $TBB_RELEASE | grep -Po '\"tag_name\": \"\\K.*?(?=\")')\nTBB_VERSION=${TBB_TAG#?}\n\nwget https://github.com/oneapi-src/oneTBB/releases/download/v$TBB_VERSION/oneapi-tbb-$TBB_VERSION-lin.tgz\ntar zxvf oneapi-tbb-$TBB_VERSION-lin.tgz -C $HOME\n\nexport TBB=\"$HOME/oneapi-tbb-$TBB_VERSION\"\n```\nNote that you may replace `TBB_VERSION=${TBB_TAG#?}` with a custom version number if needed ( check available releases [here](https://github.com/oneapi-src/oneTBB/releases) ).\n\n- Set the TBB environment variables (specifically: `TBB` for the installation prefix, `TBB_INC` for the directory that includes the header files, and `TBB_LIB` for the libraries directory).\n\nFor example, installing [`oneTBB`](https://github.com/oneapi-src/oneTBB) on Linux 64-bit (`x86_64`) to `$HOME` directory (change if needed!):\n\n```bash\nsource $TBB/env/vars.sh intel64\n\nexport TBB_INC=\"$TBB/include\"\nexport TBB_LIB=\"$TBB/lib/intel64/gcc4.8\"\n```\n\n- Build the development version of `RcppParallel`:\n\n```r\ninstall.packages(\"remotes\")\nremotes::install_github(\"RcppCore/RcppParallel\")\n```\n\n\n### License\n\nThe RcppParallel package is made available under the [GPLv2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html) license.\n\nThe [TinyThread library](https://tinythreadpp.bitsnbites.eu/) is licensed under the [zlib/libpng](https://opensource.org/licenses/zlib-license.php) license.\n\nThe Intel TBB Library is licensed under the Apache 2.0 license, as described at https://github.com/oneapi-src/oneTBB/blob/master/LICENSE.txt.\n" }, { "path": "RcppParallel.Rproj", "content": "Version: 1.0\nProjectId: 8e3d73b0-404c-42f5-b2ef-46f759f65dd4\n\nRestoreWorkspace: No\nSaveWorkspace: No\nAlwaysSaveHistory: No\n\nEnableCodeIndexing: Yes\nUseSpacesForTab: Yes\nNumSpacesForTab: 3\nEncoding: UTF-8\n\nRnwWeave: Sweave\nLaTeX: pdfLaTeX\n\nAutoAppendNewline: Yes\n\nBuildType: Package\nPackageCleanBeforeInstall: No\nPackageInstallArgs: --with-keep.source\nPackageCheckArgs: --as-cran\nPackageRoxygenize: rd,collate,namespace\n" }, { "path": "cleanup", "content": "#!/usr/bin/env sh\n: \"${R_HOME=`R RHOME`}\"\n\"${R_HOME}/bin/Rscript\" tools/config.R cleanup \"$@\"\n" }, { "path": "cleanup.win", "content": "#!/usr/bin/env sh\n\"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe\" tools/config.R cleanup \"$@\"\n" }, { "path": "configure", "content": "#!/usr/bin/env sh\n: \"${R_HOME=`R RHOME`}\"\n\"${R_HOME}/bin/Rscript\" tools/config.R configure \"$@\"\n" }, { "path": "configure.win", "content": "#!/usr/bin/env sh\n\"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe\" tools/config.R configure \"$@\"\n" }, { "path": "doc/rtools_tbb_notes.md", "content": "# Differences MinGW / Rtools\r\n\r\n## cmd\r\n\r\n### MinGW\r\n\r\ncmd is an sh wrapper which invokes the 32bit version of Windows' shell:\r\n\r\n\t$ where cmd\r\n\tC:\\MinGW\\msys\\1.0\\bin\\cmd\r\n\tc:\\Windows\\System32\\cmd.exe\r\n\t$ cat /c/MinGW/msys/1.0/bin/cmd\r\n\t#!/bin/sh\r\n\t# Copyright (C) 2002, Earnie Boyd\r\n\t# mailto:earnie@users.sf.net\r\n\t# This file is part of Minimal SYStem.\r\n\t# http://www.mingw.org/msys.shtml\r\n\t# File: cmd\r\n\r\n\t\"$COMSPEC\" \"$@\"\r\n\t$ echo $COMSPEC\r\n\tC:\\windows\\SysWOW64\\cmd.exe\r\n\t\r\n### Rtools\r\n\r\nNo wrapper; cmd is the default Windows shell (64bit on a 64bit OS):\r\n\r\n\t$ where cmd\r\n\tC:\\Windows\\System32\\cmd.exe\r\n\t\r\n\r\n## paths\r\n\r\n### MinGW\r\n\r\n- 3 styles supported: `/drive/path`, `drive:/path`, `drive:\\path` (properly escaped).\r\n- The *actual* path (returned when querying) is `/drive/path`.\r\n\r\n\r\n\t\t$ cd f:/tmp/testpath\r\n\t\t$ pwd\r\n\t\t/f/tmp/testpath\r\n\t\t$ cd f:\\\\tmp\\\\testpath\r\n\t\t$ pwd\r\n\t\t/f/tmp/testpath\r\n\t\t$ cd /f/tmp/testpath\r\n\t\t$ pwd\r\n\t\t/f/tmp/testpath\t\r\n\t\r\n### Rtools\r\n\r\n- 3 styles supported: `/cygdrive/drive/path`, `drive:/path`, `drive:\\path` (properly escaped).\r\n- The *actual* path (returned when querying) is `/cygdrive/drive/path`.\r\n- `/drive/path` is **not** supported.\r\n\r\n\t\t$ cd f:/tmp/testpath\r\n\t\t$ pwd\r\n\t\t/cygdrive/f/tmp/testpath\r\n\t\t$ cd f:\\\\tmp\\\\testpath\r\n\t\t$ pwd\r\n\t\t/cygdrive/f/tmp/testpath\r\n\t\t$ cd /f/tmp/testpath\r\n\t\tcd: can't cd to /f/tmp/testpath\r\n\t\t$ cd /cygdrive/f/tmp/testpath\r\n\t\t$ pwd\r\n\t\t/cygdrive/f/tmp/testpath\r\n\t\t\r\n\r\n## parameters\r\n\r\n### MinGW\r\n\r\nShell will sometimes automatically replace parameters starting with forward slashes using the rules described [here](http://www.mingw.org/wiki/Posix_path_conversion). \r\nSince forward slashes are often used to mark option parameters in Windows command line applications (/o is the Windows equivalent of -o and --option for Unix), this can cause very subtle and hard to debug issues. The fix is to double the forward slashes, or to quote (when possible)\r\n\r\n\t$ ls /d\r\n\t$RECYCLE.BIN System Volume Information pagefile.sys\r\n\r\n\t$ cmd /c \"echo hi\"\r\n\thi\r\n\r\n\t$ cmd /C \"echo hi\"\r\n\tMicrosoft Windows [Version 6.3.9600]\r\n\t(c) 2013 Microsoft Corporation. All rights reserved.\r\n\r\n\tf:\\tmp\\testpath>exit\r\n\r\n\t$ cmd /C \"echo hi /runtime\"\r\n\t'untime\"' is not recognized as an internal or external command,\r\n\toperable program or batch file.\r\n\t\r\n\t$ cmd //C \"echo hi /runtime\"\r\n\thi /runtime\r\n\r\n\r\n### Rtools\r\n\r\nNo parameter replacements. What you type is what you get.\r\n\r\n\t$ ls /d\r\n\tls: cannot access /d: No such file or directory\r\n\t$ cmd /c \"echo hi\"\r\n\thi\r\n\t$ cmd /C \"echo hi\"\r\n\thi\r\n\t$ cmd /C \"echo hi /runtime\"\r\n\thi /runtime\r\n\r\n\r\n## uname\r\n\r\nOnly present in MinGW\r\n\r\n## g++\r\n\r\n### MinGW\r\n\r\ndefault full paths supported\r\n\r\n\t$ g++ /f/tmp/testpath/test.cpp\r\n\t$ echo $?\r\n\t0\r\n\r\n### Rtools\r\n\r\ndefault full paths are not supported\r\n\r\n\t$ ls /cygdrive/f/tmp/testpath/test.cpp\r\n\t/cygdrive/f/tmp/testpath/test.cpp\r\n\t$ cat /cygdrive/f/tmp/testpath/test.cpp\r\n\tint main() {return 0;}\t\r\n\t$ g++ /cygdrive/f/tmp/testpath/test.cpp\r\n\tg++.exe: error: /cygdrive/f/tmp/testpath/test.cpp: No such file or directory\r\n\tg++.exe: fatal error: no input files\r\n\tcompilation terminated.\r\n\r\nThis means that great care must be taken to ensure paths are never queried and instead always manually constructed. Examples of queried paths that end up being g++ incompatible in the context of a Makefile:\r\n\r\n- don't use `$(shell pwd)`\r\n- don't use `$(CURDIR)`\r\n- only use **relative paths** in `VPATH`\r\n\t\t\r\n\r\n\t\t$ cat Makefile\r\n\t\t.PHONY: all\r\n\t\tall: test.o foo.o\r\n\t\t\t@echo DIR=$(CURDIR)\r\n\r\n\t\t%.o: %.cpp\r\n\t\t\t@echo $<\r\n\r\n\t\t$ make VPATH=f:/tmp/testpath/subdir\r\n\t\ttest.cpp\r\n\t\t/cygdrive/f/tmp/testpath/subdir/foo.cpp\r\n\t\tDIR=/cygdrive/f/tmp/testpath\r\n\t\t\r\n\t\t$ make VPATH=subdir\r\n\t\ttest.cpp\r\n\t\tsubdir/foo.cpp\r\n\t\tDIR=/cygdrive/f/tmp/testpath\r\n\t\t\r\n# Bugs and tips\r\n\r\n## Modal dialog asking to insert disk in drive\r\n\r\nThis bug is in MinGW only (not Rtools). This is due to hard-coded paths pointing to I: drive in g++\r\n\r\n\t$ strings c:/MinGW/bin/g++.exe | grep i: | grep mingw\r\n\ti:/p/giaw/mingw/share/locale\r\n\ti:/p/giaw/mingw/share/locale\r\n\t\r\nIf no I: drive exists, or the path does not exist, then g++ silently ignores it. *However*, if I: happens to point to a removable drive, then you get a modal dialog.\r\n\r\nFix: go in \"Disk Management\" and rename the drive.\r\n\r\n## Full paths\r\n\r\ng++ is not the only tool affected by full paths in the context of Rtools. Since Rtools doesn't perform any auto-conversion and `/` is used as a option marker for many Windows command line applications, \r\nsome of them end up being confused. For example:\r\n\r\n\t$ cat test.js\r\n\tWScript.Echo( \"Hi\" );\r\n\t$ cscript /nologo test.js\r\n\tHi\r\n\t$ pwd\r\n\t/cygdrive/f/tmp/testpath\r\n\t$ cscript /nologo /cygdrive/f/tmp/testpath/test.js\r\n\tInput Error: Unknown option \"/cygdrive/f/tmp/testpath/test.js\" specified.\r\n\t$ cscript /nologo f:/tmp/testpath/test.js\r\n\tHi\r\n\t$\r\n\r\nRecommendation is to always use relative paths.\r\n\r\n## Changing the shell in make\r\n\r\nThe `.SHELLFLAGS` variable doesn't work in the context of Rtools (silently ignored), making it impossible to change default shell to cmd: the default for SHELLFLAGS is `-c`, which is suitable for `sh` and not `cmd`:\r\n\r\n\r\n\t$ cmd /c dir /b notfound\r\n\tFile Not Found\r\n\t$ cmd -c dir /b notfound\r\n\tMicrosoft Windows [Version 6.3.9600]\r\n\t(c) 2013 Microsoft Corporation. All rights reserved.\r\n\r\n\tF:\\tmp\\testpath>exit\r\n\r\n\t$ cat Makefile\r\n\tSHELL=cmd\r\n\t.SHELLFLAGS=/c\r\n\tTEST=$(shell dir /b notfound)\r\n\r\n\t.PHONY: all\r\n\tall:\r\n @echo hi\r\n\r\n\t$ make\r\n\tMicrosoft Windows [Version 6.3.9600]\r\n\t(c) 2013 Microsoft Corporation. All rights reserved.\r\n\r\n\tF:\\tmp\\testpath>exit\r\n\t\r\n\r\nThe Rtools-only `--win32` command line option for make didn't properly work for me (but I was dealing with complex Makefiles).\r\n\r\n## Hangs in make\r\n\r\nThere does not appear to be a built-in tracing or time-out mechanism in make regarding sub-processes. One effective way to figure out what's hanging is \"Task Manager\", \"Details\", then add column \"Command Line\".\r\nIn my case, most hangs were due to some variation of `cmd -c something` or `cmd c: something` (both variations end up leaving cmd.exe running) instead of `cmd /c something`. The first one is due to SHELLFLAGS not working, the second to auto param replacement.\r\n\r\n## Rule not found errors\r\n\r\ni.e. \"no rule to make target\" errors. \r\nIn some cases, the `VPATH` parser gets confused, and then **all** paths specified are *silently* ignored. This can even happen for simple (no spaces, etc.), valid (exist) paths.\r\n\r\n\t$ cat Makefile\r\n\t.PHONY: all\r\n\tall: test.o foo.o\r\n\t\t@echo hi\r\n\r\n\t%.o: %.cpp\r\n\t\t@echo $<\r\n\t$ pwd\r\n\t/cygdrive/f/tmp/testpath\r\n\t$ make \"VPATH=f:/tmp/testpath/subdir\"\r\n\ttest.cpp\r\n\t/cygdrive/f/tmp/testpath/subdir/foo.cpp\r\n\thi\r\n\t$ make \"VPATH=f:/tmp/testpath subdir\"\r\n\ttest.cpp\r\n\tsubdir/foo.cpp\r\n\thi\r\n\t$ make \"VPATH=subdir f:/tmp/testpath\"\r\n\ttest.cpp\r\n\tsubdir/foo.cpp\r\n\thi\r\n\t$ make \"VPATH=f:/tmp/testpath f:/tmp/testpath/subdir\"\r\n\ttest.cpp\r\n\tmake: *** No rule to make target `foo.o', needed by `all'. Stop.\r\n\r\n\r\n### Incorrect rule firing\r\n\r\nThese can be caused by the previous issue: VPATH not working, therefore target file not found, therefore other rule firing.\r\n\r\nThis can also be caused by MinGW and RTools using a different version of make. A quick way to test it is to manually/explicitly create the rule, and see if it is firing. \r\n\r\n\r\n\r\n" }, { "path": "inst/.gitignore", "content": "lib/\nlibs/\n" }, { "path": "inst/include/.gitignore", "content": "\n# These TBB libraries are copied in at configure time.\n/index.html\n/oneapi\n/serial\n/tbb\n\n" }, { "path": "inst/include/RcppParallel/Backend.h", "content": "\n#ifndef __RCPP_PARALLEL_BACKEND__\n#define __RCPP_PARALLEL_BACKEND__\n\n#include \n#include \n\nextern \"C\" {\nvoid REprintf(const char*, ...);\n}\n\nnamespace RcppParallel {\nnamespace internal {\n\nenum backend_type {\n BACKEND_TBB,\n BACKEND_TINYTHREAD\n};\n\n#if RCPP_PARALLEL_USE_TBB\n\ninline backend_type defaultBackend()\n{\n return BACKEND_TBB;\n}\n\n#else\n\ninline backend_type defaultBackend()\n{\n return BACKEND_TINYTHREAD;\n}\n\n#endif\n\ninline const char* backendToString(backend_type backend)\n{\n switch (backend)\n {\n case BACKEND_TBB:\n return \"tbb\";\n case BACKEND_TINYTHREAD:\n return \"tinythread\";\n }\n\n // shouldn't be reached but need to silence compiler warnings\n return \"tbb\";\n}\n\ninline backend_type backend()\n{\n const char* requestedBackend = std::getenv(\"RCPP_PARALLEL_BACKEND\");\n if (requestedBackend == NULL)\n {\n return defaultBackend();\n }\n else if (std::strcmp(requestedBackend, \"tbb\") == 0)\n {\n#if RCPP_PARALLEL_USE_TBB\n return BACKEND_TBB;\n#else\n const char* msg =\n \"tbb backend is not available; using tinythread instead\";\n \n REprintf(\"%s\\n\", msg);\n return BACKEND_TINYTHREAD;\n#endif\n }\n else if (strcmp(requestedBackend, \"tinythread\") == 0)\n {\n return BACKEND_TINYTHREAD;\n }\n else\n {\n const char* fmt = \"unknown parallel backend '%s'; using '%s' instead\\n\";\n REprintf(fmt, requestedBackend, backendToString(defaultBackend()));\n return defaultBackend();\n }\n}\n\n} // namespace internal\n} // namespace RcppParallel\n\n#endif /* __RCPP_PARALLEL_BACKEND__ */\n" }, { "path": "inst/include/RcppParallel/Common.h", "content": "#ifndef __RCPP_PARALLEL_COMMON__\n#define __RCPP_PARALLEL_COMMON__\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace RcppParallel {\n\ntemplate \ninline int resolveValue(const char* envvar,\n T requestedValue,\n U defaultValue)\n{\n // if the requested value is non-zero and not the default, we can use it\n bool useRequestedValue =\n requestedValue != static_cast(defaultValue) &&\n requestedValue > 0;\n \n if (useRequestedValue)\n return requestedValue;\n\n // otherwise, try reading the default from associated envvar\n // if the environment variable is unset, use the default\n const char* var = getenv(envvar);\n if (var == NULL)\n return defaultValue;\n\n // try to convert the string to a number\n // if an error occurs during conversion, just use default\n errno = 0;\n char* end;\n long value = strtol(var, &end, 10);\n\n // check for conversion failure\n if (end == var || *end != '\\0' || errno == ERANGE)\n return defaultValue;\n\n // okay, return the parsed environment variable value\n return value;\n}\n\n// Tag type used for disambiguating splitting constructors\nstruct Split {};\n\n// Work executed within a background thread. We implement dynamic\n// dispatch using vtables so we can have a stable type to cast\n// to from the void* passed to the worker thread (required because\n// the tinythreads interface allows to pass only a void* to the\n// thread main rather than a generic type / template)\nstruct Worker\n{\n // construct and destruct (delete virtually)\n Worker() {}\n virtual ~Worker() {}\n\n // dispatch work over a range of values\n virtual void operator()(std::size_t begin, std::size_t end) = 0;\n\nprivate:\n // disable copying and assignment\n Worker(const Worker&);\n void operator=(const Worker&);\n};\n\n// Used for controlling the stack size for threads / tasks within a scope.\nclass ThreadStackSizeControl\n{\npublic:\n ThreadStackSizeControl();\n ~ThreadStackSizeControl();\n\nprivate:\n // COPYING: not copyable\n ThreadStackSizeControl(const ThreadStackSizeControl&);\n ThreadStackSizeControl& operator=(const ThreadStackSizeControl&);\n};\n\n\n} // namespace RcppParallel\n\n\n#endif // __RCPP_PARALLEL_COMMON__\n" }, { "path": "inst/include/RcppParallel/RMatrix.h", "content": "#ifndef __RCPP_PARALLEL_RMATRIX__\n#define __RCPP_PARALLEL_RMATRIX__\n\n#include \n#include \n\nnamespace RcppParallel {\n\ntemplate \nclass RMatrix {\npublic:\n class Row {\n \n public: \n \n template \n class row_iterator {\n \n public:\n using iterator_category = std::random_access_iterator_tag;\n using value_type = V;\n using difference_type = std::size_t;\n using pointer = value_type*;\n using reference = value_type&;\n\n inline row_iterator(Row& row, difference_type i)\n : start_(row.start_), parentNrow_(row.parent_.nrow()), index_(i)\n {\n }\n \n inline row_iterator(pointer start, difference_type parentNrow, difference_type index)\n : start_(start), parentNrow_(parentNrow), index_(index)\n { \n }\n \n inline row_iterator(const row_iterator& other) \n : start_(other.start_), \n parentNrow_(other.parentNrow_), \n index_(other.index_)\n {\n }\n \n inline row_iterator& operator++() { \n index_++;\n return *this;\n }\n \n inline row_iterator operator++(int) {\n row_iterator tmp(*this); \n operator++(); \n return tmp;\n }\n \n inline row_iterator& operator--() {\n index_-- ;\n return *this ;\n }\n \n inline row_iterator operator--(int) {\n row_iterator tmp(*this);\n index_-- ;\n return tmp ;\n }\n\n row_iterator operator+(difference_type n) const {\n return row_iterator(start_, parentNrow_ ,index_ + n ) ; \n }\n row_iterator operator-(difference_type n) const {\n return row_iterator(start_, parentNrow_, index_ - n ) ; \n }\n \n difference_type operator+(const row_iterator& other) const {\n return index_ + other.index_;\n }\n \n difference_type operator-(const row_iterator& other) const {\n return index_ - other.index_ ; \n }\n \n row_iterator& operator+=(difference_type n) { index_ += n ; return *this; }\n row_iterator& operator-=(difference_type n) { index_ -= n ; return *this; }\n \n bool operator==(const row_iterator& other) const { return index_ == other.index_; }\n bool operator!=(const row_iterator& other) const { return index_ != other.index_; }\n bool operator<(const row_iterator& other) const { return index_ < other.index_; }\n bool operator>(const row_iterator& other) const { return index_ > other.index_; }\n bool operator<=(const row_iterator& other) const { return index_ <= other.index_; }\n bool operator>=(const row_iterator& other) const { return index_ >= other.index_; }\n \n\n inline reference operator*() { return start_[index_ * parentNrow_]; }\n \n inline pointer operator->() { return &(start_[index_ * parentNrow_]); }\n \n inline reference operator[](int i) { return start_[(index_+i) * parentNrow_]; }\n \n private:\n pointer start_;\n difference_type parentNrow_;\n difference_type index_;\n };\n \n typedef row_iterator iterator;\n typedef row_iterator const_iterator;\n \n inline Row(RMatrix& parent, std::size_t i)\n : parent_(parent),\n start_(parent.begin() + i)\n {\n }\n \n inline Row(const Row& other)\n : parent_(other.parent_),\n start_(other.start_)\n { \n }\n \n inline iterator begin() {\n return iterator(*this, 0);\n }\n \n inline iterator end() {\n return iterator(*this, parent_.ncol());\n }\n \n inline const_iterator begin() const {\n return const_iterator(*this, 0);\n }\n \n inline const_iterator end() const {\n return const_iterator(*this, parent_.ncol());\n }\n \n inline size_t length() const {\n return parent_.ncol();\n }\n\n inline size_t size() const {\n return parent_.ncol();\n }\n\n inline T& operator[](std::size_t i) {\n return start_[i * parent_.nrow()];\n }\n \n inline const T& operator[](std::size_t i) const {\n return start_[i * parent_.nrow()];\n }\n \n private:\n RMatrix& parent_;\n T* start_;\n };\n \n class Column {\n \n public:\n \n typedef T* iterator;\n typedef const T* const_iterator;\n \n inline Column(RMatrix& parent, std::size_t i) \n : begin_(parent.begin() + (i * parent.nrow())),\n end_(begin_ + parent.nrow())\n { \n }\n \n inline Column(const Column& other) \n : begin_(other.begin_), end_(other.end_)\n { \n }\n \n inline Column& operator=(const Column& rhs) {\n begin_ = rhs.begin_;\n end_ = rhs.end_;\n return *this;\n }\n \n inline iterator begin() { return begin_; }\n inline iterator end() { return end_; }\n \n inline const_iterator begin() const { return begin_; }\n inline const_iterator end() const { return end_; }\n \n inline size_t length() const { return end_ - begin_; }\n inline size_t size() const { return end_ - begin_; }\n \n inline T& operator[](std::size_t i) {\n return *(begin_ + i);\n }\n \n inline const T& operator[](std::size_t i) const {\n return *(begin_ + i);\n }\n \n private:\n T* begin_;\n T* end_;\n };\n\n typedef T* iterator;\n typedef const T* const_iterator;\n\n template \n inline explicit RMatrix(const Source& source) \n : data_(const_cast(source).begin()),\n nrow_(source.nrow()),\n ncol_(source.ncol())\n {\n }\n\n inline RMatrix(T* data, std::size_t nrow, std::size_t ncol) \n : data_(data), nrow_(nrow), ncol_(ncol) \n {\n }\n \n inline iterator begin() { return data_; }\n inline iterator end() { return data_ + length(); }\n \n inline const_iterator begin() const { return data_; }\n inline const_iterator end() const { return data_ + length(); }\n \n inline std::size_t length() const { return nrow_ * ncol_; } \n \n inline std::size_t nrow() const { return nrow_; }\n inline std::size_t ncol() const { return ncol_; }\n \n inline T& operator()(std::size_t i, std::size_t j) {\n return *(data_ + (i + j * nrow_));\n }\n \n inline const T& operator()(std::size_t i, std::size_t j) const {\n return *(data_ + (i + j * nrow_));\n }\n \n inline Row row(std::size_t i) {\n return Row(*this, i);\n }\n \n inline const Row row(std::size_t i) const {\n return Row(*const_cast(this), i);\n }\n \n inline Column column(std::size_t i) {\n return Column(*this, i);\n }\n \n inline const Column column(std::size_t i) const {\n return Column(*const_cast(this), i);\n }\n \n inline T& operator[](std::size_t i) {\n return *(data_ + i);\n }\n \n inline const T& operator[](std::size_t i) const {\n return *(data_ + i);\n }\n \nprivate:\n T* data_;\n std::size_t nrow_;\n std::size_t ncol_;\n};\n\n} // namespace RcppParallel\n\n#endif // __RCPP_PARALLEL_RMATRIX__\n" }, { "path": "inst/include/RcppParallel/RVector.h", "content": "#ifndef __RCPP_PARALLEL_RVECTOR__\n#define __RCPP_PARALLEL_RVECTOR__\n\n#include \n\nnamespace RcppParallel {\n\ntemplate \nclass RVector {\n\npublic: \n typedef T* iterator;\n typedef const T* const_iterator;\n\n template \n inline explicit RVector(const Source& source) \n : begin_(const_cast(source).begin()),\n end_(begin_ + source.length())\n\n {\n }\n\n inline RVector(std::size_t begin, std::size_t end) \n : begin_(begin), end_(end)\n { \n }\n \n inline RVector(const RVector& other) \n : begin_(other.begin_), end_(other.end_)\n { \n }\n \n inline RVector& operator=(const RVector& rhs) {\n begin_ = rhs.begin_;\n end_ = rhs.end_;\n return *this;\n }\n \n inline iterator begin() { return begin_; }\n inline iterator end() { return end_; }\n \n inline const_iterator begin() const { return begin_; }\n inline const_iterator end() const { return end_; }\n \n inline std::size_t size() const { return end_ - begin_; }\n inline std::size_t length() const { return end_ - begin_; }\n \n inline T& operator[](std::size_t i) {\n return *(begin_ + i);\n }\n \n inline const T& operator[](std::size_t i) const {\n return *(begin_ + i);\n }\n \nprivate:\n T* begin_;\n T* end_; \n};\n\n} // namespace RcppParallel\n\n#endif // __RCPP_PARALLEL_RVECTOR__\n" }, { "path": "inst/include/RcppParallel/TBB.h", "content": "#ifndef __RCPP_PARALLEL_TBB__\n#define __RCPP_PARALLEL_TBB__\n\n#include \"Common.h\"\n\n#ifndef TBB_PREVIEW_GLOBAL_CONTROL\n# define TBB_PREVIEW_GLOBAL_CONTROL 1\n#endif\n\n// For compatibility with existing packages on CRAN.\n#include \"tbb/blocked_range.h\"\n#include \"tbb/concurrent_unordered_set.h\"\n#include \"tbb/concurrent_unordered_map.h\"\n#include \"tbb/global_control.h\"\n#include \"tbb/mutex.h\"\n#include \"tbb/parallel_for.h\"\n#include \"tbb/parallel_for_each.h\"\n#include \"tbb/parallel_reduce.h\"\n#include \"tbb/parallel_sort.h\"\n#include \"tbb/spin_mutex.h\"\n\n// For compatibility with older R packages.\nnamespace tbb {\n\n#ifndef __TBB_task_scheduler_init_H\n#define __TBB_task_scheduler_init_H\n\nclass task_scheduler_init {\n \npublic:\n task_scheduler_init(\n int number_of_threads = -1,\n std::size_t stack_size = 0)\n {\n }\n \n static int default_num_threads()\n {\n return 2;\n }\n \n static const int automatic = -1;\n static const int deferred = -2;\n \n};\n\n#endif\n\n} // end namespace tbb\n\n\nnamespace RcppParallel {\n\n// This class is primarily used to implement type erasure. The goals here were:\n//\n// 1. Hide the tbb symbols / implementation details from client R packages.\n// That is, they should get the tools they need only via RcppParallel.\n//\n// 2. Do this in a way that preserves binary compatibility with pre-existing\n// classes that make use of parallelReduce().\n//\n// 3. Ensure that those packages, when re-compiled without source changes,\n// can still function as expected.\n//\n// The downside here is that all the indirection through std::function<>\n// and the requirement for RTTI is probably expensive, but I couldn't find\n// a better way forward that could also preserve binary compatibility with\n// existing pre-built pacakges.\n//\n// Hopefully, in a future release, we can do away with this wrapper, once\n// packages have been rebuilt and no longer implicitly depend on TBB internals.\nstruct ReducerWrapper {\n\n template \n ReducerWrapper(T* reducer)\n {\n self_ = reinterpret_cast(reducer);\n owned_ = false;\n\n work_ = [&](void* self, std::size_t begin, std::size_t end)\n {\n (*reinterpret_cast(self))(begin, end);\n };\n\n split_ = [&](void* object, Split split)\n {\n return new T(*reinterpret_cast(object), split);\n };\n\n join_ = [&](void* self, void* other)\n {\n (*reinterpret_cast(self)).join(*reinterpret_cast(other));\n };\n\n deleter_ = [&](void* object)\n {\n delete (T*) object;\n };\n }\n\n ~ReducerWrapper()\n {\n if (owned_)\n {\n deleter_(self_);\n self_ = nullptr;\n }\n }\n\n void operator()(std::size_t begin, std::size_t end) const\n {\n work_(self_, begin, end);\n }\n\n ReducerWrapper(const ReducerWrapper& rhs, Split split)\n {\n self_ = rhs.split_(rhs.self_, split);\n owned_ = true;\n\n work_ = rhs.work_;\n split_ = rhs.split_;\n join_ = rhs.join_;\n deleter_ = rhs.deleter_;\n }\n\n void join(const ReducerWrapper& rhs) const\n {\n join_(self_, rhs.self_);\n }\n\nprivate:\n void* self_ = nullptr;\n bool owned_ = false;\n\n std::function work_;\n std::function split_;\n std::function join_;\n std::function deleter_;\n};\n\nvoid tbbParallelFor(std::size_t begin,\n std::size_t end,\n Worker& worker,\n std::size_t grainSize = 1,\n int numThreads = -1);\n\nvoid tbbParallelReduceImpl(std::size_t begin,\n std::size_t end,\n ReducerWrapper& wrapper,\n std::size_t grainSize = 1,\n int numThreads = -1);\n\ntemplate \nvoid tbbParallelReduce(std::size_t begin,\n std::size_t end,\n Reducer& reducer,\n std::size_t grainSize = 1,\n int numThreads = -1)\n{\n ReducerWrapper wrapper(&reducer);\n tbbParallelReduceImpl(begin, end, wrapper, grainSize, numThreads);\n}\n\n} // namespace RcppParallel\n\n#endif // __RCPP_PARALLEL_TBB__\n" }, { "path": "inst/include/RcppParallel/Timer.h", "content": "#ifndef __RCPP_PARALLEL_TIMER__\n#define __RCPP_PARALLEL_TIMER__\n\nnamespace RcppParallel {\n typedef uint64_t nanotime_t;\n\n template \n class ProportionTimer {\n public:\n ProportionTimer() : \n timer(), n(0), id(0)\n {}\n \n ProportionTimer( nanotime_t origin, int id_ ) : \n timer(origin), n(0), id(id_)\n {}\n \n inline operator SEXP() const {\n Rcpp::NumericVector out = (SEXP)timer ;\n out.attr(\"n\") = n ;\n return out ;\n }\n \n inline nanotime_t origin() const{\n return timer.origin() ;\n }\n \n inline int get_n() const {\n return n ; \n }\n \n inline void step( const std::string& name) {\n timer.step(name) ; \n }\n \n Timer timer ;\n int n ;\n int id ;\n \n } ;\n \n template \n class SingleTimer {\n public:\n SingleTimer() : timer(){}\n \n inline operator SEXP(){\n Rcpp::List out = Rcpp::List::create(timer) ;\n out.attr(\"class\") = Rcpp::CharacterVector::create( \"SingleTimer\", \"Timer\" );\n return out ;\n }\n \n inline void step( const char* name ){\n timer.step(name) ; \n }\n \n private:\n Timer timer ;\n } ;\n \n template \n class FixedSizeTimers {\n public:\n FixedSizeTimers( int n, int ndata_ ) : \n timers(n), ndata(ndata_)\n {}\n \n inline ProportionTimer& get(int i) {\n return timers[i] ;\n }\n \n inline ProportionTimer& operator[](int i){ \n return timers[i]; \n }\n \n inline operator SEXP(){\n Rcpp::List out = wrap(timers) ;\n out.attr(\"class\") = Rcpp::CharacterVector::create(\"FixedSizeTimers\", \"Timer\") ;\n out.attr(\"n\") = ndata ;\n return out ;\n }\n \n private:\n std::vector< ProportionTimer > timers ;\n int ndata ;\n } ;\n \n template \n class TimersList {\n public:\n \n TimersList(int n_): \n timers(), origin(Rcpp::get_nanotime()), n(n_)\n {\n timers.push_back( ProportionTimer( origin, 0 ) ) ;\n childs.push_back( std::vector() );\n }\n \n ProportionTimer& front(){\n return timers.front() ;\n }\n \n ProportionTimer& get_new_timer(int parent_id){\n Locker lock(mutex) ;\n int id = timers.size() ;\n timers.push_back( ProportionTimer( origin, id ) ) ;\n childs.push_back( std::vector() );\n std::list< std::vector >::iterator it = childs.begin() ;\n for(int i=0; ipush_back(id) ;\n return timers.back() ;\n }\n \n inline operator SEXP(){\n int nt = timers.size() ;\n Rcpp::List data(nt) ;\n typename std::list >::const_iterator timers_it = timers.begin() ;\n std::list >::const_iterator childs_it = childs.begin() ;\n for( int i=0; i > timers ;\n std::list< std::vector > childs ;\n Mutex mutex ;\n nanotime_t origin ;\n int n ;\n \n private:\n TimersList( const TimersList& ) ;\n \n } ;\n \n template \n class TimedReducer : public Worker {\n public:\n typedef TimersList Timers ; \n \n Reducer* reducer ;\n Timers& timers ;\n ProportionTimer& timer ;\n bool owner ;\n \n TimedReducer( Reducer& reducer_, Timers& timers_) : \n reducer(&reducer_), \n timers(timers_), \n timer(timers.get_new_timer(0)), \n owner(false)\n {\n timer.step(\"init\");\n }\n \n TimedReducer( const TimedReducer& other, Split s) : \n reducer( new Reducer(*other.reducer, s) ), \n timers( other.timers ), \n timer( timers.get_new_timer(other.timer.id) ),\n owner(true)\n {\n timer.step(\"init\") ;\n }\n \n ~TimedReducer(){\n if(owner && reducer) {\n delete reducer ;\n }\n reducer = 0 ;\n }\n \n inline void operator()( size_t begin, size_t end){\n timer.n += (end-begin) ;\n timer.step(\"start\") ;\n reducer->operator()(begin, end) ;\n timer.step(\"work\") ;\n }\n \n inline void join(const TimedReducer& rhs){\n rhs.timer.step(\"start\") ;\n reducer->join(*rhs.reducer) ;\n rhs.timer.step(\"join\") ;\n }\n \n inline Rcpp::List get() const {\n timers.front().step(\"start\") ;\n OUT out = reducer->get() ;\n timers.front().step(\"structure\") ;\n \n Rcpp::List res = Rcpp::List::create( (SEXP)timers, out );\n return res ;\n }\n \n private:\n // just to be on the safe side, making sure these are not called\n TimedReducer() ;\n TimedReducer( const TimedReducer& ) ;\n } ;\n \n \n} // namespace RcppParallel\n\n\n#endif // __RCPP_PARALLEL_COMMON__\n" }, { "path": "inst/include/RcppParallel/TinyThread.h", "content": "#ifndef __RCPP_PARALLEL_TINYTHREAD__\n#define __RCPP_PARALLEL_TINYTHREAD__\n\n#include \n#include \n\n#include \"Common.h\"\n\n#include \n\n\n#include \n\nnamespace RcppParallel {\n\nnamespace {\n\n// Class which represents a range of indexes to perform work on\n// (worker functions are passed this range so they know which\n// elements are safe to read/write to)\nclass IndexRange {\npublic:\n\n // Initizlize with a begin and (exclusive) end index\n IndexRange(std::size_t begin, std::size_t end)\n : begin_(begin), end_(end)\n {\n }\n \n // Access begin() and end()\n std::size_t begin() const { return begin_; }\n std::size_t end() const { return end_; }\n std::size_t size() const { return end_ - begin_ ; }\n \nprivate:\n std::size_t begin_;\n std::size_t end_;\n};\n\n\n// Because tinythread allows us to pass only a plain C function\n// we need to pass our worker and range within a struct that we \n// can cast to/from void*\nstruct Work {\n Work(IndexRange range, Worker& worker) \n : range(range), worker(worker)\n {\n }\n IndexRange range;\n Worker& worker;\n};\n\n// Thread which performs work (then deletes the work object\n// when it's done)\nextern \"C\" inline void workerThread(void* data) {\n try\n {\n Work* pWork = static_cast(data);\n pWork->worker(pWork->range.begin(), pWork->range.end());\n delete pWork;\n }\n catch(...)\n {\n }\n}\n\n// Function to calculate the ranges for a given input\nstd::vector splitInputRange(const IndexRange& range,\n std::size_t grainSize) {\n \n // determine max number of threads\n std::size_t threads = tthread::thread::hardware_concurrency();\n char* numThreads = ::getenv(\"RCPP_PARALLEL_NUM_THREADS\");\n if (numThreads != NULL) {\n int parsedThreads = ::atoi(numThreads);\n if (parsedThreads > 0)\n threads = parsedThreads;\n }\n \n // compute grainSize (including enforcing requested minimum)\n std::size_t length = range.end() - range.begin();\n if (threads == 1)\n grainSize = length;\n else if ((length % threads) == 0) // perfect division\n grainSize = std::max(length / threads, grainSize);\n else // imperfect division, divide by threads - 1\n grainSize = std::max(length / (threads-1), grainSize);\n \n // allocate ranges\n std::vector ranges;\n std::size_t begin = range.begin();\n std::size_t end = begin;\n while (begin < range.end()) {\n if ((range.end() - (begin + grainSize)) < grainSize)\n end = range.end();\n else\n end = std::min(begin + grainSize, range.end());\n\n ranges.push_back(IndexRange(begin, end));\n begin = end;\n }\n \n // return ranges \n return ranges;\n}\n\n} // anonymous namespace\n\n// Execute the Worker over the IndexRange in parallel\ninline void ttParallelFor(std::size_t begin,\n std::size_t end, \n Worker& worker,\n std::size_t grainSize = 1)\n{\n // split the work\n IndexRange inputRange(begin, end);\n std::vector ranges = splitInputRange(inputRange, grainSize);\n \n // create threads\n std::vector threads;\n for (std::size_t i = 0; ijoin();\n delete threads[i];\n }\n}\n\n// Execute the IWorker over the range in parallel then join results\ntemplate \ninline void ttParallelReduce(std::size_t begin,\n std::size_t end, \n Reducer& reducer,\n std::size_t grainSize = 1)\n{\n // split the work\n IndexRange inputRange(begin, end);\n std::vector ranges = splitInputRange(inputRange, grainSize);\n \n // create threads (split for each thread and track the allocated workers)\n std::vector threads;\n std::vector workers;\n for (std::size_t i = 0; ijoin();\n \n // join the results\n reducer.join(static_cast(*workers[i]));\n \n // delete the worker (which we split above) and the thread\n delete workers[i];\n delete threads[i];\n }\n}\n\n} // namespace RcppParallel\n\n#endif // __RCPP_PARALLEL_TINYTHREAD__\n" }, { "path": "inst/include/RcppParallel.h", "content": "\n#ifndef __RCPP_PARALLEL__\n#define __RCPP_PARALLEL__\n\n// TinyThread implementation\n#include \"RcppParallel/TinyThread.h\"\n\n// Use TBB only where it's known to compile and work correctly\n// (NOTE: Windows TBB is temporarily opt-in for packages for\n// compatibility with CRAN packages not previously configured\n// to link to TBB in Makevars.win)\n#ifndef RCPP_PARALLEL_USE_TBB\n# if defined(__APPLE__) || defined(__gnu_linux__) || (defined(__sun) && defined(__SVR4) && !defined(__sparc))\n# define RCPP_PARALLEL_USE_TBB 1\n# else\n# define RCPP_PARALLEL_USE_TBB 0\n# endif\n#endif\n\n#if RCPP_PARALLEL_USE_TBB\n# include \"RcppParallel/TBB.h\"\n#endif\n\n#include \"RcppParallel/Backend.h\"\n#include \"RcppParallel/RVector.h\"\n#include \"RcppParallel/RMatrix.h\"\n\nnamespace RcppParallel {\n\ninline void parallelFor(std::size_t begin,\n std::size_t end,\n Worker& worker,\n std::size_t grainSize = 1,\n int numThreads = -1)\n{\n grainSize = resolveValue(\"RCPP_PARALLEL_GRAIN_SIZE\", grainSize, std::size_t(1));\n numThreads = resolveValue(\"RCPP_PARALLEL_NUM_THREADS\", numThreads, -1);\n\n#if RCPP_PARALLEL_USE_TBB\n if (internal::backend() == internal::BACKEND_TBB)\n tbbParallelFor(begin, end, worker, grainSize, numThreads);\n else\n ttParallelFor(begin, end, worker, grainSize);\n#else\n ttParallelFor(begin, end, worker, grainSize);\n#endif\n}\n\ntemplate \ninline void parallelReduce(std::size_t begin,\n std::size_t end,\n Reducer& reducer,\n std::size_t grainSize = 1,\n int numThreads = -1)\n{\n grainSize = resolveValue(\"RCPP_PARALLEL_GRAIN_SIZE\", grainSize, std::size_t(1));\n numThreads = resolveValue(\"RCPP_PARALLEL_NUM_THREADS\", numThreads, -1);\n\n#if RCPP_PARALLEL_USE_TBB\n if (internal::backend() == internal::BACKEND_TBB)\n tbbParallelReduce(begin, end, reducer, grainSize, numThreads);\n else\n ttParallelReduce(begin, end, reducer, grainSize);\n#else\n ttParallelReduce(begin, end, reducer, grainSize);\n#endif\n}\n\n} // end namespace RcppParallel\n\n// TRUE and FALSE macros that may come with system headers on some systems\n// But conflict with R.h (R_ext/Boolean.h)\n// TRUE and FALSE macros should be undef in RcppParallel.h\n#ifdef TRUE\n #undef TRUE\n#endif\n#ifdef FALSE\n #undef FALSE\n#endif\n\n#endif // __RCPP_PARALLEL__\n" }, { "path": "inst/include/tthread/fast_mutex.h", "content": "/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*-\nCopyright (c) 2010-2012 Marcus Geelnard\n\nThis software is provided 'as-is', without any express or implied\nwarranty. In no event will the authors be held liable for any damages\narising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose,\nincluding commercial applications, and to alter it and redistribute it\nfreely, subject to the following restrictions:\n\n 1. The origin of this software must not be misrepresented; you must not\n claim that you wrote the original software. If you use this software\n in a product, an acknowledgment in the product documentation would be\n appreciated but is not required.\n\n 2. Altered source versions must be plainly marked as such, and must not be\n misrepresented as being the original software.\n\n 3. This notice may not be removed or altered from any source\n distribution.\n*/\n\n#ifndef _FAST_MUTEX_H_\n#define _FAST_MUTEX_H_\n\n/// @file\n\n// Which platform are we on?\n#if !defined(_TTHREAD_PLATFORM_DEFINED_)\n #if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__)\n #define _TTHREAD_WIN32_\n #else\n #define _TTHREAD_POSIX_\n #endif\n #define _TTHREAD_PLATFORM_DEFINED_\n#endif\n\n// Check if we can support the assembly language level implementation (otherwise\n// revert to the system API)\n#if (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || \\\n (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))) || \\\n (defined(__GNUC__) && (defined(__ppc__)))\n #define _FAST_MUTEX_ASM_\n#else\n #define _FAST_MUTEX_SYS_\n#endif\n\n#if defined(_TTHREAD_WIN32_)\n #ifndef WIN32_LEAN_AND_MEAN\n #define WIN32_LEAN_AND_MEAN\n #define __UNDEF_LEAN_AND_MEAN\n #endif\n #include \n #ifdef __UNDEF_LEAN_AND_MEAN\n #undef WIN32_LEAN_AND_MEAN\n #undef __UNDEF_LEAN_AND_MEAN\n #endif\n#else\n #ifdef _FAST_MUTEX_ASM_\n #include \n #else\n #include \n #endif\n#endif\n\nnamespace tthread {\n\n/// Fast mutex class.\n/// This is a mutual exclusion object for synchronizing access to shared\n/// memory areas for several threads. It is similar to the tthread::mutex class,\n/// but instead of using system level functions, it is implemented as an atomic\n/// spin lock with very low CPU overhead.\n///\n/// The \\c fast_mutex class is NOT compatible with the \\c condition_variable\n/// class (however, it IS compatible with the \\c lock_guard class). It should\n/// also be noted that the \\c fast_mutex class typically does not provide\n/// as accurate thread scheduling as a the standard \\c mutex class does.\n///\n/// Because of the limitations of the class, it should only be used in\n/// situations where the mutex needs to be locked/unlocked very frequently.\n///\n/// @note The \"fast\" version of this class relies on inline assembler language,\n/// which is currently only supported for 32/64-bit Intel x86/AMD64 and\n/// PowerPC architectures on a limited number of compilers (GNU g++ and MS\n/// Visual C++).\n/// For other architectures/compilers, system functions are used instead.\nclass fast_mutex {\n public:\n /// Constructor.\n#if defined(_FAST_MUTEX_ASM_)\n fast_mutex() : mLock(0) {}\n#else\n fast_mutex()\n {\n #if defined(_TTHREAD_WIN32_)\n InitializeCriticalSection(&mHandle);\n #elif defined(_TTHREAD_POSIX_)\n pthread_mutex_init(&mHandle, NULL);\n #endif\n }\n#endif\n\n#if !defined(_FAST_MUTEX_ASM_)\n /// Destructor.\n ~fast_mutex()\n {\n #if defined(_TTHREAD_WIN32_)\n DeleteCriticalSection(&mHandle);\n #elif defined(_TTHREAD_POSIX_)\n pthread_mutex_destroy(&mHandle);\n #endif\n }\n#endif\n\n /// Lock the mutex.\n /// The method will block the calling thread until a lock on the mutex can\n /// be obtained. The mutex remains locked until \\c unlock() is called.\n /// @see lock_guard\n inline void lock()\n {\n#if defined(_FAST_MUTEX_ASM_)\n bool gotLock;\n do {\n gotLock = try_lock();\n if(!gotLock)\n {\n #if defined(_TTHREAD_WIN32_)\n Sleep(0);\n #elif defined(_TTHREAD_POSIX_)\n sched_yield();\n #endif\n }\n } while(!gotLock);\n#else\n #if defined(_TTHREAD_WIN32_)\n EnterCriticalSection(&mHandle);\n #elif defined(_TTHREAD_POSIX_)\n pthread_mutex_lock(&mHandle);\n #endif\n#endif\n }\n\n /// Try to lock the mutex.\n /// The method will try to lock the mutex. If it fails, the function will\n /// return immediately (non-blocking).\n /// @return \\c true if the lock was acquired, or \\c false if the lock could\n /// not be acquired.\n inline bool try_lock()\n {\n#if defined(_FAST_MUTEX_ASM_)\n int oldLock;\n #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))\n asm volatile (\n \"movl $1,%%eax\\n\\t\"\n \"xchg %%eax,%0\\n\\t\"\n \"movl %%eax,%1\\n\\t\"\n : \"=m\" (mLock), \"=m\" (oldLock)\n :\n : \"%eax\", \"memory\"\n );\n #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))\n int *ptrLock = &mLock;\n __asm {\n mov eax,1\n mov ecx,ptrLock\n xchg eax,[ecx]\n mov oldLock,eax\n }\n #elif defined(__GNUC__) && (defined(__ppc__))\n int newLock = 1;\n asm volatile (\n \"\\n1:\\n\\t\"\n \"lwarx %0,0,%1\\n\\t\"\n \"cmpwi 0,%0,0\\n\\t\"\n \"bne- 2f\\n\\t\"\n \"stwcx. %2,0,%1\\n\\t\"\n \"bne- 1b\\n\\t\"\n \"isync\\n\"\n \"2:\\n\\t\"\n : \"=&r\" (oldLock)\n : \"r\" (&mLock), \"r\" (newLock)\n : \"cr0\", \"memory\"\n );\n #endif\n return (oldLock == 0);\n#else\n #if defined(_TTHREAD_WIN32_)\n return TryEnterCriticalSection(&mHandle) ? true : false;\n #elif defined(_TTHREAD_POSIX_)\n return (pthread_mutex_trylock(&mHandle) == 0) ? true : false;\n #endif\n#endif\n }\n\n /// Unlock the mutex.\n /// If any threads are waiting for the lock on this mutex, one of them will\n /// be unblocked.\n inline void unlock()\n {\n#if defined(_FAST_MUTEX_ASM_)\n #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))\n asm volatile (\n \"movl $0,%%eax\\n\\t\"\n \"xchg %%eax,%0\\n\\t\"\n : \"=m\" (mLock)\n :\n : \"%eax\", \"memory\"\n );\n #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))\n int *ptrLock = &mLock;\n __asm {\n mov eax,0\n mov ecx,ptrLock\n xchg eax,[ecx]\n }\n #elif defined(__GNUC__) && (defined(__ppc__))\n asm volatile (\n \"sync\\n\\t\" // Replace with lwsync where possible?\n : : : \"memory\"\n );\n mLock = 0;\n #endif\n#else\n #if defined(_TTHREAD_WIN32_)\n LeaveCriticalSection(&mHandle);\n #elif defined(_TTHREAD_POSIX_)\n pthread_mutex_unlock(&mHandle);\n #endif\n#endif\n }\n\n private:\n#if defined(_FAST_MUTEX_ASM_)\n int mLock;\n#else\n #if defined(_TTHREAD_WIN32_)\n CRITICAL_SECTION mHandle;\n #elif defined(_TTHREAD_POSIX_)\n pthread_mutex_t mHandle;\n #endif\n#endif\n};\n\n}\n\n#endif // _FAST_MUTEX_H_\n\n" }, { "path": "inst/include/tthread/tinythread.h", "content": "/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*-\nCopyright (c) 2010-2012 Marcus Geelnard\n\nThis software is provided 'as-is', without any express or implied\nwarranty. In no event will the authors be held liable for any damages\narising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose,\nincluding commercial applications, and to alter it and redistribute it\nfreely, subject to the following restrictions:\n\n 1. The origin of this software must not be misrepresented; you must not\n claim that you wrote the original software. If you use this software\n in a product, an acknowledgment in the product documentation would be\n appreciated but is not required.\n\n 2. Altered source versions must be plainly marked as such, and must not be\n misrepresented as being the original software.\n\n 3. This notice may not be removed or altered from any source\n distribution.\n*/\n\n#ifndef _TINYTHREAD_H_\n#define _TINYTHREAD_H_\n\n/// @file\n/// @mainpage TinyThread++ API Reference\n///\n/// @section intro_sec Introduction\n/// TinyThread++ is a minimal, portable implementation of basic threading\n/// classes for C++.\n///\n/// They closely mimic the functionality and naming of the C++11 standard, and\n/// should be easily replaceable with the corresponding std:: variants.\n///\n/// @section port_sec Portability\n/// The Win32 variant uses the native Win32 API for implementing the thread\n/// classes, while for other systems, the POSIX threads API (pthread) is used.\n///\n/// @section class_sec Classes\n/// In order to mimic the threading API of the C++11 standard, subsets of\n/// several classes are provided. The fundamental classes are:\n/// @li tthread::thread\n/// @li tthread::mutex\n/// @li tthread::recursive_mutex\n/// @li tthread::condition_variable\n/// @li tthread::lock_guard\n/// @li tthread::fast_mutex\n///\n/// @section misc_sec Miscellaneous\n/// The following special keywords are available: #thread_local.\n///\n/// For more detailed information (including additional classes), browse the\n/// different sections of this documentation. A good place to start is:\n/// tinythread.h.\n\n// Which platform are we on?\n#if !defined(_TTHREAD_PLATFORM_DEFINED_)\n #if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__)\n #define _TTHREAD_WIN32_\n #else\n #define _TTHREAD_POSIX_\n #endif\n #define _TTHREAD_PLATFORM_DEFINED_\n#endif\n\n// Platform specific includes\n#if defined(_TTHREAD_WIN32_)\n #ifndef WIN32_LEAN_AND_MEAN\n #define WIN32_LEAN_AND_MEAN\n #define __UNDEF_LEAN_AND_MEAN\n #endif\n #include \n #ifdef __UNDEF_LEAN_AND_MEAN\n #undef WIN32_LEAN_AND_MEAN\n #undef __UNDEF_LEAN_AND_MEAN\n #endif\n#else\n #include \n #include \n #include \n #include \n #include \n#endif\n\n// Generic includes\n#include \n\n/// TinyThread++ version (major number).\n#define TINYTHREAD_VERSION_MAJOR 1\n/// TinyThread++ version (minor number).\n#define TINYTHREAD_VERSION_MINOR 1\n/// TinyThread++ version (full version).\n#define TINYTHREAD_VERSION (TINYTHREAD_VERSION_MAJOR * 100 + TINYTHREAD_VERSION_MINOR)\n\n// Do we have a fully featured C++11 compiler?\n#if (__cplusplus > 199711L) || (defined(__STDCXX_VERSION__) && (__STDCXX_VERSION__ >= 201001L))\n #define _TTHREAD_CPP11_\n#endif\n\n// ...at least partial C++11?\n#if defined(_TTHREAD_CPP11_) || defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(__GXX_EXPERIMENTAL_CPP0X__)\n #define _TTHREAD_CPP11_PARTIAL_\n#endif\n\n// Macro for disabling assignments of objects.\n#ifdef _TTHREAD_CPP11_PARTIAL_\n #define _TTHREAD_DISABLE_ASSIGNMENT(name) \\\n name(const name&) = delete; \\\n name& operator=(const name&) = delete;\n#else\n #define _TTHREAD_DISABLE_ASSIGNMENT(name) \\\n name(const name&); \\\n name& operator=(const name&);\n#endif\n\n/// @def thread_local\n/// Thread local storage keyword.\n/// A variable that is declared with the @c thread_local keyword makes the\n/// value of the variable local to each thread (known as thread-local storage,\n/// or TLS). Example usage:\n/// @code\n/// // This variable is local to each thread.\n/// thread_local int variable;\n/// @endcode\n/// @note The @c thread_local keyword is a macro that maps to the corresponding\n/// compiler directive (e.g. @c __declspec(thread)). While the C++11 standard\n/// allows for non-trivial types (e.g. classes with constructors and\n/// destructors) to be declared with the @c thread_local keyword, most pre-C++11\n/// compilers only allow for trivial types (e.g. @c int). So, to guarantee\n/// portable code, only use trivial types for thread local storage.\n/// @note This directive is currently not supported on Mac macOS (it will give\n/// a compiler error), since compile-time TLS is not supported in the Mac macOS\n/// executable format. Also, some older versions of MinGW (before GCC 4.x) do\n/// not support this directive.\n/// @hideinitializer\n\n#if !defined(_TTHREAD_CPP11_) && !defined(thread_local)\n #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__SUNPRO_CC) || defined(__IBMCPP__)\n #define thread_local __thread\n #else\n #define thread_local __declspec(thread)\n #endif\n#endif\n\n\n/// Main name space for TinyThread++.\n/// This namespace is more or less equivalent to the @c std namespace for the\n/// C++11 thread classes. For instance, the tthread::mutex class corresponds to\n/// the std::mutex class.\nnamespace tthread {\n\n/// Mutex class.\n/// This is a mutual exclusion object for synchronizing access to shared\n/// memory areas for several threads. The mutex is non-recursive (i.e. a\n/// program may deadlock if the thread that owns a mutex object calls lock()\n/// on that object).\n/// @see recursive_mutex\nclass mutex {\n public:\n /// Constructor.\n mutex()\n#if defined(_TTHREAD_WIN32_)\n : mAlreadyLocked(false)\n#endif\n {\n#if defined(_TTHREAD_WIN32_)\n InitializeCriticalSection(&mHandle);\n#else\n pthread_mutex_init(&mHandle, NULL);\n#endif\n }\n\n /// Destructor.\n ~mutex()\n {\n#if defined(_TTHREAD_WIN32_)\n DeleteCriticalSection(&mHandle);\n#else\n pthread_mutex_destroy(&mHandle);\n#endif\n }\n\n /// Lock the mutex.\n /// The method will block the calling thread until a lock on the mutex can\n /// be obtained. The mutex remains locked until @c unlock() is called.\n /// @see lock_guard\n inline void lock()\n {\n#if defined(_TTHREAD_WIN32_)\n EnterCriticalSection(&mHandle);\n while(mAlreadyLocked) Sleep(1000); // Simulate deadlock...\n mAlreadyLocked = true;\n#else\n pthread_mutex_lock(&mHandle);\n#endif\n }\n\n /// Try to lock the mutex.\n /// The method will try to lock the mutex. If it fails, the function will\n /// return immediately (non-blocking).\n /// @return @c true if the lock was acquired, or @c false if the lock could\n /// not be acquired.\n inline bool try_lock()\n {\n#if defined(_TTHREAD_WIN32_)\n bool ret = (TryEnterCriticalSection(&mHandle) ? true : false);\n if(ret && mAlreadyLocked)\n {\n LeaveCriticalSection(&mHandle);\n ret = false;\n }\n return ret;\n#else\n return (pthread_mutex_trylock(&mHandle) == 0) ? true : false;\n#endif\n }\n\n /// Unlock the mutex.\n /// If any threads are waiting for the lock on this mutex, one of them will\n /// be unblocked.\n inline void unlock()\n {\n#if defined(_TTHREAD_WIN32_)\n mAlreadyLocked = false;\n LeaveCriticalSection(&mHandle);\n#else\n pthread_mutex_unlock(&mHandle);\n#endif\n }\n\n _TTHREAD_DISABLE_ASSIGNMENT(mutex)\n\n private:\n#if defined(_TTHREAD_WIN32_)\n CRITICAL_SECTION mHandle;\n bool mAlreadyLocked;\n#else\n pthread_mutex_t mHandle;\n#endif\n\n friend class condition_variable;\n};\n\n/// Recursive mutex class.\n/// This is a mutual exclusion object for synchronizing access to shared\n/// memory areas for several threads. The mutex is recursive (i.e. a thread\n/// may lock the mutex several times, as long as it unlocks the mutex the same\n/// number of times).\n/// @see mutex\nclass recursive_mutex {\n public:\n /// Constructor.\n recursive_mutex()\n {\n#if defined(_TTHREAD_WIN32_)\n InitializeCriticalSection(&mHandle);\n#else\n pthread_mutexattr_t attr;\n pthread_mutexattr_init(&attr);\n pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);\n pthread_mutex_init(&mHandle, &attr);\n#endif\n }\n\n /// Destructor.\n ~recursive_mutex()\n {\n#if defined(_TTHREAD_WIN32_)\n DeleteCriticalSection(&mHandle);\n#else\n pthread_mutex_destroy(&mHandle);\n#endif\n }\n\n /// Lock the mutex.\n /// The method will block the calling thread until a lock on the mutex can\n /// be obtained. The mutex remains locked until @c unlock() is called.\n /// @see lock_guard\n inline void lock()\n {\n#if defined(_TTHREAD_WIN32_)\n EnterCriticalSection(&mHandle);\n#else\n pthread_mutex_lock(&mHandle);\n#endif\n }\n\n /// Try to lock the mutex.\n /// The method will try to lock the mutex. If it fails, the function will\n /// return immediately (non-blocking).\n /// @return @c true if the lock was acquired, or @c false if the lock could\n /// not be acquired.\n inline bool try_lock()\n {\n#if defined(_TTHREAD_WIN32_)\n return TryEnterCriticalSection(&mHandle) ? true : false;\n#else\n return (pthread_mutex_trylock(&mHandle) == 0) ? true : false;\n#endif\n }\n\n /// Unlock the mutex.\n /// If any threads are waiting for the lock on this mutex, one of them will\n /// be unblocked.\n inline void unlock()\n {\n#if defined(_TTHREAD_WIN32_)\n LeaveCriticalSection(&mHandle);\n#else\n pthread_mutex_unlock(&mHandle);\n#endif\n }\n\n _TTHREAD_DISABLE_ASSIGNMENT(recursive_mutex)\n\n private:\n#if defined(_TTHREAD_WIN32_)\n CRITICAL_SECTION mHandle;\n#else\n pthread_mutex_t mHandle;\n#endif\n\n friend class condition_variable;\n};\n\n/// Lock guard class.\n/// The constructor locks the mutex, and the destructor unlocks the mutex, so\n/// the mutex will automatically be unlocked when the lock guard goes out of\n/// scope. Example usage:\n/// @code\n/// mutex m;\n/// int counter;\n///\n/// void increment()\n/// {\n/// lock_guard guard(m);\n/// ++ counter;\n/// }\n/// @endcode\n\ntemplate \nclass lock_guard {\n public:\n typedef T mutex_type;\n\n lock_guard() : mMutex(0) {}\n\n /// The constructor locks the mutex.\n explicit lock_guard(mutex_type &aMutex)\n {\n mMutex = &aMutex;\n mMutex->lock();\n }\n\n /// The destructor unlocks the mutex.\n ~lock_guard()\n {\n if(mMutex)\n mMutex->unlock();\n }\n\n private:\n mutex_type * mMutex;\n};\n\n/// Condition variable class.\n/// This is a signalling object for synchronizing the execution flow for\n/// several threads. Example usage:\n/// @code\n/// // Shared data and associated mutex and condition variable objects\n/// int count;\n/// mutex m;\n/// condition_variable cond;\n///\n/// // Wait for the counter to reach a certain number\n/// void wait_counter(int targetCount)\n/// {\n/// lock_guard guard(m);\n/// while(count < targetCount)\n/// cond.wait(m);\n/// }\n///\n/// // Increment the counter, and notify waiting threads\n/// void increment()\n/// {\n/// lock_guard guard(m);\n/// ++ count;\n/// cond.notify_all();\n/// }\n/// @endcode\nclass condition_variable {\n public:\n /// Constructor.\n#if defined(_TTHREAD_WIN32_)\n condition_variable();\n#else\n condition_variable()\n {\n pthread_cond_init(&mHandle, NULL);\n }\n#endif\n\n /// Destructor.\n#if defined(_TTHREAD_WIN32_)\n ~condition_variable();\n#else\n ~condition_variable()\n {\n pthread_cond_destroy(&mHandle);\n }\n#endif\n\n /// Wait for the condition.\n /// The function will block the calling thread until the condition variable\n /// is woken by @c notify_one(), @c notify_all() or a spurious wake up.\n /// @param[in] aMutex A mutex that will be unlocked when the wait operation\n /// starts, an locked again as soon as the wait operation is finished.\n template \n inline void wait(_mutexT &aMutex)\n {\n#if defined(_TTHREAD_WIN32_)\n // Increment number of waiters\n EnterCriticalSection(&mWaitersCountLock);\n ++ mWaitersCount;\n LeaveCriticalSection(&mWaitersCountLock);\n\n // Release the mutex while waiting for the condition (will decrease\n // the number of waiters when done)...\n aMutex.unlock();\n _wait();\n aMutex.lock();\n#else\n pthread_cond_wait(&mHandle, &aMutex.mHandle);\n#endif\n }\n\n /// Notify one thread that is waiting for the condition.\n /// If at least one thread is blocked waiting for this condition variable,\n /// one will be woken up.\n /// @note Only threads that started waiting prior to this call will be\n /// woken up.\n#if defined(_TTHREAD_WIN32_)\n void notify_one();\n#else\n inline void notify_one()\n {\n pthread_cond_signal(&mHandle);\n }\n#endif\n\n /// Notify all threads that are waiting for the condition.\n /// All threads that are blocked waiting for this condition variable will\n /// be woken up.\n /// @note Only threads that started waiting prior to this call will be\n /// woken up.\n#if defined(_TTHREAD_WIN32_)\n void notify_all();\n#else\n inline void notify_all()\n {\n pthread_cond_broadcast(&mHandle);\n }\n#endif\n\n _TTHREAD_DISABLE_ASSIGNMENT(condition_variable)\n\n private:\n#if defined(_TTHREAD_WIN32_)\n void _wait();\n HANDLE mEvents[2]; ///< Signal and broadcast event HANDLEs.\n unsigned int mWaitersCount; ///< Count of the number of waiters.\n CRITICAL_SECTION mWaitersCountLock; ///< Serialize access to mWaitersCount.\n#else\n pthread_cond_t mHandle;\n#endif\n};\n\n\n/// Thread class.\nclass thread {\n public:\n#if defined(_TTHREAD_WIN32_)\n typedef HANDLE native_handle_type;\n#else\n typedef pthread_t native_handle_type;\n#endif\n\n class id;\n\n /// Default constructor.\n /// Construct a @c thread object without an associated thread of execution\n /// (i.e. non-joinable).\n thread() : mHandle(0), mJoinable(false)\n#if defined(_TTHREAD_WIN32_)\n , mWin32ThreadID(0)\n#endif\n {}\n\n /// Thread starting constructor.\n /// Construct a @c thread object with a new thread of execution.\n /// @param[in] aFunction A function pointer to a function of type:\n /// void fun(void * arg)\n /// @param[in] aArg Argument to the thread function.\n /// @note This constructor is not fully compatible with the standard C++\n /// thread class. It is more similar to the pthread_create() (POSIX) and\n /// CreateThread() (Windows) functions.\n thread(void (*aFunction)(void *), void * aArg);\n\n /// Destructor.\n /// @note If the thread is joinable upon destruction, @c std::terminate()\n /// will be called, which terminates the process. It is always wise to do\n /// @c join() before deleting a thread object.\n ~thread();\n\n /// Wait for the thread to finish (join execution flows).\n /// After calling @c join(), the thread object is no longer associated with\n /// a thread of execution (i.e. it is not joinable, and you may not join\n /// with it nor detach from it).\n void join();\n\n /// Check if the thread is joinable.\n /// A thread object is joinable if it has an associated thread of execution.\n bool joinable() const;\n\n /// Detach from the thread.\n /// After calling @c detach(), the thread object is no longer assicated with\n /// a thread of execution (i.e. it is not joinable). The thread continues\n /// execution without the calling thread blocking, and when the thread\n /// ends execution, any owned resources are released.\n void detach();\n\n /// Return the thread ID of a thread object.\n id get_id() const;\n\n /// Get the native handle for this thread.\n /// @note Under Windows, this is a @c HANDLE, and under POSIX systems, this\n /// is a @c pthread_t.\n inline native_handle_type native_handle()\n {\n return mHandle;\n }\n\n /// Determine the number of threads which can possibly execute concurrently.\n /// This function is useful for determining the optimal number of threads to\n /// use for a task.\n /// @return The number of hardware thread contexts in the system.\n /// @note If this value is not defined, the function returns zero (0).\n static unsigned hardware_concurrency();\n\n _TTHREAD_DISABLE_ASSIGNMENT(thread)\n\n private:\n native_handle_type mHandle; ///< Thread handle.\n mutable mutex mDataMutex; ///< Serializer for access to the thread private data.\n bool mJoinable; ///< Is the thread joinable?\n#if defined(_TTHREAD_WIN32_)\n unsigned int mWin32ThreadID; ///< Unique thread ID (filled out by _beginthreadex).\n#endif\n\n // This is the internal thread wrapper function.\n#if defined(_TTHREAD_WIN32_)\n static unsigned WINAPI wrapper_function(void * aArg);\n#else\n static void * wrapper_function(void * aArg);\n#endif\n};\n\n/// Thread ID.\n/// The thread ID is a unique identifier for each thread.\n/// @see thread::get_id()\nclass thread::id {\n public:\n /// Default constructor.\n /// The default constructed ID is that of thread without a thread of\n /// execution.\n id() : mId(0) {};\n\n id(unsigned long int aId) : mId(aId) {};\n\n id(const id& aId) : mId(aId.mId) {};\n\n inline id & operator=(const id &aId)\n {\n mId = aId.mId;\n return *this;\n }\n\n inline friend bool operator==(const id &aId1, const id &aId2)\n {\n return (aId1.mId == aId2.mId);\n }\n\n inline friend bool operator!=(const id &aId1, const id &aId2)\n {\n return (aId1.mId != aId2.mId);\n }\n\n inline friend bool operator<=(const id &aId1, const id &aId2)\n {\n return (aId1.mId <= aId2.mId);\n }\n\n inline friend bool operator<(const id &aId1, const id &aId2)\n {\n return (aId1.mId < aId2.mId);\n }\n\n inline friend bool operator>=(const id &aId1, const id &aId2)\n {\n return (aId1.mId >= aId2.mId);\n }\n\n inline friend bool operator>(const id &aId1, const id &aId2)\n {\n return (aId1.mId > aId2.mId);\n }\n\n inline friend std::ostream& operator <<(std::ostream &os, const id &obj)\n {\n os << obj.mId;\n return os;\n }\n\n private:\n unsigned long int mId;\n};\n\n\n// Related to - minimal to be able to support chrono.\ntypedef long long __intmax_t;\n\n/// Minimal implementation of the @c ratio class. This class provides enough\n/// functionality to implement some basic @c chrono classes.\ntemplate <__intmax_t N, __intmax_t D = 1> class ratio {\n public:\n static double _as_double() { return double(N) / double(D); }\n};\n\n/// Minimal implementation of the @c chrono namespace.\n/// The @c chrono namespace provides types for specifying time intervals.\nnamespace chrono {\n /// Duration template class. This class provides enough functionality to\n /// implement @c this_thread::sleep_for().\n template > class duration {\n private:\n _Rep rep_;\n public:\n typedef _Rep rep;\n typedef _Period period;\n\n /// Construct a duration object with the given duration.\n template \n explicit duration(const _Rep2& r) : rep_(r) {}\n\n /// Return the value of the duration object.\n rep count() const\n {\n return rep_;\n }\n };\n\n // Standard duration types.\n typedef duration<__intmax_t, ratio<1, 1000000000> > nanoseconds; ///< Duration with the unit nanoseconds.\n typedef duration<__intmax_t, ratio<1, 1000000> > microseconds; ///< Duration with the unit microseconds.\n typedef duration<__intmax_t, ratio<1, 1000> > milliseconds; ///< Duration with the unit milliseconds.\n typedef duration<__intmax_t> seconds; ///< Duration with the unit seconds.\n typedef duration<__intmax_t, ratio<60> > minutes; ///< Duration with the unit minutes.\n typedef duration<__intmax_t, ratio<3600> > hours; ///< Duration with the unit hours.\n}\n\n/// The namespace @c this_thread provides methods for dealing with the\n/// calling thread.\nnamespace this_thread {\n /// Return the thread ID of the calling thread.\n thread::id get_id();\n\n /// Yield execution to another thread.\n /// Offers the operating system the opportunity to schedule another thread\n /// that is ready to run on the current processor.\n inline void yield()\n {\n#if defined(_TTHREAD_WIN32_)\n Sleep(0);\n#else\n sched_yield();\n#endif\n }\n\n /// Blocks the calling thread for a period of time.\n /// @param[in] aTime Minimum time to put the thread to sleep.\n /// Example usage:\n /// @code\n /// // Sleep for 100 milliseconds\n /// this_thread::sleep_for(chrono::milliseconds(100));\n /// @endcode\n /// @note Supported duration types are: nanoseconds, microseconds,\n /// milliseconds, seconds, minutes and hours.\n template void sleep_for(const chrono::duration<_Rep, _Period>& aTime)\n {\n#if defined(_TTHREAD_WIN32_)\n Sleep(int(double(aTime.count()) * (1000.0 * _Period::_as_double()) + 0.5));\n#else\n usleep(int(double(aTime.count()) * (1000000.0 * _Period::_as_double()) + 0.5));\n#endif\n }\n}\n\n}\n\n// Define/macro cleanup\n#undef _TTHREAD_DISABLE_ASSIGNMENT\n\n//////////////////////////////////////////////////////////////////////////\n// Inline implementation (ported from tinythread.cpp)\n/////////////////////////////////////////////////////////////////////////\n\n#include \n\n#if defined(_TTHREAD_POSIX_)\n #include \n #include \n#elif defined(_TTHREAD_WIN32_)\n #include \n#endif\n\n\nnamespace tthread {\n\n//------------------------------------------------------------------------------\n// condition_variable\n//------------------------------------------------------------------------------\n// NOTE 1: The Win32 implementation of the condition_variable class is based on\n// the corresponding implementation in GLFW, which in turn is based on a\n// description by Douglas C. Schmidt and Irfan Pyarali:\n// http://www.cs.wustl.edu/~schmidt/win32-cv-1.html\n//\n// NOTE 2: Windows Vista actually has native support for condition variables\n// (InitializeConditionVariable, WakeConditionVariable, etc), but we want to\n// be portable with pre-Vista Windows versions, so TinyThread++ does not use\n// Vista condition variables.\n//------------------------------------------------------------------------------\n\n#if defined(_TTHREAD_WIN32_)\n #define _CONDITION_EVENT_ONE 0\n #define _CONDITION_EVENT_ALL 1\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline condition_variable::condition_variable() : mWaitersCount(0)\n{\n mEvents[_CONDITION_EVENT_ONE] = CreateEvent(NULL, FALSE, FALSE, NULL);\n mEvents[_CONDITION_EVENT_ALL] = CreateEvent(NULL, TRUE, FALSE, NULL);\n InitializeCriticalSection(&mWaitersCountLock);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline condition_variable::~condition_variable()\n{\n CloseHandle(mEvents[_CONDITION_EVENT_ONE]);\n CloseHandle(mEvents[_CONDITION_EVENT_ALL]);\n DeleteCriticalSection(&mWaitersCountLock);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::_wait()\n{\n // Wait for either event to become signaled due to notify_one() or\n // notify_all() being called\n int result = WaitForMultipleObjects(2, mEvents, FALSE, INFINITE);\n\n // Check if we are the last waiter\n EnterCriticalSection(&mWaitersCountLock);\n -- mWaitersCount;\n bool lastWaiter = (result == (WAIT_OBJECT_0 + _CONDITION_EVENT_ALL)) &&\n (mWaitersCount == 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we are the last waiter to be notified to stop waiting, reset the event\n if(lastWaiter)\n ResetEvent(mEvents[_CONDITION_EVENT_ALL]);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::notify_one()\n{\n // Are there any waiters?\n EnterCriticalSection(&mWaitersCountLock);\n bool haveWaiters = (mWaitersCount > 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we have any waiting threads, send them a signal\n if(haveWaiters)\n SetEvent(mEvents[_CONDITION_EVENT_ONE]);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::notify_all()\n{\n // Are there any waiters?\n EnterCriticalSection(&mWaitersCountLock);\n bool haveWaiters = (mWaitersCount > 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we have any waiting threads, send them a signal\n if(haveWaiters)\n SetEvent(mEvents[_CONDITION_EVENT_ALL]);\n}\n#endif\n\n\n//------------------------------------------------------------------------------\n// POSIX pthread_t to unique thread::id mapping logic.\n// Note: Here we use a global thread safe std::map to convert instances of\n// pthread_t to small thread identifier numbers (unique within one process).\n// This method should be portable across different POSIX implementations.\n//------------------------------------------------------------------------------\n\n#if defined(_TTHREAD_POSIX_)\ninline static thread::id _pthread_t_to_ID(const pthread_t &aHandle)\n{\n static mutex idMapLock;\n static std::map idMap;\n static unsigned long int idCount(1);\n\n lock_guard guard(idMapLock);\n if(idMap.find(aHandle) == idMap.end())\n idMap[aHandle] = idCount ++;\n return thread::id(idMap[aHandle]);\n}\n#endif // _TTHREAD_POSIX_\n\n\n//------------------------------------------------------------------------------\n// thread\n//------------------------------------------------------------------------------\n\n/// Information to pass to the new thread (what to run).\nstruct _thread_start_info {\n void (*mFunction)(void *); ///< Pointer to the function to be executed.\n void * mArg; ///< Function argument for the thread function.\n thread * mThread; ///< Pointer to the thread object.\n};\n\n// Thread wrapper function.\n#if defined(_TTHREAD_WIN32_)\ninline unsigned WINAPI thread::wrapper_function(void * aArg)\n#elif defined(_TTHREAD_POSIX_)\ninline void * thread::wrapper_function(void * aArg)\n#endif\n{\n // Get thread startup information\n _thread_start_info * ti = (_thread_start_info *) aArg;\n\n try\n {\n // Call the actual client thread function\n ti->mFunction(ti->mArg);\n }\n catch(...)\n {\n // Uncaught exceptions will terminate the application (default behavior\n // according to C++11)\n std::terminate();\n }\n\n // The thread is no longer executing\n lock_guard guard(ti->mThread->mDataMutex);\n\n // On POSIX, we allow the thread to be joined even after execution has finished.\n // This is necessary to ensure that thread-local memory can be reclaimed.\n#if defined(_TTHREAD_WIN32_)\n ti->mThread->mJoinable = false;\n#endif\n\n // The thread is responsible for freeing the startup information\n delete ti;\n\n return 0;\n}\n\ninline thread::thread(void (*aFunction)(void *), void * aArg)\n{\n // Serialize access to this thread structure\n lock_guard guard(mDataMutex);\n\n // Fill out the thread startup information (passed to the thread wrapper,\n // which will eventually free it)\n _thread_start_info * ti = new _thread_start_info;\n ti->mFunction = aFunction;\n ti->mArg = aArg;\n ti->mThread = this;\n\n // Mark thread as joinable\n mJoinable = true;\n\n // Create the thread\n#if defined(_TTHREAD_WIN32_)\n mHandle = (HANDLE) _beginthreadex(0, 0, wrapper_function, (void *) ti, 0, &mWin32ThreadID);\n#elif defined(_TTHREAD_POSIX_)\n if(pthread_create(&mHandle, NULL, wrapper_function, (void *) ti) != 0)\n mHandle = 0;\n#endif\n\n // Did we fail to create the thread?\n if(!mHandle)\n {\n mJoinable = false;\n delete ti;\n }\n\n}\n\ninline thread::~thread()\n{\n if(joinable())\n std::terminate();\n}\n\ninline void thread::join()\n{\n if(joinable())\n {\n#if defined(_TTHREAD_WIN32_)\n WaitForSingleObject(mHandle, INFINITE);\n CloseHandle(mHandle);\n#elif defined(_TTHREAD_POSIX_)\n pthread_join(mHandle, NULL);\n#endif\n\n // https://linux.die.net/man/3/pthread_join states:\n //\n // Joining with a thread that has previously been joined results in undefined behavior.\n //\n // We just allow a thread to be joined once.\n mJoinable = false;\n }\n}\n\ninline bool thread::joinable() const\n{\n mDataMutex.lock();\n bool result = mJoinable;\n mDataMutex.unlock();\n return result;\n}\n\ninline void thread::detach()\n{\n // TODO: Attempting to detach a non-joinable thread should throw.\n // https://en.cppreference.com/w/cpp/thread/thread/detach\n mDataMutex.lock();\n if(mJoinable)\n {\n#if defined(_TTHREAD_WIN32_)\n CloseHandle(mHandle);\n#elif defined(_TTHREAD_POSIX_)\n pthread_detach(mHandle);\n#endif\n mJoinable = false;\n }\n mDataMutex.unlock();\n}\n\ninline thread::id thread::get_id() const\n{\n if(!joinable())\n return id();\n#if defined(_TTHREAD_WIN32_)\n return id((unsigned long int) mWin32ThreadID);\n#elif defined(_TTHREAD_POSIX_)\n return _pthread_t_to_ID(mHandle);\n#endif\n}\n\ninline unsigned thread::hardware_concurrency()\n{\n#if defined(_TTHREAD_WIN32_)\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n return (int) si.dwNumberOfProcessors;\n#elif defined(_SC_NPROCESSORS_ONLN)\n return (int) sysconf(_SC_NPROCESSORS_ONLN);\n#elif defined(_SC_NPROC_ONLN)\n return (int) sysconf(_SC_NPROC_ONLN);\n#else\n // The standard requires this function to return zero if the number of\n // hardware cores could not be determined.\n return 0;\n#endif\n}\n\n\n//------------------------------------------------------------------------------\n// this_thread\n//------------------------------------------------------------------------------\n\ninline thread::id this_thread::get_id()\n{\n#if defined(_TTHREAD_WIN32_)\n return thread::id((unsigned long int) GetCurrentThreadId());\n#elif defined(_TTHREAD_POSIX_)\n return _pthread_t_to_ID(pthread_self());\n#endif\n}\n\n}\n\n\n\n#endif // _TINYTHREAD_H_\n" }, { "path": "inst/include/tthread/tinythread.inl", "content": "/* -*- mode: c++; tab-width: 2; indent-tabs-mode: nil; -*-\nCopyright (c) 2010-2012 Marcus Geelnard\n\nThis software is provided 'as-is', without any express or implied\nwarranty. In no event will the authors be held liable for any damages\narising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose,\nincluding commercial applications, and to alter it and redistribute it\nfreely, subject to the following restrictions:\n\n 1. The origin of this software must not be misrepresented; you must not\n claim that you wrote the original software. If you use this software\n in a product, an acknowledgment in the product documentation would be\n appreciated but is not required.\n\n 2. Altered source versions must be plainly marked as such, and must not be\n misrepresented as being the original software.\n\n 3. This notice may not be removed or altered from any source\n distribution.\n*/\n\n#include \n#include \"tinythread.h\"\n\n#if defined(_TTHREAD_POSIX_)\n #include \n #include \n#elif defined(_TTHREAD_WIN32_)\n #include \n#endif\n\n\nnamespace tthread {\n\n//------------------------------------------------------------------------------\n// condition_variable\n//------------------------------------------------------------------------------\n// NOTE 1: The Win32 implementation of the condition_variable class is based on\n// the corresponding implementation in GLFW, which in turn is based on a\n// description by Douglas C. Schmidt and Irfan Pyarali:\n// http://www.cs.wustl.edu/~schmidt/win32-cv-1.html\n//\n// NOTE 2: Windows Vista actually has native support for condition variables\n// (InitializeConditionVariable, WakeConditionVariable, etc), but we want to\n// be portable with pre-Vista Windows versions, so TinyThread++ does not use\n// Vista condition variables.\n//------------------------------------------------------------------------------\n\n#if defined(_TTHREAD_WIN32_)\n #define _CONDITION_EVENT_ONE 0\n #define _CONDITION_EVENT_ALL 1\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline condition_variable::condition_variable() : mWaitersCount(0)\n{\n mEvents[_CONDITION_EVENT_ONE] = CreateEvent(NULL, FALSE, FALSE, NULL);\n mEvents[_CONDITION_EVENT_ALL] = CreateEvent(NULL, TRUE, FALSE, NULL);\n InitializeCriticalSection(&mWaitersCountLock);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline condition_variable::~condition_variable()\n{\n CloseHandle(mEvents[_CONDITION_EVENT_ONE]);\n CloseHandle(mEvents[_CONDITION_EVENT_ALL]);\n DeleteCriticalSection(&mWaitersCountLock);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::_wait()\n{\n // Wait for either event to become signaled due to notify_one() or\n // notify_all() being called\n int result = WaitForMultipleObjects(2, mEvents, FALSE, INFINITE);\n\n // Check if we are the last waiter\n EnterCriticalSection(&mWaitersCountLock);\n -- mWaitersCount;\n bool lastWaiter = (result == (WAIT_OBJECT_0 + _CONDITION_EVENT_ALL)) &&\n (mWaitersCount == 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we are the last waiter to be notified to stop waiting, reset the event\n if(lastWaiter)\n ResetEvent(mEvents[_CONDITION_EVENT_ALL]);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::notify_one()\n{\n // Are there any waiters?\n EnterCriticalSection(&mWaitersCountLock);\n bool haveWaiters = (mWaitersCount > 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we have any waiting threads, send them a signal\n if(haveWaiters)\n SetEvent(mEvents[_CONDITION_EVENT_ONE]);\n}\n#endif\n\n#if defined(_TTHREAD_WIN32_)\ninline void condition_variable::notify_all()\n{\n // Are there any waiters?\n EnterCriticalSection(&mWaitersCountLock);\n bool haveWaiters = (mWaitersCount > 0);\n LeaveCriticalSection(&mWaitersCountLock);\n\n // If we have any waiting threads, send them a signal\n if(haveWaiters)\n SetEvent(mEvents[_CONDITION_EVENT_ALL]);\n}\n#endif\n\n\n//------------------------------------------------------------------------------\n// POSIX pthread_t to unique thread::id mapping logic.\n// Note: Here we use a global thread safe std::map to convert instances of\n// pthread_t to small thread identifier numbers (unique within one process).\n// This method should be portable across different POSIX implementations.\n//------------------------------------------------------------------------------\n\n#if defined(_TTHREAD_POSIX_)\ninline static thread::id _pthread_t_to_ID(const pthread_t &aHandle)\n{\n static mutex idMapLock;\n static std::map idMap;\n static unsigned long int idCount(1);\n\n lock_guard guard(idMapLock);\n if(idMap.find(aHandle) == idMap.end())\n idMap[aHandle] = idCount ++;\n return thread::id(idMap[aHandle]);\n}\n#endif // _TTHREAD_POSIX_\n\n\n//------------------------------------------------------------------------------\n// thread\n//------------------------------------------------------------------------------\n\n/// Information to pass to the new thread (what to run).\nstruct _thread_start_info {\n void (*mFunction)(void *); ///< Pointer to the function to be executed.\n void * mArg; ///< Function argument for the thread function.\n thread * mThread; ///< Pointer to the thread object.\n};\n\n// Thread wrapper function.\n#if defined(_TTHREAD_WIN32_)\ninline unsigned WINAPI thread::wrapper_function(void * aArg)\n#elif defined(_TTHREAD_POSIX_)\ninline void * thread::wrapper_function(void * aArg)\n#endif\n{\n // Get thread startup information\n _thread_start_info * ti = (_thread_start_info *) aArg;\n\n try\n {\n // Call the actual client thread function\n ti->mFunction(ti->mArg);\n }\n catch(...)\n {\n // Uncaught exceptions will terminate the application (default behavior\n // according to C++11)\n std::terminate();\n }\n\n // The thread is no longer executing\n lock_guard guard(ti->mThread->mDataMutex);\n ti->mThread->mNotAThread = true;\n\n // The thread is responsible for freeing the startup information\n delete ti;\n\n return 0;\n}\n\ninline thread::thread(void (*aFunction)(void *), void * aArg)\n{\n // Serialize access to this thread structure\n lock_guard guard(mDataMutex);\n\n // Fill out the thread startup information (passed to the thread wrapper,\n // which will eventually free it)\n _thread_start_info * ti = new _thread_start_info;\n ti->mFunction = aFunction;\n ti->mArg = aArg;\n ti->mThread = this;\n\n // The thread is now alive\n mNotAThread = false;\n\n // Create the thread\n#if defined(_TTHREAD_WIN32_)\n mHandle = (HANDLE) _beginthreadex(0, 0, wrapper_function, (void *) ti, 0, &mWin32ThreadID);\n#elif defined(_TTHREAD_POSIX_)\n if(pthread_create(&mHandle, NULL, wrapper_function, (void *) ti) != 0)\n mHandle = 0;\n#endif\n\n // Did we fail to create the thread?\n if(!mHandle)\n {\n mNotAThread = true;\n delete ti;\n }\n}\n\ninline thread::~thread()\n{\n if(joinable())\n std::terminate();\n}\n\ninline void thread::join()\n{\n if(joinable())\n {\n#if defined(_TTHREAD_WIN32_)\n WaitForSingleObject(mHandle, INFINITE);\n CloseHandle(mHandle);\n#elif defined(_TTHREAD_POSIX_)\n pthread_join(mHandle, NULL);\n#endif\n }\n}\n\ninline bool thread::joinable() const\n{\n mDataMutex.lock();\n bool result = !mNotAThread;\n mDataMutex.unlock();\n return result;\n}\n\ninline void thread::detach()\n{\n mDataMutex.lock();\n if(!mNotAThread)\n {\n#if defined(_TTHREAD_WIN32_)\n CloseHandle(mHandle);\n#elif defined(_TTHREAD_POSIX_)\n pthread_detach(mHandle);\n#endif\n mNotAThread = true;\n }\n mDataMutex.unlock();\n}\n\ninline thread::id thread::get_id() const\n{\n if(!joinable())\n return id();\n#if defined(_TTHREAD_WIN32_)\n return id((unsigned long int) mWin32ThreadID);\n#elif defined(_TTHREAD_POSIX_)\n return _pthread_t_to_ID(mHandle);\n#endif\n}\n\ninline unsigned thread::hardware_concurrency()\n{\n#if defined(_TTHREAD_WIN32_)\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n return (int) si.dwNumberOfProcessors;\n#elif defined(_SC_NPROCESSORS_ONLN)\n return (int) sysconf(_SC_NPROCESSORS_ONLN);\n#elif defined(_SC_NPROC_ONLN)\n return (int) sysconf(_SC_NPROC_ONLN);\n#else\n // The standard requires this function to return zero if the number of\n // hardware cores could not be determined.\n return 0;\n#endif\n}\n\n\n//------------------------------------------------------------------------------\n// this_thread\n//------------------------------------------------------------------------------\n\ninline thread::id this_thread::get_id()\n{\n#if defined(_TTHREAD_WIN32_)\n return thread::id((unsigned long int) GetCurrentThreadId());\n#elif defined(_TTHREAD_POSIX_)\n return _pthread_t_to_ID(pthread_self());\n#endif\n}\n\n}\n" }, { "path": "inst/presentations/.gitignore", "content": "*.tex\n*.pdf\n" }, { "path": "inst/presentations/rcpp_parallel_talk_jan2015.Rmd", "content": "---\ntitle: \"R, Rcpp and Parallel Computing\"\nauthor: \"Dirk Eddelbuettel and JJ Allaire\"\ndate: \"Jan 26-27, 2015\\\\newline Workshop for Distributed Computing in R\\\\newline HP Research, Palo Alto, CA\"\noutput:\n beamer_presentation:\n includes:\n in_header: header.tex\n keep_tex: yes\n theme: Warsaw\nfontsize: 12pt\nsubtitle: Notes from our Rcpp Experience\nclassoption: compress\n---\n\n# Intro\n\n## One View on Parallel Computing\n\n> The whole \"let's parallelize\" thing is a huge waste of everybody's\n> time. There's this huge body of \"knowledge\" that parallel is somehow more\n> efficient, and that whole huge body is pure and utter garbage. Big caches\n> are efficient. Parallel stupid small cores without caches are horrible\n> unless you have a very specific load that is hugely regular (ie graphics). \n> \n> [...] \n> \n> Give it up. The whole \"parallel computing is the future\" is a bunch of crock.\n\n\n[Linus Torvalds, Dec 2014](http://www.realworldtech.com/forum/?threadid=146066&curpostid=146227)\n\n\n## Another View on Big Data\n\\framesubtitle{Imagine a \\texttt{gsub(\"DBMs\", \"\", tweet)} to complement further...}\n\n\\centering{\\includegraphics[width=\\textwidth,height=0.8\\textheight,keepaspectratio]{images/big-data-big-machine-tweet.png}}\n\n# R\n\n## CRAN Task View on HPC\n\n\\framesubtitle{\\texttt{http://cran.r-project.org/web/views/HighPerformanceComputing.html}}\n\nThings R does well:\n\n\\medskip\n\n- Package snow by Tierney et al a trailblazer\n- Package Rmpi by Yu equally important\n- multicore / snow / parallel even work on Windows\n- Hundreds of applications\n- _It just works_ for data-parallel tasks\n\n# Rcpp\n\n## Rcpp: Early Days\n\nIn the fairly early days of Rcpp, we also put out RInside as a simple C++\nclass wrapper around the R-embedding API.\n\nIt got one clever patch taking this (ie: R wrapped in C++ with its own\n`main()` function) and encapsulating it within MPI. \n\nHP Vertica also uses Rcpp and RInside in \n[DistributedR](https://github.com/vertica/DistributedR/tree/master/third_party).\n\n## Rcpp: More recently\n\nRcpp is now easy to deploy; Rcpp Attributes played a key role:\n\n```r\n#include \nusing namespace Rcpp;\n\n// [[Rcpp::export]]\ndouble piSugar(const int N) {\n NumericVector x = runif(N);\n NumericVector y = runif(N);\n NumericVector d = sqrt(x*x + y*y);\n return 4.0 * sum(d < 1.0) / N;\n}\n```\n\n## Rcpp: Extensions\n\nRcpp Attributes also support \"plugins\"\n\nOpenMP is easy to use and widely\nsupported (on suitable OS / compiler combinations).\n\nSo we added support via a plugin. Use is still not as wide-spread.\n\nErrors have commonality: calling back into R.\n\n\n# RcppParallel\n\n## Parallel Programming for Rcpp Users\n\n\\framesubtitle{NOT like this...}\n\n```cpp\nusing namespace boost;\n\nvoid task()\n{\n lock_guard lock(mutex);\n // etc...\n}\n\nthreadpool::pool tp(thread::hardware_concurrency());\nfor (int i=0; i Calling any of the R API from threaded code is ‘for experts only’: they will need to read the source code to determine if it is thread-safe. In particular, code which makes use of the stack-checking mechanism must not be called from threaded code.\n\nHowever we don't really want to force Rcpp users to resort to reading the Rcpp and R source code to assess thread safety issues.\n\n## RcppParallel Threadsafe Accessors\n\nSince R vectors and matrices are just raw contiguous arrays it's easy to create threadsafe C++ wrappers for them:\n\n* `RVector` is a very thin wrapper over a C array.\n\n* `RMatrix` is the same but also provides `Row` and `Column` accessors/iterators.\n\nThe implementions of these classes are extremely lightweight and never call into Rcpp or the R API (so are always threadsafe).\n\n## RcppParallel Operations\n\nTwo high-level operations are provided (with TBB and TinyThread implementations of each):\n\n* `parallelFor` -- Convert the work of a standard serial \"for\" loop into a parallel one\n\n* `parallelReduce` -- Used for accumulating aggregate or other values.\n\nNot surprisingly the TBB versions of these operations perform ~ 50% better than the \"naive\" parallel implementation provided by TinyThread.\n\n## Basic Mechanics: Create a Worker\n\nCreate a `Worker` class with `operator()` that RcppParallel uses to operate on discrete slices of the input data on different threads:\n\n```cpp\n\nclass MyWorker : public RcppParallel::Worker {\n\n void operator()(size_t begin, size_t end) {\n // do some work from begin to end \n // within the input data\n }\n \n}\n\n```\n\n## Basic Mechanics: Call the Worker\n\nWorker would typically take input and output data in it's constructor then save them as members (for reading/writing within `operator()`):\n\n```cpp\nNumericMatrix matrixSqrt(NumericMatrix x) {\n \n NumericMatrix output(x.nrow(), x.ncol());\n\n SquareRootWorker worker(x, output);\n \n parallelFor(0, x.length(), worker);\n \n return output;\n}\n```\n\n## Basic Mechanics: Join Function\n\nFor `parallelReduce` you need to specify how data is to be combined. Typically you save data in a member within `operator()` then fuse it with another `Worker` instance in the `join` function.\n\n```cpp\nclass SumWorker : public RcppParallel::Worker\n \n // join my value with that of another SumWorker\n void join(const SumWorker& rhs) { \n value += rhs.value; \n }\n}\n\n```\n\n## What does all of this buy us?\n\n* Developers just write pieces of code that are called at the correct time by an intelligent parallel supervisor\n* In most cases no locking or explicit thread management required!\n* Supervisor does some intelligent optimization around:\n - Grain size (which affects locality of reference and therefore cache hit rates). Note that grain size can also be tuned directly per-application.\n - Work stealing (detecting idle threads and pushing work to them)\n* In the case of TBB, high performance concurrent containers are available if necessary\n\n## Examples\n\n* All available on the Rcpp Gallery \n\n* Tested with 4 cores on a 2.6GHz Haswell MacBook Pro\n\n* Note that benchmarks will be 30-50% slower on Windows because we aren't using the more sophisticated scheduling of TBB\n\n## Example: Transforming a Matrix in Parallel\n\n\\framesubtitle{\\texttt{http://gallery.rcpp.org/articles/parallel-matrix-transform}}\n\n```cpp\n void operator()(size_t begin, size_t end) {\n std::transform(input.begin() + begin, \n input.begin() + end, \n output.begin() + begin, \n ::sqrt);\n }\n```\n\n```\n test replications elapsed relative\n2 parallelMatrixSqrt(m) 100 0.294 1.000\n1 matrixSqrt(m) 100 0.755 2.568\n```\n\n## Example: Summing a Vector in Parallel\n\n\\framesubtitle{\\texttt{http://gallery.rcpp.org/articles/parallel-vector-sum}}\n\n\n```cpp\nvoid operator()(size_t begin, size_t end) {\n value += std::accumulate(input.begin() + begin, \n input.begin() + end, \n 0.0);\n} \nvoid join(const Sum& rhs) { \n value += rhs.value; \n}\n```\n```\n test replications elapsed relative\n2 parallelVectorSum(v) 100 0.182 1.000\n1 vectorSum(v) 100 0.857 4.709\n```\n\n## Example: Parallel Distance Matrix Calculation\n\n\\framesubtitle{\\texttt{http://gallery.rcpp.org/articles/parallel-distance-matrix}}\n\n```\n test reps elapsed relative\n3 rcpp_parallel_distance(m) 3 0.110 1.000\n2 rcpp_distance(m) 3 0.618 5.618\n1 distance(m) 3 35.560 323.273\n```\n\n* Rcpp + RcppParallel = 323x over R implementation!\n* Unbalanced workload benefits from work stealing\n\n## The Rest of TBB\n\n* Advanced algorithms: `parallel_scan`, `parallel_while`, `parallel_do`, `parallel_pipeline`, `parallel_sort`\n* Containers: `concurrent_queue`, `concurrent_priority_queue`, `concurrent_vector`, `concurrent_hash_map`\n* Mutual exclusion: `mutex`, `spin_mutex`, `queuing_mutex`, `spin_rw_mutex`, `queuing_rw_mutex`, `recursive_mutex`\n* Atomic operations: `fetch_and_add`, `fetch_and_increment`, `fetch_and_decrement`, `compare_and_swap`, `fetch_and_store`\n* Timing: portable fine grained global time stamp\n* Task Scheduler: direct access to control the creation and activation of tasks\n\n## Open Issues\n\n* Additional (portable to Win32 via TinyThread) wrappers for other TBB constructs?\n\n* Alternatively, sort out Rtools configuration issues required to get TBB working on Windows.\n\n* Education: Parallel Programming is _hard_. \n \n* Simple `parallelFor` and `parallelReduce` are reasonably easy to grasp, but more advanced idioms aren't trivial to learn and use (but for some applications have lots of upside so are worth the effort).\n\n \n" }, { "path": "inst/rstudio/templates/project/RcppParallel.package.skeleton.dcf", "content": "Binding: RcppParallel.package.skeleton\nTitle: R Package using RcppParallel\nSubtitle: Create a new R Package using RcppParallel\nCaption: Create R Package using RcppParallel\nOpenFiles: src/vector-sum.cpp\n\nParameter: example_code\nWidget: CheckboxInput\nLabel: Include an example C++ file using RcppParallel\nDefault: On\n" }, { "path": "inst/skeleton/vector-sum.Rd", "content": "\\name{parallelVectorSum}\n\\alias{parallelVectorSum}\n\\docType{package}\n\\title{\nSimple function using RcppParallel\n}\n\\description{\nSimple function using RcppParallel\n}\n\\usage{\nparallelVectorSum(x)\n}\n\\arguments{\n\\item{x}{A numeric vector.}\n}\n\\examples{\n\\dontrun{\nparallelVectorSum(1:20)\n}\n}\n" }, { "path": "inst/skeleton/vector-sum.cpp", "content": "/**\n *\n * This file contains example code showcasing how RcppParallel\n * can be used. In this file, we define and export a function called\n * 'parallelVectorSum()', which computes the sum of a numeric vector\n * in parallel.\n *\n * Please see https://rcppcore.github.io/RcppParallel/ for more\n * details on how to use RcppParallel in an R package, and the\n * Rcpp gallery at http://gallery.rcpp.org/ for more examples.\n *\n */\n\n// [[Rcpp::depends(RcppParallel)]]\n#include \n#include \n\nusing namespace Rcpp;\nusing namespace RcppParallel;\n\nstruct Sum : public Worker\n{\n // source vector\n const RVector input;\n\n // accumulated value\n double value;\n\n // constructors\n Sum(const NumericVector input) : input(input), value(0) {}\n Sum(const Sum& sum, Split) : input(sum.input), value(0) {}\n\n // accumulate just the element of the range I've been asked to\n void operator()(std::size_t begin, std::size_t end) {\n value += std::accumulate(input.begin() + begin, input.begin() + end, 0.0);\n }\n\n // join my value with that of another Sum\n void join(const Sum& rhs) {\n value += rhs.value;\n }\n};\n\n// [[Rcpp::export]]\ndouble parallelVectorSum(NumericVector x) {\n\n // declare the SumBody instance\n Sum sum(x);\n\n // call parallel_reduce to start the work\n parallelReduce(0, x.length(), sum);\n\n // return the computed sum\n return sum.value;\n}\n" }, { "path": "inst/tests/cpp/distance.cpp", "content": "/**\n * @title Parallel Distance Matrix Calculation with RcppParallel\n * @author JJ Allaire and Jim Bullard\n * @license GPL (>= 2)\n */\n\n#include \nusing namespace Rcpp;\n\n#include \n#include \n\n// generic function for kl_divergence\ntemplate \ninline double kl_divergence(InputIterator1 begin1, InputIterator1 end1, \n InputIterator2 begin2) {\n \n // value to return\n double rval = 0;\n \n // set iterators to beginning of ranges\n InputIterator1 it1 = begin1;\n InputIterator2 it2 = begin2;\n \n // for each input item\n while (it1 != end1) {\n \n // take the value and increment the iterator\n double d1 = *it1++;\n double d2 = *it2++;\n \n // accumulate if appropirate\n if (d1 > 0 && d2 > 0)\n rval += std::log(d1 / d2) * d1;\n }\n return rval; \n}\n\n// helper function for taking the average of two numbers\ninline double average(double val1, double val2) {\n return (val1 + val2) / 2;\n}\n\n// [[Rcpp::export]]\nNumericMatrix rcpp_js_distance(NumericMatrix mat) {\n \n // allocate the matrix we will return\n NumericMatrix rmat(mat.nrow(), mat.nrow());\n \n for (int i = 0; i < rmat.nrow(); i++) {\n for (int j = 0; j < i; j++) {\n \n // rows we will operate on\n NumericMatrix::Row row1 = mat.row(i);\n NumericMatrix::Row row2 = mat.row(j);\n \n // compute the average using std::tranform from the STL\n std::vector avg(row1.size());\n std::transform(row1.begin(), row1.end(), // input range 1\n row2.begin(), // input range 2\n avg.begin(), // output range \n average); // function to apply\n \n // calculate divergences\n double d1 = kl_divergence(row1.begin(), row1.end(), avg.begin());\n double d2 = kl_divergence(row2.begin(), row2.end(), avg.begin());\n \n // write to output matrix\n rmat(i,j) = std::sqrt(.5 * (d1 + d2));\n }\n }\n \n return rmat;\n}\n\n// [[Rcpp::depends(RcppParallel)]]\n#include \nusing namespace RcppParallel;\n\nstruct JsDistance : public Worker {\n \n // input matrix to read from\n const RMatrix mat;\n \n // output matrix to write to\n RMatrix rmat;\n \n // initialize from Rcpp input and output matrixes (the RMatrix class\n // can be automatically converted to from the Rcpp matrix type)\n JsDistance(const NumericMatrix mat, NumericMatrix rmat)\n : mat(mat), rmat(rmat) {}\n \n // function call operator that work for the specified range (begin/end)\n void operator()(std::size_t begin, std::size_t end) {\n for (std::size_t i = begin; i < end; i++) {\n for (std::size_t j = 0; j < i; j++) {\n \n // rows we will operate on\n RMatrix::Row row1 = mat.row(i);\n RMatrix::Row row2 = mat.row(j);\n \n // compute the average using std::tranform from the STL\n std::vector avg(row1.length());\n std::transform(row1.begin(), row1.end(), // input range 1\n row2.begin(), // input range 2\n avg.begin(), // output range \n average); // function to apply\n \n // calculate divergences\n double d1 = kl_divergence(row1.begin(), row1.end(), avg.begin());\n double d2 = kl_divergence(row2.begin(), row2.end(), avg.begin());\n \n // write to output matrix\n rmat(i,j) = sqrt(.5 * (d1 + d2));\n }\n }\n }\n};\n\n// [[Rcpp::export]]\nNumericMatrix rcpp_parallel_js_distance(NumericMatrix mat) {\n \n // allocate the matrix we will return\n NumericMatrix rmat(mat.nrow(), mat.nrow());\n\n // create the worker\n JsDistance jsDistance(mat, rmat);\n \n // call it with parallelFor\n parallelFor(0, mat.nrow(), jsDistance);\n\n return rmat;\n}\n\n" }, { "path": "inst/tests/cpp/innerproduct.cpp", "content": "/**\n * @title Computing an Inner Product with RcppParallel\n * @author JJ Allaire\n * @license GPL (>= 2)\n */\n\n#include \nusing namespace Rcpp;\n\n#include \n\n// [[Rcpp::export]]\ndouble innerProduct(NumericVector x, NumericVector y) {\n return std::inner_product(x.begin(), x.end(), y.begin(), 0.0);\n}\n\n// [[Rcpp::depends(RcppParallel)]]\n#include \nusing namespace RcppParallel;\n\nstruct InnerProduct : public Worker\n{\n // source vectors\n const RVector x;\n const RVector y;\n\n // product that I have accumulated\n double product;\n\n // constructors\n InnerProduct(const NumericVector x, const NumericVector y)\n : x(x), y(y), product(0) {}\n InnerProduct(const InnerProduct& innerProduct, Split)\n : x(innerProduct.x), y(innerProduct.y), product(0) {}\n\n // process just the elements of the range I have been asked to\n void operator()(std::size_t begin, std::size_t end) {\n product += std::inner_product(x.begin() + begin,\n x.begin() + end,\n y.begin() + begin,\n 0.0);\n }\n\n // join my value with that of another InnerProduct\n void join(const InnerProduct& rhs) {\n product += rhs.product;\n }\n};\n\n// [[Rcpp::export]]\ndouble parallelInnerProduct(NumericVector x, NumericVector y) {\n\n // declare the InnerProduct instance that takes a pointer to the vector data\n InnerProduct innerProduct(x, y);\n\n // call paralleReduce to start the work\n parallelReduce(0, x.length(), innerProduct);\n\n // return the computed product\n return innerProduct.product;\n}\n\n" }, { "path": "inst/tests/cpp/sum.cpp", "content": "/**\n * @title Summing a Vector in Parallel with RcppParallel\n * @author JJ Allaire\n * @license GPL (>= 2)\n */\n\n#include \n#include \n\n// [[Rcpp::depends(RcppParallel)]]\nusing namespace RcppParallel;\nusing namespace Rcpp;\n\nstruct Sum : public Worker\n{\n // source vector\n const RVector input;\n\n // accumulated value\n double value;\n\n // constructors\n Sum(const NumericVector input) : input(input), value(0) {}\n Sum(const Sum& sum, Split) : input(sum.input), value(0) {}\n\n // accumulate just the element of the range I have been asked to\n void operator()(std::size_t begin, std::size_t end) {\n value += std::accumulate(input.begin() + begin, input.begin() + end, 0.0);\n }\n\n // join my value with that of another Sum\n void join(const Sum& rhs) {\n value += rhs.value;\n }\n};\n\n// [[Rcpp::export]]\ndouble parallelVectorSum(NumericVector x) {\n\n // declare the SumBody instance\n Sum sum(x);\n\n // call parallel_reduce to start the work\n parallelReduce(0, x.length(), sum);\n\n // return the computed sum\n return sum.value;\n}\n\n// [[Rcpp::export]]\ndouble vectorSum(NumericVector x) {\n return std::accumulate(x.begin(), x.end(), 0.0);\n}\n\n\n" }, { "path": "inst/tests/cpp/transform.cpp", "content": "/**\n * @title Transforming a Matrix in Parallel using RcppParallel\n * @author JJ Allaire\n * @license GPL (>= 2)\n */\n \n#include \nusing namespace Rcpp;\n\n#include \n#include \n\ndouble squareRoot(double x) {\n return ::sqrt(x);\n}\n\n// [[Rcpp::export]]\nNumericMatrix matrixSqrt(NumericMatrix orig) {\n\n // allocate the matrix we will return\n NumericMatrix mat(orig.nrow(), orig.ncol());\n\n // transform it\n std::transform(orig.begin(), orig.end(), mat.begin(), squareRoot);\n\n // return the new matrix\n return mat;\n}\n// [[Rcpp::depends(RcppParallel)]]\n#include \nusing namespace RcppParallel;\n\nstruct SquareRoot : public Worker\n{\n // source matrix\n const RMatrix input;\n \n // destination matrix\n RMatrix output;\n \n // initialize with source and destination\n SquareRoot(const NumericMatrix input, NumericMatrix output) \n : input(input), output(output) {}\n \n // take the square root of the range of elements requested\n void operator()(std::size_t begin, std::size_t end) {\n std::transform(input.begin() + begin, \n input.begin() + end, \n output.begin() + begin, \n squareRoot);\n }\n};\n\n// [[Rcpp::export]]\nNumericMatrix parallelMatrixSqrt(NumericMatrix x) {\n \n // allocate the output matrix\n NumericMatrix output(x.nrow(), x.ncol());\n \n // SquareRoot functor (pass input and output matrixes)\n SquareRoot squareRoot(x, output);\n \n // call parallelFor to do the work\n parallelFor(0, x.nrow() * x.ncol(), squareRoot);\n \n // return the output matrix\n return output;\n}\n\n" }, { "path": "inst/tests/cpp/truefalse_macros.cpp", "content": "/**\n * @title Test for TRUE and FALSE macros\n * @author Travers Ching\n * @license GPL (>= 2)\n */\n\n// TRUE and FALSE macros that may come with system headers on some systems\n// But conflict with R.h (R_ext/Boolean.h)\n// TRUE and FALSE macros should be undef in RcppParallel.h\n\n#include \n#include \n\n// [[Rcpp::depends(RcppParallel)]]\n\n#ifndef TRUE\nstatic_assert(true, \"Macro TRUE does not exist\");\n#else\nstatic_assert(false, \"Macro TRUE exists\");\n#endif\n\n#ifndef FALSE\nstatic_assert(true, \"Macro FALSE does not exist\");\n#else\nstatic_assert(false, \"Macro FALSE exists\");\n#endif\n\n// [[Rcpp::export]]\nint hush_no_export_warning() {\n return 1;\n}" }, { "path": "inst/tests/runit.distance.R", "content": "\nlibrary(Rcpp)\nlibrary(RUnit)\n\nsourceCpp(system.file(\"tests/cpp/distance.cpp\", package = \"RcppParallel\")) \n\ntest.distance <- function() {\n \n n <- 1000\n m <- matrix(runif(n*10), ncol = 10)\n m <- m/rowSums(m)\n \n checkEquals(\n rcpp_js_distance(m), \n rcpp_parallel_js_distance(m)\n )\n \n}\n" }, { "path": "inst/tests/runit.innerproduct.R", "content": "\nlibrary(Rcpp)\nlibrary(RUnit)\n\nsourceCpp(system.file(\"tests/cpp/innerproduct.cpp\", package = \"RcppParallel\"))\n\ntest.innerproduct <- function() {\n \n x <- runif(1000000)\n y <- runif(1000000)\n \n checkEquals(\n innerProduct(x, y), \n parallelInnerProduct(x, y)\n )\n \n}\n" }, { "path": "inst/tests/runit.sum.R", "content": "\nlibrary(Rcpp)\nlibrary(RUnit)\n\nsourceCpp(system.file(\"tests/cpp/sum.cpp\", package = \"RcppParallel\"))\n\ntest.sum <- function() {\n \n v <- as.numeric(c(1:10000000))\n \n checkEquals(\n vectorSum(v), \n parallelVectorSum(v)\n )\n}\n" }, { "path": "inst/tests/runit.transform.R", "content": "\nlibrary(Rcpp)\nlibrary(RUnit)\n\nsourceCpp(system.file(\"tests/cpp/transform.cpp\", package = \"RcppParallel\"))\n\ntest.transform <- function() {\n \n m <- matrix(as.numeric(c(1:1000000)), nrow = 1000, ncol = 1000)\n \n checkEquals(\n matrixSqrt(m), \n parallelMatrixSqrt(m)\n ) \n}\n" }, { "path": "inst/tests/runit.truefalse_macros.R", "content": "\nlibrary(Rcpp)\nlibrary(RUnit)\n\nsourceCpp(system.file(\"tests/cpp/truefalse_macros.cpp\", package = \"RcppParallel\"))\n\n" }, { "path": "man/RcppParallel-package.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/RcppParallel-package.R\n\\docType{package}\n\\name{RcppParallel-package}\n\\alias{RcppParallel-package}\n\\alias{RcppParallel}\n\\title{Parallel programming tools for Rcpp}\n\\description{\nHigh level functions for doing parallel programming with Rcpp. For example,\nthe \\code{parallelFor()} function can be used to convert the work of a\nstandard serial \"for\" loop into a parallel one, and the \\code{parallelReduce()}\nfunction can be used for accumulating aggregate or other values.\n}\n\\details{\nThe high level interface enables safe and robust parallel programming\nwithout direct manipulation of operating system threads. On Windows, macOS,\nand Linux systems the underlying implementation is based on Intel TBB\n(Threading Building Blocks). On other platforms, a less-performant fallback\nimplementation based on the TinyThread library is used.\n\nFor additional documentation, see the package website at:\n\n\\url{https://rcppcore.github.io/RcppParallel/}\n}\n\\keyword{package}\n\\keyword{parallel}\n" }, { "path": "man/RcppParallel.package.skeleton.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/skeleton.R\n\\name{RcppParallel.package.skeleton}\n\\alias{RcppParallel.package.skeleton}\n\\title{Create a skeleton for a new package depending on RcppParallel}\n\\usage{\nRcppParallel.package.skeleton(name = \"anRpackage\", example_code = TRUE, ...)\n}\n\\arguments{\n\\item{name}{The name of your R package.}\n\n\\item{example_code}{If \\code{TRUE}, example C++ code using RcppParallel is\nadded to the package.}\n\n\\item{...}{Optional arguments passed to \\link[Rcpp]{Rcpp.package.skeleton}.}\n}\n\\value{\nNothing, used for its side effects\n}\n\\description{\n\\code{RcppParallel.package.skeleton} automates the creation of a new source\npackage that intends to use features of RcppParallel.\n}\n\\details{\nIt is based on the \\link[utils]{package.skeleton} function which it executes\nfirst.\n\nIn addition to \\link[Rcpp]{Rcpp.package.skeleton} :\n\nThe \\samp{DESCRIPTION} file gains an Imports line requesting that the\npackage depends on RcppParallel and a LinkingTo line so that the package\nfinds RcppParallel header files.\n\nThe \\samp{NAMESPACE} gains a \\code{useDynLib} directive as well as an\n\\code{importFrom(RcppParallel, evalCpp} to ensure instantiation of\nRcppParallel.\n\nThe \\samp{src} directory is created if it does not exists and a\n\\samp{Makevars} file is added setting the environment variables\n\\samp{PKG_LIBS} to accomodate the necessary flags to link with the\nRcppParallel library.\n\nIf the \\code{example_code} argument is set to \\code{TRUE}, example files\n\\samp{vector-sum.cpp} is created in the \\samp{src} directory.\n\\code{Rcpp::compileAttributes()} is then called to generate\n\\code{src/RcppExports.cpp} and \\code{R/RcppExports.R}. These files are given\nas an example and should eventually by removed from the generated package.\n}\n\\examples{\n\n\\dontrun{\n# simple package\nRcppParallel.package.skeleton(\"foobar\")\n}\n\n}\n\\references{\nRead the \\emph{Writing R Extensions} manual for more details.\n\nOnce you have created a \\emph{source} package you need to install it: see\nthe \\emph{R Installation and Administration} manual, \\code{\\link{INSTALL}}\nand \\code{\\link{install.packages}}.\n}\n\\seealso{\n\\link[utils]{package.skeleton}\n}\n\\keyword{programming}\n" }, { "path": "man/flags.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/flags.R\n\\name{flags}\n\\alias{flags}\n\\alias{RcppParallelLibs}\n\\alias{LdFlags}\n\\alias{CxxFlags}\n\\title{Compilation flags for RcppParallel}\n\\usage{\nCxxFlags()\n\nLdFlags()\n\nRcppParallelLibs()\n}\n\\value{\nReturns \\code{NULL}, invisibly. These functions are called for\ntheir side effects (writing the associated flags to stdout).\n}\n\\description{\nOutput the compiler or linker flags required to build against RcppParallel.\n}\n\\details{\nThese functions are typically called from \\code{Makevars} as follows:\\preformatted{PKG_LIBS += $(shell \"$\\{R_HOME\\}/bin/Rscript\" -e \"RcppParallel::LdFlags()\")\n}\n\nOn Windows, the flags ensure that the package links with the built-in TBB\nlibrary. On Linux and macOS, the output is empty, because TBB is loaded\ndynamically on load by \\code{RcppParallel}.\n\n\\R packages using RcppParallel should also add the following to their\n\\code{NAMESPACE} file:\\preformatted{importFrom(RcppParallel, RcppParallelLibs)\n}\n\nThis is necessary to ensure that \\pkg{RcppParallel} (and so, TBB) is loaded\nand available.\n}\n" }, { "path": "man/setThreadOptions.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/options.R\n\\name{setThreadOptions}\n\\alias{setThreadOptions}\n\\alias{defaultNumThreads}\n\\title{Thread options for RcppParallel}\n\\usage{\nsetThreadOptions(numThreads = \"auto\", stackSize = \"auto\")\n\ndefaultNumThreads()\n}\n\\arguments{\n\\item{numThreads}{Number of threads to use for task scheduling. Call \\code{defaultNumThreads()}\nto determine the the default value used for \"auto\".}\n\n\\item{stackSize}{Stack size (in bytes) to use for worker threads. The\ndefault used for \"auto\" is 2MB on 32-bit systems and 4MB on 64-bit systems\n(note that this parameter has no effect on Windows).}\n}\n\\value{\n\\code{defaultNumThreads()} returns the default number of threads used by\nRcppParallel, if another value isn't specified either via\n\\code{setThreadOptions()} or explicitly in calls to \\code{parallelFor()} and\n\\code{parallelReduce()}.\n}\n\\description{\nSet thread options (number of threads to use for task scheduling and stack\nsize per-thread) for RcppParallel.\n}\n\\details{\nRcppParallel is automatically initialized with the default number of threads\nand thread stack size when it loads. You can call \\code{setThreadOptions()} at\nany time to change the defaults.\n\nThe \\code{parallelFor()} and \\code{parallelReduce()} also accept \\code{numThreads} as\nan argument, if you'd like to control the number of threads specifically\nto be made available for a particular parallel function call. Note that\nthis value is advisory, and TBB may choose a smaller number of threads\nif the number of requested threads cannot be honored on the system.\n}\n\\examples{\n\n\\dontrun{\nlibrary(RcppParallel)\nsetThreadOptions(numThreads = 4)\ndefaultNumThreads()\n}\n\n}\n" }, { "path": "man/tbbLibraryPath.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/tbb.R\n\\name{tbbLibraryPath}\n\\alias{tbbLibraryPath}\n\\title{Get the Path to a TBB Library}\n\\usage{\ntbbLibraryPath(name = NULL)\n}\n\\arguments{\n\\item{name}{The name of the TBB library to be resolved. Normally, this is one of\n\\code{tbb}, \\code{tbbmalloc}, or \\code{tbbmalloc_proxy}. When \\code{NULL}, the library\npath containing the TBB libraries is returned instead.}\n}\n\\description{\nRetrieve the path to a TBB library. This can be useful for \\R packages\nusing RcppParallel that wish to use, or re-use, the version of TBB that\nRcppParallel has been configured to use.\n}\n" }, { "path": "patches/windows_arm64.diff", "content": "diff --git a/src/tbb/build/Makefile.tbb b/src/tbb/build/Makefile.tbb\nindex 8d155f80..c58f4fb1 100644\n--- a/src/tbb/build/Makefile.tbb\n+++ b/src/tbb/build/Makefile.tbb\n@@ -91,7 +91,11 @@ ifneq (,$(TBB.DEF))\n tbb.def: $(TBB.DEF) $(TBB.LST)\n \t$(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(INCLUDES) > $@\n \n-LIB_LINK_FLAGS += $(EXPORT_KEY)tbb.def\n+# LLVM on Windows doesn't need --version-script export\n+# https://reviews.llvm.org/D63743\n+ifeq (, $(WINARM64_CLANG))\n+ LIB_LINK_FLAGS += $(EXPORT_KEY)tbb.def\n+endif\n $(TBB.DLL): tbb.def\n endif\n \ndiff --git a/src/tbb/build/Makefile.tbbmalloc b/src/tbb/build/Makefile.tbbmalloc\nindex 421e95c5..e7c38fa4 100644\n--- a/src/tbb/build/Makefile.tbbmalloc\n+++ b/src/tbb/build/Makefile.tbbmalloc\n@@ -74,7 +74,11 @@ ifneq (,$(MALLOC.DEF))\n tbbmalloc.def: $(MALLOC.DEF)\n \t$(CPLUS) $(PREPROC_ONLY) $< $(M_CPLUS_FLAGS) $(WARNING_SUPPRESS) $(INCLUDES) > $@\n \n-MALLOC_LINK_FLAGS += $(EXPORT_KEY)tbbmalloc.def\n+# LLVM on Windows doesn't need --version-script export\n+# https://reviews.llvm.org/D63743\n+ifeq (, $(WINARM64_CLANG))\n+ MALLOC_LINK_FLAGS += $(EXPORT_KEY)tbbmalloc.def\n+endif\n $(MALLOC.DLL): tbbmalloc.def\n endif\n \ndiff --git a/src/tbb/src/tbbmalloc/TypeDefinitions.h b/src/tbb/src/tbbmalloc/TypeDefinitions.h\nindex 3178442e..fd4b7956 100644\n--- a/src/tbb/src/tbbmalloc/TypeDefinitions.h\n+++ b/src/tbb/src/tbbmalloc/TypeDefinitions.h\n@@ -25,7 +25,7 @@\n # define __ARCH_ipf 1\n # elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support\n # define __ARCH_x86_32 1\n-# elif defined(_M_ARM)\n+# elif defined(_M_ARM) || defined(__aarch64__)\n # define __ARCH_other 1\n # else\n # error Unknown processor architecture for Windows\n" }, { "path": "src/.gitignore", "content": "*.o\n*.so\n*.dll\n\nMakevars\nMakevars.win\n" }, { "path": "src/Makevars.in", "content": "\n# This needs to expand to something the shell will accept as '$ORIGIN',\n# including a literal '$' (no variable expansion)\nORIGIN = \\$$ORIGIN\n\nCMAKE = @CMAKE@\nR = @R@\n\nTBB_LIB = @TBB_LIB@\nTBB_INC = @TBB_INC@\nTBB_NAME = @TBB_NAME@\nTBB_MALLOC_NAME = @TBB_MALLOC_NAME@\n\nPKG_CPPFLAGS = @PKG_CPPFLAGS@\nPKG_CXXFLAGS = @PKG_CXXFLAGS@\n\nPKG_LIBS = @PKG_LIBS@ @PKG_LIBS_EXTRA@\n\nall: tbb $(SHLIB)\n\n# TBB needs to be built before our C++ sources are built, so that\n# headers are copied and available from the expected locations.\n$(OBJECTS): tbb\n\n# NOTE: TBB libraries are installed via install.libs.R.\n# However, we need to copy headers here so that they are visible during compilation.\ntbb: tbb-clean\n\t@TBB_LIB=\"$(TBB_LIB)\" TBB_INC=\"$(TBB_INC)\" \\\n\tTBB_NAME=\"$(TBB_NAME)\" TBB_MALLOC_NAME=\"$(TBB_MALLOC_NAME)\" \\\n\tCC=\"$(CC)\" CFLAGS=\"$(CFLAGS)\" CPPFLAGS=\"$(CPPFLAGS)\" \\\n\tCXX=\"$(CXX)\" CXXFLAGS=\"$(CXXFLAGS)\"\tLDFLAGS=\"$(LDFLAGS)\" \\\n\tCMAKE=\"$(CMAKE)\" \"@R@\" -s -f install.libs.R --args build\n\n# NOTE: we do not want to clean ../inst/lib or ../inst/libs here,\n# as we may be writing to those locations in multiarch builds\ntbb-clean:\n\t@rm -rf ../inst/include/tbb\n\t@rm -rf ../inst/include/oneapi\n\t@rm -rf ../inst/include/tbb_local\n\t@rm -rf ../inst/include/serial\n" }, { "path": "src/init.cpp", "content": "\n#include \n#include \n#include // for NULL\n#include \n\n/* .Call calls */\nextern \"C\" SEXP defaultNumThreads();\n\nstatic const R_CallMethodDef CallEntries[] = {\n {\"defaultNumThreads\", (DL_FUNC) &defaultNumThreads, 0},\n {NULL, NULL, 0}\n};\n\nextern \"C\" void R_init_RcppParallel(DllInfo *dll)\n{\n R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);\n R_useDynamicSymbols(dll, FALSE);\n}\n" }, { "path": "src/install.libs.R", "content": "\n# !diagnostics suppress=R_PACKAGE_DIR,SHLIB_EXT,R_ARCH\n.install.libs <- function(tbbLib) {\n\n # copy default library\n files <- Sys.glob(paste0(\"*\", SHLIB_EXT))\n dest <- file.path(R_PACKAGE_DIR, paste0(\"libs\", R_ARCH))\n dir.create(dest, recursive = TRUE, showWarnings = FALSE)\n file.copy(files, dest, overwrite = TRUE)\n\n # copy symbols if available\n if (file.exists(\"symbols.rds\"))\n file.copy(\"symbols.rds\", dest, overwrite = TRUE)\n\n # also copy to package 'libs' folder, for devtools\n libsDest <- paste0(\"../libs\", R_ARCH)\n dir.create(libsDest, recursive = TRUE, showWarnings = FALSE)\n file.copy(files, libsDest, overwrite = TRUE)\n\n # copy tbb (NOTE: do not use inst/ folder as R will resolve symlinks,\n # behavior which we do _not_ want here!)\n tbbDest <- file.path(R_PACKAGE_DIR, paste0(\"lib\", R_ARCH))\n dir.create(tbbDest, recursive = TRUE, showWarnings = FALSE)\n\n # note: on Linux, TBB gets compiled with extensions like\n # '.so.2', so be ready to handle those\n shlibPattern <- switch(\n Sys.info()[[\"sysname\"]],\n Windows = \"^tbb.*\\\\.dll$\",\n Darwin = \"^libtbb.*\\\\.dylib$\",\n \"^libtbb.*\\\\.so.*$\"\n )\n # WASM only supports static libraries\n if (R.version$os == \"emscripten\") {\n shlibPattern <- \"^libtbb.*\\\\.a$\"\n }\n\n if (!nzchar(tbbLib)) {\n\n # using bundled TBB\n tbbLibs <- list.files(\n path = \"tbb/build/lib_release\",\n pattern = shlibPattern,\n full.names = TRUE\n )\n\n for (tbbLib in tbbLibs) {\n system2(\"cp\", c(\"-P\", shQuote(tbbLib), shQuote(tbbDest)))\n }\n\n } else {\n\n # using system tbb\n tbbLibs <- list.files(\n path = tbbLib,\n pattern = shlibPattern,\n full.names = TRUE\n )\n\n # don't copy symlinks\n tbbLibs <- tbbLibs[!nzchar(Sys.readlink(tbbLibs))]\n\n # copy / link the libraries\n useSymlinks <- Sys.getenv(\"TBB_USE_SYMLINKS\", unset = .Platform$OS.type != \"windows\")\n if (useSymlinks) {\n file.symlink(tbbLibs, tbbDest)\n } else {\n for (tbbLib in tbbLibs) {\n system2(\"cp\", c(\"-P\", shQuote(tbbLib), shQuote(tbbDest)))\n }\n }\n\n }\n\n # on Windows, we create a stub library that links to us so that\n # older binaries (like rstan) can still load\n if (.Platform$OS.type == \"windows\") {\n tbbDll <- file.path(tbbDest, \"tbb.dll\")\n if (!file.exists(tbbDll)) {\n writeLines(\"** creating tbb stub library\")\n status <- system(\"R CMD SHLIB -o tbb-compat/tbb.dll tbb-compat/tbb-compat.cpp\")\n if (status != 0)\n stop(\"error building tbb stub library\")\n file.copy(\"tbb-compat/tbb.dll\", file.path(tbbDest, \"tbb.dll\"))\n }\n }\n\n}\n\nuseTbbPreamble <- function(tbbInc) {\n dir.create(\"../inst/include\", recursive = TRUE, showWarnings = FALSE)\n for (suffix in c(\"oneapi\", \"serial\", \"tbb\")) {\n tbbPath <- file.path(tbbInc, suffix)\n if (file.exists(tbbPath)) {\n file.copy(tbbPath, \"../inst/include\", recursive = TRUE)\n }\n }\n}\n\nuseSystemTbb <- function(tbbLib, tbbInc) {\n useTbbPreamble(tbbInc)\n}\n\nuseBundledTbb <- function() {\n\n useTbbPreamble(\"tbb/include\")\n dir.create(\"tbb/build-tbb\", showWarnings = FALSE)\n\n cmake <- Sys.getenv(\"CMAKE\", unset = \"cmake\")\n buildType <- Sys.getenv(\"CMAKE_BUILD_TYPE\", unset = \"Release\")\n verbose <- Sys.getenv(\"VERBOSE\", unset = \"0\")\n\n splitCompilerVar(\"CC\", \"CFLAGS\")\n splitCompilerVar(\"CXX\", \"CXXFLAGS\")\n\n prependFlags(\"CPPFLAGS\", \"CFLAGS\")\n prependFlags(\"CPPFLAGS\", \"CXXFLAGS\")\n\n cmakeFlags <- c(\n forwardEnvVar(\"CC\", \"CMAKE_C_COMPILER\"),\n forwardEnvVar(\"CXX\", \"CMAKE_CXX_COMPILER\"),\n forwardEnvVar(\"CFLAGS\", \"CMAKE_C_FLAGS\"),\n forwardEnvVar(\"CXXFLAGS\", \"CMAKE_CXX_FLAGS\"),\n forwardEnvVar(\"CMAKE_BUILD_TYPE\", \"CMAKE_BUILD_TYPE\"),\n \"-DTBB_TEST=0\",\n \"-DTBB_EXAMPLES=0\",\n \"-DTBB_STRICT=0\",\n \"..\"\n )\n\n if (R.version$os == \"emscripten\") {\n cmakeFlags <- c(\n \"-DEMSCRIPTEN=1\",\n \"-DTBBMALLOC_BUILD=0\",\n \"-DBUILD_SHARED_LIBS=0\",\n cmakeFlags\n )\n }\n\n writeLines(\"*** configuring tbb\")\n owd <- setwd(\"tbb/build-tbb\")\n output <- system2(cmake, shQuote(cmakeFlags), stdout = TRUE, stderr = TRUE)\n status <- attr(output, \"status\")\n if (is.numeric(status) && status != 0L) {\n writeLines(output)\n stop(\"error configuring tbb (status code \", status, \")\")\n } else if (!identical(verbose, \"0\")) {\n writeLines(output)\n }\n setwd(owd)\n\n if (!identical(verbose, \"0\")) {\n writeLines(\"*** dumping CMakeCache.txt\")\n writeLines(readLines(\"tbb/build-tbb/CMakeCache.txt\"))\n }\n\n writeLines(\"*** building tbb\")\n owd <- setwd(\"tbb/build-tbb\")\n output <- system2(cmake, c(\"--build\", \".\", \"--config\", \"release\"), stdout = TRUE, stderr = TRUE)\n status <- attr(output, \"status\")\n if (is.numeric(status) && status != 0L) {\n writeLines(output)\n stop(\"error building tbb (status code \", status, \")\")\n } else if (!identical(verbose, \"0\")) {\n writeLines(output)\n }\n setwd(owd)\n\n shlibPattern <- switch(\n Sys.info()[[\"sysname\"]],\n Windows = \"^tbb.*\\\\.dll$\",\n Darwin = \"^libtbb.*\\\\.dylib$\",\n \"^libtbb.*\\\\.so.*$\"\n )\n\n # WASM only supports static libraries\n if (R.version$os == \"emscripten\") {\n shlibPattern <- \"^libtbb.*\\\\.a$\"\n }\n\n tbbFiles <- list.files(\n file.path(getwd(), \"tbb/build-tbb\"),\n pattern = shlibPattern,\n recursive = TRUE,\n full.names = TRUE\n )\n\n dir.create(\"tbb/build/lib_release\", recursive = TRUE, showWarnings = FALSE)\n file.copy(tbbFiles, \"tbb/build/lib_release\", overwrite = TRUE)\n unlink(\"tbb/build-tbb\", recursive = TRUE)\n writeLines(\"*** finished building tbb\")\n\n}\n\ngetenv <- function(key, unset = \"\") {\n Sys.getenv(key, unset = unset)\n}\n\n\nsetenv <- function(key, value) {\n args <- list(paste(value, collapse = \" \"))\n names(args) <- key\n do.call(Sys.setenv, args)\n}\n\n\n# CMake doesn't support flags specified directly as part of the compiler\n# definition, so manually split it here.\nsplitCompilerVar <- function(compilerVar, flagsVar) {\n\n compiler <- Sys.getenv(compilerVar, unset = NA)\n if (is.na(compiler))\n return(FALSE)\n\n tokens <- scan(text = compiler, what = character(), quiet = TRUE)\n if (length(tokens) < 2L)\n return(FALSE)\n\n setenv(compilerVar, tokens[[1L]])\n\n oldFlags <- Sys.getenv(flagsVar)\n newFlags <- c(tokens[-1L], oldFlags)\n setenv(flagsVar, newFlags[nzchar(newFlags)])\n\n TRUE\n\n}\n\n\n# Given an environment variable like 'CC', forward that to CMake\n# via the corresponding CMAKE_C_COMPILER flag.\nforwardEnvVar <- function(envVar, cmakeVar) {\n envVal <- Sys.getenv(envVar, unset = NA)\n if (!is.na(envVal)) {\n sprintf(\"-D%s=%s\", cmakeVar, envVal)\n }\n}\n\nprependFlags <- function(prependFlags, toFlags) {\n\n prependVal <- Sys.getenv(prependFlags, unset = NA)\n if (is.na(prependVal))\n return(FALSE)\n\n oldVal <- Sys.getenv(toFlags, unset = NA)\n if (is.na(oldVal)) {\n setenv(toFlags, prependVal)\n } else {\n setenv(toFlags, paste(prependVal, oldVal))\n }\n\n TRUE\n\n}\n\n# Main ----\n\ntbbLib <- Sys.getenv(\"TBB_LIB\")\ntbbInc <- Sys.getenv(\"TBB_INC\")\n\nargs <- commandArgs(trailingOnly = TRUE)\nif (identical(args, \"build\")) {\n if (nzchar(tbbLib) && nzchar(tbbInc)) {\n useSystemTbb(tbbLib, tbbInc)\n } else if (.Platform$OS.type == \"windows\") {\n writeLines(\"** building RcppParallel without tbb backend\")\n } else {\n useBundledTbb()\n }\n} else {\n source(\"../R/tbb-autodetected.R\")\n .install.libs(tbbLib)\n}\n" }, { "path": "src/options.cpp", "content": "\n#include \n\n#include \n\n#if RCPP_PARALLEL_USE_TBB // TBB support turned on\n\n#include \n#include \n\nextern \"C\" SEXP defaultNumThreads() {\n SEXP threadsSEXP = Rf_allocVector(INTSXP, 1);\n#ifdef __TBB_task_arena_H\n INTEGER(threadsSEXP)[0] = tbb::this_task_arena::max_concurrency();\n#else\n INTEGER(threadsSEXP)[0] = tbb::task_scheduler_init::default_num_threads();\n#endif\n return threadsSEXP;\n}\n\n#else // TBB support not turned on\n\n#include \n\nextern \"C\" SEXP defaultNumThreads() {\n SEXP threadsSEXP = Rf_allocVector(INTSXP, 1);\n INTEGER(threadsSEXP)[0] = tthread::thread::hardware_concurrency();\n return threadsSEXP;\n}\n\n#endif\n" }, { "path": "src/tbb/.gitignore", "content": "# -------- C++ --------\n# Prerequisites\n*.d\n\n# Compiled Object files\n*.slo\n*.lo\n*.o\n*.obj\n\n# Precompiled Headers\n*.gch\n*.pch\n\n# Compiled Dynamic libraries\n*.so\n*.so.*\n*.dylib\n*.dll\n\n# Fortran module files\n*.mod\n*.smod\n\n# Compiled Static libraries\n*.lai\n*.la\n*.a\n*.lib\n\n# Executables\n*.exe\n*.out\n*.app\n\n# -------- CMake --------\nCMakeCache.txt\nCMakeFiles\nCMakeScripts\nTesting\nMakefile\ncmake_install.cmake\ninstall_manifest.txt\ncompile_commands.json\nCTestTestfile.cmake\nbuild/*\n\n# -------- Python --------\n__pycache__/\n*.py[cod]\n*$py.class\n\n# -------- IDE --------\n.vscode/*\n.vs/*\nout/*\nCMakeSettings.json\n\n# -------- CTags --------\n.tags\n.ctags\n\n" }, { "path": "src/tbb/CMakeLists.txt", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ncmake_minimum_required(VERSION 3.5)\n\n# Enable CMake policies\n\nif (POLICY CMP0068)\n # RPATH settings do not affect install_name on macOS since CMake 3.9\n cmake_policy(SET CMP0068 NEW)\nendif()\n\nif (POLICY CMP0091)\n # The NEW behavior for this policy is to not place MSVC runtime library flags in the default\n # CMAKE__FLAGS_ cache entries and use CMAKE_MSVC_RUNTIME_LIBRARY abstraction instead.\n cmake_policy(SET CMP0091 NEW)\nelseif (DEFINED CMAKE_MSVC_RUNTIME_LIBRARY)\n message(FATAL_ERROR \"CMAKE_MSVC_RUNTIME_LIBRARY was defined while policy CMP0091 is not available. Use CMake 3.15 or newer.\")\nendif()\n\nif (TBB_WINDOWS_DRIVER AND (NOT (\"${CMAKE_MSVC_RUNTIME_LIBRARY}\" STREQUAL MultiThreaded OR \"${CMAKE_MSVC_RUNTIME_LIBRARY}\" STREQUAL MultiThreadedDebug)))\n message(FATAL_ERROR \"Enabled TBB_WINDOWS_DRIVER requires CMAKE_MSVC_RUNTIME_LIBRARY to be set to MultiThreaded or MultiThreadedDebug.\")\nendif()\n\n# Enable support of minimum supported macOS version flag\nif (APPLE)\n if (NOT CMAKE_CXX_OSX_DEPLOYMENT_TARGET_FLAG)\n set(CMAKE_CXX_OSX_DEPLOYMENT_TARGET_FLAG \"-mmacosx-version-min=\" CACHE STRING \"Minimum macOS version flag\")\n endif()\n if (NOT CMAKE_C_OSX_DEPLOYMENT_TARGET_FLAG)\n set(CMAKE_C_OSX_DEPLOYMENT_TARGET_FLAG \"-mmacosx-version-min=\" CACHE STRING \"Minimum macOS version flag\")\n endif()\nendif()\n\nfile(READ include/oneapi/tbb/version.h _tbb_version_info)\nstring(REGEX REPLACE \".*#define TBB_VERSION_MAJOR ([0-9]+).*\" \"\\\\1\" _tbb_ver_major \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define TBB_VERSION_MINOR ([0-9]+).*\" \"\\\\1\" _tbb_ver_minor \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define TBB_VERSION_PATCH ([0-9]+).*\" \"\\\\1\" _tbb_ver_patch \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define TBB_INTERFACE_VERSION ([0-9]+).*\" \"\\\\1\" TBB_INTERFACE_VERSION \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define __TBB_BINARY_VERSION ([0-9]+).*\" \"\\\\1\" TBB_BINARY_VERSION \"${_tbb_version_info}\")\nstring(REGEX REPLACE \"..(..).\" \"\\\\1\" TBB_BINARY_MINOR_VERSION \"${TBB_INTERFACE_VERSION}\")\nset(TBBMALLOC_BINARY_VERSION 2)\nset(TBBBIND_BINARY_VERSION 3)\n\nproject(TBB VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch} LANGUAGES CXX)\nunset(_tbb_ver_major)\nunset(_tbb_ver_minor)\n\ninclude(CheckCXXCompilerFlag)\ninclude(GNUInstallDirs)\ninclude(CMakeDependentOption)\n\n# ---------------------------------------------------------------------------------------------------------\n# Handle C++ standard version.\nif (NOT MSVC) # no need to cover MSVC as it uses C++14 by default.\n if (NOT CMAKE_CXX_STANDARD)\n set(CMAKE_CXX_STANDARD 11)\n endif()\n\n if (CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION) # if standard option was detected by CMake\n set(CMAKE_CXX_STANDARD_REQUIRED ON)\n else() # if standard option wasn't detected by CMake (e.g. for Intel Compiler with CMake 3.1)\n # TBB_CXX_STD_FLAG should be added to targets via target_compile_options\n set(TBB_CXX_STD_FLAG -std=c++${CMAKE_CXX_STANDARD})\n\n check_cxx_compiler_flag(${TBB_CXX_STD_FLAG} c++${CMAKE_CXX_STANDARD})\n if (NOT c++${CMAKE_CXX_STANDARD})\n message(FATAL_ERROR \"C++${CMAKE_CXX_STANDARD} (${TBB_CXX_STD_FLAG}) support is required\")\n endif()\n unset(c++${CMAKE_CXX_STANDARD})\n endif()\nendif()\n\nset(CMAKE_CXX_EXTENSIONS OFF) # use -std=c++... instead of -std=gnu++...\n# ---------------------------------------------------------------------------------------------------------\n\n# Detect architecture (bitness).\nif (CMAKE_SIZEOF_VOID_P EQUAL 4)\n set(TBB_ARCH 32)\nelse()\n set(TBB_ARCH 64)\nendif()\n\noption(TBB_TEST \"Enable testing\" ON)\noption(TBB_EXAMPLES \"Enable examples\" OFF)\noption(TBB_STRICT \"Treat compiler warnings as errors\" ON)\noption(TBB_WINDOWS_DRIVER \"Build as Universal Windows Driver (UWD)\" OFF)\noption(TBB_NO_APPCONTAINER \"Apply /APPCONTAINER:NO (for testing binaries for Windows Store)\" OFF)\noption(TBB4PY_BUILD \"Enable tbb4py build\" OFF)\noption(TBB_BUILD \"Enable tbb build\" ON)\noption(TBBMALLOC_BUILD \"Enable tbbmalloc build\" ON)\ncmake_dependent_option(TBBMALLOC_PROXY_BUILD \"Enable tbbmalloc_proxy build\" ON \"TBBMALLOC_BUILD\" OFF)\noption(TBB_CPF \"Enable preview features of the library\" OFF)\noption(TBB_FIND_PACKAGE \"Enable search for external oneTBB using find_package instead of build from sources\" OFF)\noption(TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH \"Disable HWLOC automatic search by pkg-config tool\" ${CMAKE_CROSSCOMPILING})\noption(TBB_ENABLE_IPO \"Enable Interprocedural Optimization (IPO) during the compilation\" ON)\noption(TBB_FUZZ_TESTING \"Enable fuzz testing\" OFF)\noption(TBB_INSTALL \"Enable installation\" ON)\nif(LINUX)\noption(TBB_LINUX_SEPARATE_DBG \"Enable separation of the debug symbols during the build\" OFF)\nendif()\nif(APPLE)\noption(TBB_BUILD_APPLE_FRAMEWORKS \"Build as Apple Frameworks\" OFF)\nendif()\n\nif (NOT DEFINED BUILD_SHARED_LIBS)\n set(BUILD_SHARED_LIBS ON)\nendif()\n\nif (NOT BUILD_SHARED_LIBS)\n if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)\n set(CMAKE_POSITION_INDEPENDENT_CODE ON)\n endif()\n message(WARNING \"You are building oneTBB as a static library. This is highly discouraged and such configuration is not supported. Consider building a dynamic library to avoid unforeseen issues.\")\nendif()\n\nif (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)\n set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING \"Build type\" FORCE)\n message(STATUS \"CMAKE_BUILD_TYPE is not specified. Using default: ${CMAKE_BUILD_TYPE}\")\n # Possible values of build type for cmake-gui\n set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS \"Debug\" \"Release\" \"MinSizeRel\" \"RelWithDebInfo\")\nendif()\n\nif (CMAKE_BUILD_TYPE)\n string(TOLOWER ${CMAKE_BUILD_TYPE} _tbb_build_type)\n if (_tbb_build_type STREQUAL \"debug\")\n set(TBB_ENABLE_IPO OFF)\n endif()\n unset(_tbb_build_type)\nendif()\n\n# -------------------------------------------------------------------\n# Files and folders naming\nset(CMAKE_DEBUG_POSTFIX _debug)\n\nif (NOT DEFINED TBB_OUTPUT_DIR_BASE)\n if (MSVC)\n if (NOT DEFINED CMAKE_MSVC_RUNTIME_LIBRARY OR CMAKE_MSVC_RUNTIME_LIBRARY MATCHES DLL)\n set(_tbb_msvc_runtime _md)\n else()\n set(_tbb_msvc_runtime _mt)\n endif()\n\n if (WINDOWS_STORE)\n if (TBB_NO_APPCONTAINER)\n set(_tbb_win_store _wsnoappcont)\n else()\n set(_tbb_win_store _ws)\n endif()\n elseif(TBB_WINDOWS_DRIVER)\n set(_tbb_win_store _wd)\n endif()\n endif()\n\n string(REGEX MATCH \"^([0-9]+\\.[0-9]+|[0-9]+)\" _tbb_compiler_version_short ${CMAKE_CXX_COMPILER_VERSION})\n string(TOLOWER ${CMAKE_CXX_COMPILER_ID}_${_tbb_compiler_version_short}_cxx${CMAKE_CXX_STANDARD}_${TBB_ARCH}${_tbb_msvc_runtime}${_tbb_win_store} TBB_OUTPUT_DIR_BASE)\n unset(_tbb_msvc_runtime)\n unset(_tbb_win_store)\n unset(_tbb_compiler_version_short)\nendif()\n\nforeach(output_type LIBRARY ARCHIVE PDB RUNTIME)\n if (CMAKE_BUILD_TYPE)\n string(TOLOWER ${CMAKE_BUILD_TYPE} _tbb_build_type_lower)\n set(CMAKE_${output_type}_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_${_tbb_build_type_lower})\n unset(_tbb_build_type_lower)\n endif()\n\n if (CMAKE_CONFIGURATION_TYPES)\n foreach(suffix ${CMAKE_CONFIGURATION_TYPES})\n string(TOUPPER ${suffix} _tbb_suffix_upper)\n string(TOLOWER ${suffix} _tbb_suffix_lower)\n set(CMAKE_${output_type}_OUTPUT_DIRECTORY_${_tbb_suffix_upper} ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_${_tbb_suffix_lower})\n endforeach()\n unset(_tbb_suffix_lower)\n unset(_tbb_suffix_upper)\n endif()\nendforeach()\n\nif (CMAKE_CONFIGURATION_TYPES)\n # We can't use generator expressions in a cmake variable name.\n set(TBB_TEST_WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${TBB_OUTPUT_DIR_BASE}_$>)\nelse()\n set(TBB_TEST_WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})\nendif()\n\n# -------------------------------------------------------------------\n\n# -------------------------------------------------------------------\n# Common dependencies\n#force -pthread during compilation for Emscripten\nif (EMSCRIPTEN AND NOT EMSCRIPTEN_WITHOUT_PTHREAD)\n set(THREADS_HAVE_PTHREAD_ARG TRUE)\nendif()\n\nset(THREADS_PREFER_PTHREAD_FLAG TRUE)\nfind_package(Threads REQUIRED)\n# -------------------------------------------------------------------\n\nfile(GLOB FILES_WITH_EXTRA_TARGETS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/*.cmake)\nforeach(FILE_WITH_EXTRA_TARGETS ${FILES_WITH_EXTRA_TARGETS})\n include(${FILE_WITH_EXTRA_TARGETS})\nendforeach()\n\n# - Enabling LTO on Android causes the NDK bug.\n# NDK throws the warning: \"argument unused during compilation: '-Wa,--noexecstack'\"\n# - For some reason GCC does not instrument code with Thread Sanitizer when lto is enabled and C linker is used.\nif (TBB_ENABLE_IPO AND BUILD_SHARED_LIBS AND NOT ANDROID_PLATFORM AND NOT TBB_SANITIZE MATCHES \"thread\")\n if (NOT CMAKE_VERSION VERSION_LESS 3.9)\n cmake_policy(SET CMP0069 NEW)\n include(CheckIPOSupported)\n check_ipo_supported(RESULT TBB_IPO_PROPERTY)\n else()\n set(TBB_IPO_FLAGS TRUE)\n endif()\n if (TBB_IPO_PROPERTY OR TBB_IPO_FLAGS)\n message(STATUS \"IPO enabled\")\n endif()\nendif()\n\nset(TBB_COMPILER_SETTINGS_FILE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilers/${CMAKE_CXX_COMPILER_ID}.cmake)\nif (EXISTS ${TBB_COMPILER_SETTINGS_FILE})\n include(${TBB_COMPILER_SETTINGS_FILE})\nelse()\n message(WARNING \"TBB compiler settings not found ${TBB_COMPILER_SETTINGS_FILE}\")\nendif()\n\nif (TBB_FIND_PACKAGE AND TBB_DIR)\n # Allow specifying external TBB to test with.\n # Do not add main targets and installation instructions in that case.\n message(STATUS \"Using external TBB for testing\")\n find_package(TBB REQUIRED)\nelse()\n if (TBB_BUILD)\n add_subdirectory(src/tbb)\n endif()\n if (TBBMALLOC_BUILD)\n add_subdirectory(src/tbbmalloc)\n if(TBBMALLOC_PROXY_BUILD AND NOT \"${MSVC_CXX_ARCHITECTURE_ID}\" MATCHES \"ARM64\")\n add_subdirectory(src/tbbmalloc_proxy)\n endif()\n endif()\n if (NOT BUILD_SHARED_LIBS)\n message(STATUS \"TBBBind build targets are disabled due to unsupported environment\")\n else()\n add_subdirectory(src/tbbbind)\n endif()\n if (TBB_INSTALL)\n # -------------------------------------------------------------------\n # Installation instructions\n include(CMakePackageConfigHelpers)\n\n install(DIRECTORY include/\n DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}\n COMPONENT devel)\n\n install(EXPORT ${PROJECT_NAME}Targets\n NAMESPACE TBB::\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\n COMPONENT devel)\n file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake\n \"include(\\${CMAKE_CURRENT_LIST_DIR}/${PROJECT_NAME}Targets.cmake)\\n\")\n if (NOT BUILD_SHARED_LIBS)\n file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake\n \"include(CMakeFindDependencyMacro)\\nfind_dependency(Threads)\\n\")\n endif()\n\n write_basic_package_version_file(\"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake\"\n COMPATIBILITY AnyNewerVersion)\n\n install(FILES \"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake\"\n \"${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake\"\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}\n COMPONENT devel)\n\n install(FILES \"README.md\"\n DESTINATION ${CMAKE_INSTALL_DOCDIR}\n COMPONENT devel)\n # -------------------------------------------------------------------\n endif()\nendif()\n\nif (TBB_TEST)\n enable_testing()\n add_subdirectory(test)\nendif()\n\nif (TBB_EXAMPLES)\n add_subdirectory(examples)\nendif()\n\nif (TBB_BENCH)\n if (NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/benchmark)\n message(FATAL_ERROR \"Benchmarks are not supported yet\")\n endif()\n\n enable_testing()\n add_subdirectory(benchmark)\nendif()\n\nif (ANDROID_PLATFORM)\n if (\"${ANDROID_STL}\" STREQUAL \"c++_shared\")\n if (${ANDROID_NDK_MAJOR} GREATER_EQUAL \"25\")\n if(ANDROID_ABI STREQUAL \"arm64-v8a\")\n set(ANDROID_TOOLCHAIN_NAME \"aarch64-linux-android\")\n elseif(ANDROID_ABI STREQUAL \"x86_64\")\n set(ANDROID_TOOLCHAIN_NAME \"x86_64-linux-android\")\n elseif(ANDROID_ABI STREQUAL \"armeabi-v7a\")\n set(ANDROID_TOOLCHAIN_NAME \"arm-linux-androideabi\")\n elseif(ANDROID_ABI STREQUAL \"x86\")\n set(ANDROID_TOOLCHAIN_NAME \"i686-linux-android\")\n endif()\n\n configure_file(\n \"${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/lib/${ANDROID_TOOLCHAIN_NAME}/libc++_shared.so\"\n \"${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libc++_shared.so\"\n COPYONLY)\n else()\n configure_file(\n \"${ANDROID_NDK}/sources/cxx-stl/llvm-libc++/libs/${ANDROID_ABI}/libc++_shared.so\"\n \"${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/libc++_shared.so\"\n COPYONLY)\n endif()\n endif()\n # This custom target may be implemented without separate CMake script, but it requires\n # ADB(Android Debug Bridge) executable file availability, so to incapsulate this requirement\n # only for corresponding custom target, it was implemented by this way.\n add_custom_target(device_environment_cleanup COMMAND ${CMAKE_COMMAND}\n -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/android/device_environment_cleanup.cmake)\nendif()\n\nif (TBB4PY_BUILD)\n add_subdirectory(python)\nendif()\n\n# Keep it the last instruction.\nadd_subdirectory(cmake/post_install)\n" }, { "path": "src/tbb/CODEOWNERS", "content": "# Where component owners are known, add them here.\n\n/oneTBB/src/tbb/ @pavelkumbrasev\n/oneTBB/src/tbb/ @dnmokhov\n/oneTBB/src/tbb/ @JhaShweta1\n/oneTBB/src/tbb/ @sarathnandu\n/oneTBB/include/oneapi/tbb/parallel_* @pavelkumbrasev\n/oneTBB/include/oneapi/tbb/concurrent_* @kboyarinov\n/oneTBB/include/oneapi/tbb/flow_graph* @kboyarinov\n/oneTBB/include/oneapi/tbb/flow_graph* @aleksei-fedotov\n/oneTBB/include/oneapi/tbb/detail/_flow_graph* @kboyarinov\n/oneTBB/include/oneapi/tbb/detail/_flow_graph* @aleksei-fedotov\n/oneTBB/include/oneapi/tbb/detail/_concurrent* @kboyarinov\n/oneTBB/src/doc @aepanchi\n/oneTBB/src/tbbbind/ @isaevil\n/oneTBB/src/tbbmalloc/ @lplewa\n/oneTBB/src/tbbmalloc_proxy/ @lplewa\n/oneTBB/cmake/ @isaevil\n/oneTBB/*CMakeLists.txt @isaevil\n/oneTBB/python/ @sarathnandu\n/oneTBB/python/ @isaevil\n\n# Bazel build related files.\n/oneTBB/.bazelversion @Vertexwahn\n/oneTBB/Bazel.md @Vertexwahn\n/oneTBB/BUILD.bazel @Vertexwahn\n/oneTBB/MODULE.bazel @Vertexwahn\n" }, { "path": "src/tbb/CODE_OF_CONDUCT.md", "content": "\n# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, caste, color, religion, or sexual\nidentity and orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the overall\n community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or advances of\n any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email address,\n without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official email address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\noneTBBCodeOfConduct At intel DOT com.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series of\nactions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or permanent\nban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within the\ncommunity.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.1, available at\n[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].\n\nCommunity Impact Guidelines were inspired by\n[Mozilla's code of conduct enforcement ladder][Mozilla CoC].\n\nFor answers to common questions about this code of conduct, see the FAQ at\n[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at\n[https://www.contributor-covenant.org/translations][translations].\n\n[homepage]: https://www.contributor-covenant.org\n[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html\n[Mozilla CoC]: https://github.com/mozilla/diversity\n[FAQ]: https://www.contributor-covenant.org/faq\n[translations]: https://www.contributor-covenant.org/translations\n\n" }, { "path": "src/tbb/CONTRIBUTING.md", "content": "\n\n# How to Contribute\nAs an open source project, we welcome community contributions to oneAPI Threading Building Blocks (oneTBB). This document explains how to participate in project conversations, log bugs and enhancement requests, and submit code patches to the project. \n\n## Licensing \n\nLicensing is very important to open source projects. It helps ensure the software continues to be available under the terms that the author desired. The oneTBB project uses the [Apache 2.0 License](https://github.com/oneapi-src/oneTBB/blob/master/LICENSE.txt), a permissive open source license that allows you to freely use, modify, and distribute your own products that include Apache 2.0 licensed software. By contributing to the oneTBB project, you agree to the license and copyright terms therein and release your own contributions under these terms. \n\nSome imported or reused components within oneTBB use other licenses, as described in [third-party-programs.txt](https://github.com/oneapi-src/oneTBB/blob/master/third-party-programs.txt). By carefully reviewing potential contributions, we can ensure that the community can develop products with oneTBB without concerns over patent or copyright issues. \n\n## Prerequisites \n\nAs a contributor, you’ll want to be familiar with the oneTBB project and the repository layout. You should also know how to use it as explained in the [oneTBB documentation](https://oneapi-src.github.io/oneTBB/) and how to set up your build development environment to configure, build, and test oneTBB as explained in the [oneTBB Build System Description](cmake/README.md). \n\n## Pull Requests \n\nYou can find all [open oneTBB pull requests](https://github.com/oneapi-src/oneTBB/pulls) on GitHub. \n \n### Before contributing changes directly to the oneTBB repository\n\n* Make sure you can build the product and run all the tests with your patch. \n* For a larger feature, provide a relevant test. \n* Document your code. The oneTBB project uses reStructuredText for documentation. \n* Update the copyright year in the first line of the changing file(s). \n For example, if you commit your changes in 2022:\n * the copyright year should be `2005-2022` for existing files\n * the copyright year should be `2022` for new files\n* Submit a pull request into the master branch. You can submit changes with a pull request (preferred) or by sending an email patch. \n\nContinuous Integration (CI) testing is enabled for the repository. Your pull request must pass all checks before it can be merged. We will review your contribution and may provide feedback to guide you if any additional fixes or modifications are necessary. When reviewed and accepted, your pull request will be merged into our GitHub repository. \n" }, { "path": "src/tbb/INSTALL.md", "content": "# Installation from Sources\n\n\n## Prerequisites \n \n - Make sure you have installed CMake version 3.1 (or newer) on your system. oneTBB uses CMake build configuration.\n - Configure and build oneTBB. To work with build configurations, see [Build System Description](cmake/README.md). \n\n\n## Configure oneTBB\n\nAt the command prompt, type:\n```\ncmake \n```\n\nYou may want to use some additional options for configuration:\n\n| Option | Purpose | Description |\n| ------ |------ | ------ |\n| `-G ` | Specify project generator | For more information, run cmake `–help`. |\n|`-DCMAKE_BUILD_TYPE=Debug` | Specify for Debug build | Not applicable for multi-configuration generators such as Visual Studio generator. |\n\n\n## Build oneTBB\n \nTo build the system, run:\n```\ncmake --build . \n```\n\nSome useful build options:\n- `--target ` - specific target, \"all\" is default.\n-\t`--config ` - build configuration, applicable only for multi-config generators such as Visual Studio generator.\n\n\n## Install and Pack oneTBB\n\n---\n**NOTE**\n\nBe careful about installing prefix. It defaults to `/usr/local` on UNIX* and `c:/Program Files/${PROJECT_NAME}` on Windows* OS.\nYou can define custom `CMAKE_INSTALL_PREFIX` during configuration:\n\n```\ncmake -DCMAKE_INSTALL_PREFIX=/my/install/prefix ..\n```\n\n---\n\nInstallation can also be done using:\n\n```\ncmake --install \n```\n\nSpecial ``--install`` target can alternatively be used for installation, e.g. ``make install``.\n\nYou can use the ``install`` components for partial installation.\n\nThe following install components are supported:\n- `runtime` - oneTBB runtime package (core shared libraries and `.dll` files on Windows* OS).\n- `devel` - oneTBB development package (header files, CMake integration files, library symbolic links, and `.lib` files on Windows* OS).\n- `tbb4py` - [oneTBB Module for Python](https://github.com/oneapi-src/oneTBB/blob/master/python/README.md).\n\nIf you want to install specific components after configuration and build, run:\n\n```bash\ncmake -DCOMPONENT= [-DBUILD_TYPE=] -P cmake_install.cmake\n```\n\nSimple packaging using CPack is supported.\nThe following commands allow you to create a simple portable package that includes header files, libraries, and integration files for CMake:\n\n```bash\ncmake ..\ncpack\n```\n\n## Installation from vcpkg\n\nYou can download and install oneTBB using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager:\n```sh\n git clone https://github.com/Microsoft/vcpkg.git\n cd vcpkg\n ./bootstrap-vcpkg.sh #.\\bootstrap-vcpkg.bat(for Windows)\n ./vcpkg integrate install\n ./vcpkg install tbb\n```\n\nThe oneTBB port in vcpkg is kept up to date by Microsoft* team members and community contributors. If the version is out of date, create an issue or pull request on the [vcpkg repository](https://github.com/Microsoft/vcpkg).\n\n## Example of Installation\n\n### Single-configuration generators\n\nThe following example demonstrates how to install oneTBB for single-configuration generators (e.g. GNU Make, Ninja, etc.).\n```bash\n# Do our experiments in /tmp\ncd /tmp\n# Clone oneTBB repository\ngit clone https://github.com/oneapi-src/oneTBB.git\ncd oneTBB\n# Create binary directory for out-of-source build\nmkdir build && cd build\n# Configure: customize CMAKE_INSTALL_PREFIX and disable TBB_TEST to avoid tests build\ncmake -DCMAKE_INSTALL_PREFIX=/tmp/my_installed_onetbb -DTBB_TEST=OFF ..\n# Build\ncmake --build .\n# Install\ncmake --install .\n# Well done! Your installed oneTBB is in /tmp/my_installed_onetbb\n```\n\n### Multi-configuration generators\n\nThe following example demonstrates how to install oneTBB for multi-configuration generators such as Visual Studio*. \n\nChoose the configuration during the build and install steps:\n```batch\nREM Do our experiments in %TMP%\ncd %TMP%\nREM Clone oneTBB repository\ngit clone https://github.com/oneapi-src/oneTBB.git\ncd oneTBB\nREM Create binary directory for out-of-source build\nmkdir build && cd build\nREM Configure: customize CMAKE_INSTALL_PREFIX and disable TBB_TEST to avoid tests build\ncmake -DCMAKE_INSTALL_PREFIX=%TMP%\\my_installed_onetbb -DTBB_TEST=OFF ..\nREM Build \"release with debug information\" configuration \ncmake --build . --config relwithdebinfo\nREM Install \"release with debug information\" configuration \ncmake --install . --config relwithdebinfo\nREM Well done! Your installed oneTBB is in %TMP%\\my_installed_onetbb\n```\n" }, { "path": "src/tbb/LICENSE.txt", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "src/tbb/README.md", "content": "# oneAPI Threading Building Blocks (oneTBB) \n[![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE.txt) [![oneTBB CI](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml/badge.svg)](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml?query=branch%3Amaster)\n[![Join the community on GitHub Discussions](https://badgen.net/badge/join%20the%20discussion/on%20github/blue?icon=github)](https://github.com/oneapi-src/oneTBB/discussions)\n[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9125/badge)](https://www.bestpractices.dev/projects/9125)\n[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/oneapi-src/oneTBB/badge)](https://securityscorecards.dev/viewer/?uri=github.com/oneapi-src/oneTBB)\n\noneTBB is a flexible C++ library that simplifies the work of adding parallelism\nto complex applications, even if you are not a threading expert. \n\nThe library lets you easily write parallel programs that take full advantage of the multi-core performance. Such programs are portable, \ncomposable and have a future-proof scalability. oneTBB provides you with functions, interfaces, and classes to parallelize and scale the code.\nAll you have to do is to use the templates. \n\nThe library differs from typical threading packages in the following ways:\n* oneTBB enables you to specify logical parallelism instead of threads.\n* oneTBB targets threading for performance.\n* oneTBB is compatible with other threading packages.\n* oneTBB emphasizes scalable, data parallel programming.\n* oneTBB relies on generic programming.\n\n\nRefer to oneTBB [examples](examples) and [samples](https://github.com/oneapi-src/oneAPI-samples/tree/master/Libraries/oneTBB) to see how you can use the library.\n\noneTBB is a part of the [UXL Foundation](http://www.uxlfoundation.org) and is an implementation of [oneAPI specification](https://oneapi.io).\n\n> **_NOTE:_** Threading Building Blocks (TBB) is now called oneAPI Threading Building Blocks (oneTBB) to highlight that the tool is a part of the oneAPI ecosystem.\n\n## Release Information\n\nSee [Release Notes](RELEASE_NOTES.md) and [System Requirements](SYSTEM_REQUIREMENTS.md).\n\n## Documentation\n* [oneTBB Specification](https://spec.oneapi.com/versions/latest/elements/oneTBB/source/nested-index.html)\n* [oneTBB Developer Guide and Reference](https://oneapi-src.github.io/oneTBB)\n* [Migrating from TBB to oneTBB](https://oneapi-src.github.io/oneTBB/main/tbb_userguide/Migration_Guide.html)\n* [README for the CMake build system](cmake/README.md)\n* [oneTBB Testing Approach](https://oneapi-src.github.io/oneTBB/main/intro/testing_approach.html)\n* [Basic support for the Bazel build system](Bazel.md)\n* [oneTBB Discussions](https://github.com/oneapi-src/oneTBB/discussions)\n* [WASM Support](WASM_Support.md)\n\n## Installation \nSee [Installation from Sources](INSTALL.md) to learn how to install oneTBB. \n\n## Governance\n\nThe oneTBB project is governed by the UXL Foundation.\nYou can get involved in this project in following ways:\n* Join the [Open Source and Specification Working Group](https://github.com/uxlfoundation/foundation/tree/main?tab=readme-ov-file#working-groups) meetings.\n* Join the mailing lists for the [UXL Foundation](https://lists.uxlfoundation.org/g/main/subgroups) to receive meetings schedule and latest updates.\n* Contribute to oneTBB project or oneTBB specification. Read [CONTRIBUTING](./CONTRIBUTING.md) for more information.\n\n## Support\nSee our [documentation](./SUPPORT.md) to learn how to request help.\n\n## How to Contribute\nWe welcome community contributions, so check our [Contributing Guidelines](CONTRIBUTING.md)\nto learn more.\n\nUse GitHub Issues for feature requests, bug reports, and minor inquiries. For broader questions and development-related discussions, use GitHub Discussions.\n\n## License\noneAPI Threading Building Blocks is licensed under [Apache License, Version 2.0](LICENSE.txt).\nBy its terms, contributions submitted to the project are also done under that license.\n\n## Engineering team contacts\n* [Email us.](mailto:inteltbbdevelopers@intel.com)\n\n------------------------------------------------------------------------\n\\* All names and brands may be claimed as the property of others.\n" }, { "path": "src/tbb/RELEASE_NOTES.md", "content": "\n\n# Release Notes \nThis document contains changes of oneTBB compared to the last release.\n\n## Table of Contents \n- [Known Limitations](#known-limitations)\n- [Fixed Issues](#fixed-issues)\n\n## :rotating_light: Known Limitations\n- The ``oneapi::tbb::info`` namespace interfaces might unexpectedly change the process affinity mask on Windows* OS systems (see https://github.com/open-mpi/hwloc/issues/366 for details) when using hwloc version lower than 2.5.\n- Using a hwloc version other than 1.11, 2.0, or 2.5 may cause an undefined behavior on Windows OS. See https://github.com/open-mpi/hwloc/issues/477 for details.\n- The NUMA topology may be detected incorrectly on Windows* OS machines where the number of NUMA node threads exceeds the size of 1 processor group.\n- On Windows OS on ARM64*, when compiling an application using oneTBB with the Microsoft* Compiler, the compiler issues a warning C4324 that a structure was padded due to the alignment specifier. Consider suppressing the warning by specifying /wd4324 to the compiler command line.\n- C++ exception handling mechanism on Windows* OS on ARM64* might corrupt memory if an exception is thrown from any oneTBB parallel algorithm (see Windows* OS on ARM64* compiler issue: https://developercommunity.visualstudio.com/t/ARM64-incorrect-stack-unwinding-for-alig/1544293.\n- When CPU resource coordination is enabled, tasks from a lower-priority ``task_arena`` might be executed before tasks from a higher-priority ``task_arena``.\n\n> **_NOTE:_** To see known limitations that impact all versions of oneTBB, refer to [oneTBB Documentation](https://oneapi-src.github.io/oneTBB/main/intro/limitations.html).\n\n\n## :hammer: Fixed Issues\n- Fixed ``parallel_for_each`` algorithm behavior for iterators defining ``iterator_concept`` trait instead of ``iterator_category``.\n- Fixed the redefinition issue for ``std::min`` and ``std::max`` on Windows* OS ([GitHub* #832](https://github.com/oneapi-src/oneTBB/issues/832)).\n- Fixed the incorrect binary search order in ``TBBConfig.cmake``.\n- Enabled the oneTBB library search using the pkg-config tool in Conda packages.\n\n## :octocat: Open-source Contributions Integrated\n- Fixed the compiler warning for missing virtual destructor. Contributed by Elias Engelbert Plank (https://github.com/oneapi-src/oneTBB/pull/1215).\n" }, { "path": "src/tbb/SECURITY.md", "content": "# Security Policy\nAs an open-source project, we understand the importance of and responsibility\nfor security. This Security Policy outlines our guidelines and procedures to\nensure the highest level of security and trust for oneTBB users. \n\n## Supported Versions\nSecurity vulnerabilities are fixed in the [latest version][1]\nand delivered as a patch release. We don't guarantee security fixes to be\nback-ported to older oneTBB versions.\n\n## Report a Vulnerability\nWe are very grateful to the security researchers and users that report back\nsecurity vulnerabilities. We investigate every report thoroughly.\nWe strongly encourage you to report security vulnerabilities to us privately,\nbefore disclosing them on public forums or opening a public GitHub* issue. \n\nReport a vulnerability to us in one of two ways:\n* Open a draft **[GitHub* Security Advisory][2]**\n* Send an e-mail to: **security@uxlfoundation.org**.\nAlong with the report, provide the following info:\n * A descriptive title.\n * Your name and affiliation (if any).\n * A description of the technical details of the vulnerabilities.\n * A minimal example of the vulnerability so we can reproduce your findings.\n * An explanation of who can exploit this vulnerability, and what they gain\n doing so. \n * Whether this vulnerability is public or known to third parties. If it is,\n provide details.\n\n### When Should I Report a Vulnerability?\n* You think you discovered a potential security vulnerability in oneTBB.\n* You are unsure how the potential vulnerability affects oneTBB.\n* You think you discovered a vulnerability in another project or 3rd party\ncomponent on which oneTBB depends. If the issue is not fixed in the 3rd party\ncomponent, try to report directly there first.\n\n### When Should I NOT Report a Vulnerability?\n* You got an automated scan hit and are unable to provide details.\n* You need help using oneTBB for security.\n* You need help applying security-related updates.\n* Your issue is not security-related.\n\n## Security Reports Review Process\nWe aim to respond quickly to your inquiry and coordinate a fix and\ndisclosure with you. All confirmed security vulnerabilities will be addressed\naccording to severity level and impact on oneTBB. Normally, security issues\nare fixed in the next planned release.\n\n## Disclosure Policy\nWe will publish security advisories using the \n[**GitHub Security Advisories feature**][3]\nto keep our community well-informed, and will credit you for your findings\nunless you prefer to stay anonymous. We request that you refrain from\nexploiting the vulnerability or making it public before the official disclosure.\n\nWe will disclose the vulnerabilities and bugs as soon as possible once\nmitigation is implemented and available. \n\n## Feedback on This Policy\nIf you have any suggestions on how this Policy could be improved, submit\nan issue or a pull request to this repository. **Do not** report\npotential vulnerabilities or security flaws via a pull request.\n\n[1]: https://github.com/oneapi-src/oneTBB/releases/latest\n[2]: https://github.com/oneapi-src/oneTBB/security/advisories/new\n[3]: https://github.com/oneapi-src/oneTBB/security/advisories\n" }, { "path": "src/tbb/SUPPORT.md", "content": "\n\n# oneTBB Support\n\nWe are committed to providing support and assistance to help you make the most out of oneTBB. \nUse the following methods if you face any challenges. \n\n## Issues\n\nIf you have a problem, check out the [GitHub Issues](https://github.com/oneapi-src/oneTBB/issues) to see if the issue you want to address is already reported. \nYou may find users that have encountered the same bug or have similar ideas for changes or updates.\n\nYou can use issues to report a problem, make a feature request, or add comments on an existing issue.\n\n## Discussions \n\nVisit the [GitHub Discussions](https://github.com/oneapi-src/oneTBB/discussions) to engage with the community, ask questions, or help others. \n\n## Email\n\nReach out to us privately via [email](mailto:inteltbbdevelopers@intel.com). " }, { "path": "src/tbb/SYSTEM_REQUIREMENTS.md", "content": "\n\n# System Requirements \nThis document provides details about hardware, operating system, and software prerequisites for the oneAPI Threading Building Blocks (oneTBB). \n\n## Table of Contents \n- [Supported Hardware](#supported-hardware)\n- [Software](#software)\n - [Supported Operating Systems](#supported-operating-systems)\n - [Community-Supported Platforms](#community-supported-platforms)\n - [Supported Compilers](#supported-compilers)\n- [Limitations](#limitations)\n\n\n## Supported Hardware\n- Intel(R) Celeron(R) processor family\n- Intel(R) Core* processor family\n- Intel(R) Xeon(R) processor family\n- Intel(R) Atom* processor family\n- Non-Intel(R) processors compatible with the processors listed above\n\n\n## Software\n\n### Supported Operating Systems\n- Systems with Microsoft* Windows* operating systems:\n - Microsoft* Windows* 10\n - Microsoft* Windows* 11\n - Microsoft* Windows* Server 2019\n - Microsoft* Windows* Server 2022\n- Systems with Linux* operating systems:\n - Oracle Linux* 8\n - Amazon* Linux 2, 2022\n - Debian* 9, 10, 11\n - Fedora* 36, 37, 38\n - Rocky* Linux* 8, 9\n - Red Hat* Enterprise Linux* 8, 9\n - SuSE* Linux* Enterprise Server 15\n - Ubuntu* 20.04, 22.04\n- Systems with macOS* operating systems:\n - macOS* 12.x, 13.x\n- Systems with Android* operating systems:\n - Android* 9\n\n### Community-Supported Platforms\n- MinGW*\n- FreeBSD*\n- Microsoft* Windows* on ARM*/ARM64*\n- macOS* on ARM64*\n\n### Supported Compilers\n- Intel* oneAPI DPC++/C++ Compiler\n- Intel® C++ Compiler Classic 2021.1 - 2021.9\n- Microsoft* Visual C++ 14.2 (Microsoft* Visual Studio* 2019, Windows* OS only)\n- Microsoft* Visual C++ 14.3 (Microsoft* Visual Studio* 2022, Windows* OS only)\n- For each supported Linux* operating system, the standard gcc version provided with that operating system is supported:\n - GNU Compilers (gcc) 8.x – 12.x\n - GNU C Library (glibc) version 2.28 – 2.36\n - Clang* 6.0.0 - 13.0.0\n\n## Limitations\nThere are some cases where we cannot provide support for your platforms. It includes: \n\n1. The platform is out of official support (met end of life). When you use an unsupported platform, you can face a security risk that can be difficult to resolve.\n2. We do not have the infrastructure to test a platform. Therefore we cannot guarantee that oneTBB works correctly on that platform. \n3. Changes affect more code than just platform-specific macros.\n4. The platform is incompatible with oneTBB. Some platforms may have limitations that prevent oneTBB from working correctly. We cannot provide support in these cases as the issue is beyond our control.\n5. The platform is modified or customized. If you made significant updates to your platform, it might be hard for us to find the root cause of the issue. Therefore, we may not be able to provide support as the modification could affect the oneTBB functionality. \n\n\nWe understand that these limitations can be frustrating. Thus, we suggest creating a branch specifically for the unsupported platform, allowing other users to contribute to or use your implementation.\n\n" }, { "path": "src/tbb/WASM_Support.md", "content": "\n\n# WASM Support\n\noneTBB extends its capabilities by offering robust support for ``WASM`` (see ``Limitation`` sections).\n\n``WASM`` stands for WebAssembly, a low-level binary format for executing code in web browsers. \nIt is designed to be a portable target for compilers and efficient to parse and execute. \n\nUsing oneTBB with WASM, you can take full advantage of parallelism and concurrency while working on web-based applications, interactive websites, and a variety of other WASM-compatible platforms.\n\noneTBB offers WASM support through the integration with [Emscripten*](https://emscripten.org/docs/introducing_emscripten/index.html), a powerful toolchain for compiling C and C++ code into WASM-compatible runtimes. \n\n## Build\n\n**Prerequisites:** Download and install Emscripten*. See the [instructions](https://emscripten.org/docs/getting_started/downloads.html). \n\nTo build the system, run:\n\n```\nmkdir build && cd build\nemcmake cmake .. -DCMAKE_CXX_COMPILER=em++ -DCMAKE_C_COMPILER=emcc -DTBB_STRICT=OFF -DCMAKE_CXX_FLAGS=-Wno-unused-command-line-argument -DTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH=ON -DBUILD_SHARED_LIBS=ON -DTBB_EXAMPLES=ON -DTBB_TEST=ON\n```\nTo compile oneTBB without ``pthreads``, set the flag ``-DEMSCRIPTEN_WITHOUT_PTHREAD=true`` in the command above. By default, oneTBB uses the ``pthreads``.\n```\ncmake --build . \ncmake --install . \n```\nWhere:\n\n* ``emcmake`` - a tool that sets up the environment for Emscripten*. \n* ``-DCMAKE_CXX_COMPILER=em++`` - specifies the C++ compiler as Emscripten* C++ compiler. \n* ``-DCMAKE_C_COMPILER=emcc`` - specifies the C compiler as Emscripten* C compiler.\n\n\n> **_NOTE:_** See [CMake documentation](https://github.com/oneapi-src/oneTBB/blob/master/cmake/README.md) to learn about other options. \n\n\n## Run Test\n\nTo run tests, use:\n\n```\nctest\n```\n\n# Limitations\n\nYou can successfully build your application with oneTBB using WASM, but you may not achieve optimal performance immediately. This is due to the limitation with nested Web Workers: a Web Worker cannot schedule another worker without help from a browser thread. This can lead to unexpected performance outcomes, such as the application running in serial.\nFind more information in the [issue](https://github.com/emscripten-core/emscripten/discussions/21963) in the Emscripten repository.\nTo workaround this issue, try one of the following ways:\n1. **Recommended Solution: Use the ``-sPROXY_TO_PTHREAD`` Flag**. \nThis flag splits the initial thread into a browser thread and a main thread (proxied by a Web Worker), effectively resolving the issue as the browser thread is always present in the event loop and can participate in Web Workers scheduling. Refer to the [Emscripten documentation](https://emscripten.org/docs/porting/pthreads.html) for more details about ``-sPROXY_TO_PTHREAD`` since using this flag may require refactoring the code.\n2. **Alternative Solution: Warm Up the oneTBB Thread Pool**\nInitialize the oneTBB thread pool before making the first call to oneTBB. This approach forces the browser thread to participate in Web Workers scheduling.\n```cpp\n int num_threads = tbb::this_task_arena::max_concurrency();\n std::atomic barrier{num_threads};\n tbb::parallel_for(0, num_threads, [&barrier] (int) {\n barrier--;\n while (barrier > 0) {\n // Send browser thread to event loop\n std::this_thread::yield();\n }\n }, tbb::static_partitioner{});\n```\n> **_NOTE:_** Be aware that it might cause delays on the browser side.\n" }, { "path": "src/tbb/cmake/README.md", "content": "# Build System Description\n\nThe project uses CMake* build configuration.\n\nThe following controls are available during the configure stage:\n```\nTBB_TEST:BOOL - Enable testing (ON by default)\nTBB_STRICT:BOOL - Treat compiler warnings as errors (ON by default)\nTBB_SANITIZE:STRING - Sanitizer parameter, passed to compiler/linker\nTBB_SIGNTOOL:FILEPATH - Tool for digital signing, used in post-install step for libraries if provided.\nTBB_SIGNTOOL_ARGS:STRING - Additional arguments for TBB_SIGNTOOL, used if TBB_SIGNTOOL is set.\nTBB_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) build (ON by default)\nTBB_FIND_PACKAGE - Enable search for external oneTBB using find_package instead of build from sources (OFF by default)\nTBBMALLOC_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) memory allocator build (ON by default)\nTBBMALLOC_PROXY_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) memory allocator proxy build (requires TBBMALLOC_BUILD. ON by default)\nTBB4PY_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) Python module build (OFF by default)\nTBB_CPF:BOOL - Enable preview features of the library (OFF by default)\nTBB_INSTALL:BOOL - Enable installation (ON by default)\nTBB_INSTALL_VARS:BOOL - Enable auto-generated vars installation(packages generated by `cpack` and `make install` will also include the vars script)(OFF by default)\nTBB_VALGRIND_MEMCHECK:BOOL - Enable scan for memory leaks using Valgrind (OFF by default)\nTBB_DISABLE_HWLOC_AUTOMATIC_SEARCH - Disable HWLOC automatic search by pkg-config tool (OFF by default)\nTBB_ENABLE_IPO - Enable Interprocedural Optimization (IPO) during the compilation (ON by default)\nTBB_BUILD_APPLE_FRAMEWORKS - Enable the Apple* frameworks instead of dylibs, only available on the Apple platform. (OFF by default)\n```\n\n## Configure, Build, and Test\n\n### Preparation\n\nTo perform an out-of-source build, create a build directory and go there:\n\n```bash\nmkdir /tmp/my-build\ncd /tmp/my-build\n```\n\n### Configure\n\n```bash\ncmake \n```\n\nSome useful options:\n- `-G ` - specify particular project generator. See `cmake --help` for details.\n- `-DCMAKE_BUILD_TYPE=Debug` - specify for Debug build. It is not applicable for multi-config generators, e.g., Microsoft* Visual Studio* generator.\n\n#### TBBBind Library Configuration\n\n> **_TIP:_** It is recommended to install the HWLOC* library. See [oneTBB documentation](https://oneapi-src.github.io/oneTBB/GSG/next_steps.html#hybrid-cpu-and-numa-support) for details.\n\nThe TBBbind library has three versions: `tbbbind`, `tbbbind_2_0`, and `tbbbind_2_5`. Each of these versions is linked with the corresponding HWLOC* library version: \n- `tbbbind` links with `HWLOC 1.11.x`\n- `tbbbind_2_0` links with `HWLOC 2.1–2.4`\n- `tbbbind_2_5` links with `HWLOC 2.5` and later\n\nThe search for a suitable version of the HWLOC library is enabled by default. If you want to use a specific version of the library, you can specify the path to it manually using the following CMake variables:\n\n - `CMAKE_HWLOC__LIBRARY_PATH` - path to the corresponding HWLOC version shared library on Linux* OS or path to `.lib` file on Windows* OS\n - `CMAKE_HWLOC__INCLUDE_PATH` - path to the corresponding HWLOC version including directory\n\n\n---\n**NOTE:** Automatic HWLOC searching requires CMake version 3.6 or higher.\n\n---\n\n\nWindows* OS requires an additional variable for correct TBBBind library building:\n - `CMAKE_HWLOC__DLL_PATH` - path to the corresponding HWLOC version `.dll` file.\n\nThe `HWLOC_VER` substring used earlier can be replaced with one of the three values:\n- `1_11` for the `tbbbind` library configuration\n- `2` for the `tbbbind_2_0` library configuration\n- `2_5` for the `tbbbind_2_5` library configuration\n\nIf you specify variables for several TBBBind versions, the building process for all of these versions is performed during a single build session.\n\n---\n**TIP**\n\nSpecify the `TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH` to turn off the HWLOC library's automatic search.\n\n---\n\n\n### Build\n\n```bash\ncmake --build . \n```\n\nSome useful options:\n- `--target ` - specific target, \"all\" is the default.\n- `--config ` - build configuration, applicable only for multi-config generators, e.g., Visual Studio* generator.\n\nThe binaries are placed to `./__cxx_`. For example, `./gnu_4.8_cxx11_release`.\n\n#### Build For 32-bit\n\n* **Intel(R) Compiler**. Source Intel(R) C++ Compiler with `ia32` and build as usual.\n* **MSVC**. Use switch for [generator](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html) (e.g., `-A Win32` for [VS2019](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2016%202019.html)) during the configuration stage and then build as usual.\n* **GCC/Clang**. Specify `-m32` during the configuration. It can be `CXXFLAGS=-m32 cmake ..` or `cmake -DCMAKE_CXX_FLAGS=-m32 ..`\n* For any other compiler, which builds for 64-bit by default, specify a 32-bit compiler key during the configuration as above.\n\n#### Windows* OS-Specific Builds\n\n---\n**NOTE**\n\nThe following builds require CMake version 3.15 or higher. \n\n---\n\n* **Dynamic linkage with C Runtime Library (CRT)**. The default behavior can be explicitly specified by setting `CMAKE_MSVC_RUNTIME_LIBRARY` to `MultiThreadedDLL` or `MultiThreadedDebugDLL`.\n```bash\ncmake .. # dynamic linkage is used by default\n```\n```bash\ncmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL ..\n```\n```bash\ncmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL -DCMAKE_BUILD_TYPE=Debug ..\n```\n* **Static linkage with CRT**. Set `CMAKE_MSVC_RUNTIME_LIBRARY` to `MultiThreaded` or `MultiThreadedDebug`.\n```bash\ncmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ..\n```\n```bash\ncmake -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug -DCMAKE_BUILD_TYPE=Debug ..\n```\n* **Windows OS 10 Universal Windows application build**. Set `CMAKE_SYSTEM_NAME` to `WindowsStore` and `CMAKE_SYSTEM_VERSION` to `10.0`.\n\n---\n**NOTE**\n\nSet `TBB_NO_APPCONTAINER` to `ON` to apply the `/APPCONTAINER:NO` option during the compilation (used for testing).\n\n---\n\n```bash\ncmake -DCMAKE_SYSTEM_NAME:STRING=WindowsStore -DCMAKE_SYSTEM_VERSION:STRING=10.0 ..\n```\n\n* **Universal Windows OS Driver build**. Set `TBB_WINDOWS_DRIVER` to `ON` and use static linkage with CRT.\n\n```bash\ncmake -DTBB_WINDOWS_DRIVER=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ..\n```\n\n#### Example\n\n```bash\ncmake -DCMAKE_CXX_COMPILER=icpc -DCMAKE_C_COMPILER=icc -DTBB_TEST=off -DCMAKE_HWLOC_1_11_LIBRARY_PATH=/libhwloc.so.15\n-DCMAKE_HWLOC_1_11_INCLUDE_PATH= -DCMAKE_INSTALL_PREFIX=/oneTBB_install ..\nmake -j8 && make install\n```\n\n---\n**NOTE**\n\nThe library path points to a file, while the include path points to a directory and not to ``hwloc.h``.\n\n---\n\n### Test\n\n#### Build test\nTo build a test, use the default target ``all``:\n```\ncmake --build .\n```\n\nOr use a specific test target:\n```\ncmake --build . --target # e.g. test_version\n```\n\n#### Run Test\n\nYou can run a test by using CTest:\n```bash\nctest\n```\n\nOr by using the ``test`` target:\n```bash\ncmake --build . --target test # currently does not work on Windows* OS\n```\n\n## Installation\nSee [Installation from Sources](../INSTALL.md) to learn how to install oneTBB.\n\nTo install oneTBB from the release packages, use the following commands: \n```bash\ntar -xvf oneapi-tbb-xxx.xx.x-*.tgz\nsource env/vars.sh\n```\n\n\n## Sanitizers - Configure, Build, and Run\n\n```bash\nmkdir build\ncd build\ncmake -DTBB_SANITIZE=thread .. # or -DTBB_SANITIZE=memory or any other sanitizer\nmake -j\nctest -V\n```\n\n## Valgrind Memcheck - Configure, Build, and Run\n\n### Prerequisites\n* Valgrind tool executable\n### Example\n```bash\nmkdir build\ncd build\ncmake -DTBB_VALGRIND_MEMCHECK=ON ..\nmake -j memcheck- # or memcheck-all to scan all tests \n```\n\n## Test Specification\n\nUse Doxygen* to generate oneTBB test specification:\n\n```bash\nmkdir build\ncd build\ncmake -DTBB_TEST_SPEC=ON ..\nmake test_spec\n```\n\n## TBBConfig - Integration of Binary Packages\n\nIt is a configuration module that is used for the integration of prebuilt oneTBB. It consists of two files (``TBBConfig.cmake`` and ``TBBConfigVersion.cmake``) and can be used via the [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) function.\n\nTo use this module in your CMake project:\n 1. Let CMake know where to search for TBBConfig, e.g. specify the location of ``TBBConfig.cmake`` in `TBB_DIR` (for more details about search paths, see [find_package](https://cmake.org/cmake/help/latest/command/find_package.html)).\n 2. Use [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) to find oneTBB.\n 3. Use provided variables and/or imported targets (described below) to work with the found oneTBB.\n\nExample:\n\n```cmake\nadd_executable(foo foo.cpp)\nfind_package(TBB)\ntarget_link_libraries(foo TBB::tbb)\n```\n\noneTBB components can be passed to [find_package](https://cmake.org/cmake/help/latest/command/find_package.html) after keyword ``COMPONENTS`` or ``REQUIRED``.\nUse basic names of components (`tbb`, `tbbmalloc`, etc.).\n\nIf components are not specified, then the default set is used: `tbb`, `tbbmalloc`, and ``tbbmalloc_proxy``.\n\nIf `tbbmalloc_proxy` is requested, the `tbbmalloc` component is also added and set as a dependency for `tbbmalloc_proxy`.\n\nTBBConfig creates [imported targets](https://cmake.org/cmake/help/latest/manual/cmake-buildsystem.7.html#imported-targets>) as\nshared libraries using the following format: `TBB::`. For example, `TBB::tbb` or `TBB::tbbmalloc`.\n\nTo search only for release oneTBB version, set `TBB_FIND_RELEASE_ONLY` to `TRUE` before calling `find_package`. This variable helps to avoid simultaneous linkage of release and debug oneTBB versions when CMake configuration is `Debug,` but a third-party component depends on the release oneTBB version.\n\nVariables set during TBB configuration:\n\nVariable | Description\n--- | ---\n`TBB_FOUND` | oneTBB is found\n`TBB__FOUND` | Specific oneTBB component is found\n`TBB_VERSION` | oneTBB version (format: `...`)\n`TBB_IMPORTED_TARGETS` | All created oneTBB imported targets (not supported for builds from source code)\n\nStarting from [oneTBB 2021.1](https://github.com/oneapi-src/oneTBB/releases/tag/v2021.1), GitHub* release TBBConfig files in the binary packages are located under `/lib/cmake/TBB`.\nFor example, `TBB_DIR` should be set to `/lib/cmake/TBB`.\n\nTBBConfig files are automatically created during the build from source code and can be installed together with the library.\nAlso, oneTBB provides a helper function that creates TBBConfig files from predefined templates. See `tbb_generate_config` in `cmake/config_generation.cmake`.\n\n## oneTBB Python Module Support\nThe `TBB4PY_BUILD` Cmake option provides the ability to build a Python module for oneTBB.\n\n### Targets:\n - `irml` - build IPC RML server\n - `python_build` - build oneTBB module for Python\n\n`python_build` target requirements:\n - Python version 3.5 or newer\n - SWIG version 3.0.6 or newer\n\n## CMake Files\n\n### Compile and Link Options\n\nCompile and link options may be specific for certain compilers. This part is handled in `cmake/compilers/*` files.\n\nOptions in TBB CMake are handled via variables in two ways for convenience:\n\n* by options group\n* by the specific option\n\n#### Options Group\n\nNaming convention is the following: `TBB___`, where:\n\n* `` can be:\n * `LIB` - options applied during libraries build.\n * `TEST` - options applied during test build.\n * `BENCH` - options applied during benchmarks build.\n * `COMMON` - options applied during all (libraries, test, benchmarks) builds.\n* `` can be:\n * `COMPILE` - options applied during the compilation.\n * `LINK` - options applied during the linkage.\n* `` can be:\n * `FLAGS` - list of flags\n * `LIBS` - list of libraries\n\n*Examples*\n\nVariable | Description\n--- | ---\n`TBB_COMMON_COMPILE_FLAGS` | Applied to libraries, tests, and benchmarks as compile options\n`TBB_LIB_LINK_FLAGS` | Applied to libraries as link options\n`TBB_LIB_LINK_LIBS ` | Applied to libraries as link libraries\n`TBB_TEST_COMPILE_FLAGS` | Applied to tests as compile options\n\nSpecify the `LINK` options prefixed with a dash(-) for MSVC(Visual Studio) compiler with CMake < 3.13 to avoid issues caused by `target_link_libraries` CMake command usage.\n\n#### Specific Options\n\nIf the option is used only in part of the places (library, tests, benchmarks) and adding this option to the group of other options is not possible,\nthen the option must be named using common sense.\n\nWarning suppressions should be added to the `TBB_WARNING_SUPPRESS` variable, which is applied during the compilation of libraries, tests, and benchmarks.\nAdditional warnings should be added to the `TBB_WARNING_TEST_FLAGS` variable, which is applied during the compilation of tests.\n" }, { "path": "src/tbb/cmake/android/device_environment_cleanup.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ninclude(${CMAKE_CURRENT_LIST_DIR}/environment.cmake)\n\nexecute_on_device(\"rm -rf ${ANDROID_DEVICE_TESTING_DIRECTORY}\")\n" }, { "path": "src/tbb/cmake/android/environment.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(ANDROID_DEVICE_TESTING_DIRECTORY \"/data/local/tmp/tbb_testing\")\n\nfind_program(adb_executable adb)\nif (NOT adb_executable)\n message(FATAL_ERROR \"Could not find adb\")\nendif()\n\nmacro(execute_on_device cmd)\n execute_process(COMMAND ${adb_executable} shell ${cmd} RESULT_VARIABLE CMD_RESULT)\n if (CMD_RESULT)\n message(FATAL_ERROR \"Error while on device execution: ${cmd} error_code: ${CMD_RESULT}\")\n endif()\nendmacro()\n\nmacro(transfer_data data_path)\n execute_process(COMMAND ${adb_executable} push --sync ${data_path} ${ANDROID_DEVICE_TESTING_DIRECTORY}\n RESULT_VARIABLE CMD_RESULT OUTPUT_QUIET)\n if (CMD_RESULT)\n message(FATAL_ERROR \"Error while data transferring: ${data_path} error_code: ${CMD_RESULT}\")\n endif()\nendmacro()\n" }, { "path": "src/tbb/cmake/android/test_launcher.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ninclude(${CMAKE_CURRENT_LIST_DIR}/environment.cmake)\n\n# transfer data to device\nexecute_on_device(\"mkdir -m 755 -p ${ANDROID_DEVICE_TESTING_DIRECTORY}\")\n\nfile (GLOB_RECURSE BINARIES_LIST \"${BINARIES_PATH}/*.so*\" \"${BINARIES_PATH}/${TEST_NAME}\")\nforeach(BINARY_FILE ${BINARIES_LIST})\n transfer_data(${BINARY_FILE})\nendforeach()\n\n# execute binary\nexecute_on_device(\"chmod -R 755 ${ANDROID_DEVICE_TESTING_DIRECTORY}\")\nexecute_on_device(\"LD_LIBRARY_PATH=${ANDROID_DEVICE_TESTING_DIRECTORY} ${ANDROID_DEVICE_TESTING_DIRECTORY}/${TEST_NAME}\")\n" }, { "path": "src/tbb/cmake/compilers/AppleClang.cmake", "content": "# Copyright (c) 2020-2022 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,)\nset(TBB_DEF_FILE_PREFIX mac${TBB_ARCH})\nset(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror>)\nset(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor)\nset(TBB_WARNING_SUPPRESS -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else)\n# For correct ucontext.h structures layout\nset(TBB_COMMON_COMPILE_FLAGS -D_XOPEN_SOURCE)\n\n# Depfile options (e.g. -MD) are inserted automatically in some cases.\n# Don't add -MMD to avoid conflicts in such cases.\nif (NOT CMAKE_GENERATOR MATCHES \"Ninja\" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER)\n set(TBB_MMD_FLAG -MMD)\nendif()\n\n# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled.\nif (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)\n tbb_remove_compile_flag(-Werror)\nendif()\n\n# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors\nif (CMAKE_OSX_ARCHITECTURES)\n set(_tbb_target_architectures \"${CMAKE_OSX_ARCHITECTURES}\")\nelse()\n set(_tbb_target_architectures \"${CMAKE_SYSTEM_PROCESSOR}\")\nendif()\nif (\"${_tbb_target_architectures}\" MATCHES \"(x86_64|amd64|AMD64)\") # OSX systems are 64-bit only\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>:-mwaitpkg>)\nendif()\nunset(_tbb_target_architectures)\n\n# TBB malloc settings\nset(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions)\n\n" }, { "path": "src/tbb/cmake/compilers/Clang.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif (EMSCRIPTEN)\n set(TBB_EMSCRIPTEN 1)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fexceptions)\n set(TBB_TEST_LINK_FLAGS ${TBB_COMMON_LINK_FLAGS} -fexceptions -sINITIAL_MEMORY=65536000 -sALLOW_MEMORY_GROWTH=1 -sMALLOC=mimalloc -sEXIT_RUNTIME=1)\n if (NOT EMSCRIPTEN_WITHOUT_PTHREAD)\n set_property(TARGET Threads::Threads PROPERTY INTERFACE_LINK_LIBRARIES \"-pthread\")\n endif()\n set(TBB_EMSCRIPTEN_STACK_SIZE 65536)\n set(TBB_LIB_COMPILE_FLAGS -D__TBB_EMSCRIPTEN_STACK_SIZE=${TBB_EMSCRIPTEN_STACK_SIZE})\n set(TBB_TEST_LINK_FLAGS ${TBB_TEST_LINK_FLAGS} -sTOTAL_STACK=${TBB_EMSCRIPTEN_STACK_SIZE})\n unset(TBB_EMSCRIPTEN_STACK_SIZE)\nendif()\n\nif (MINGW)\n set(TBB_LINK_DEF_FILE_FLAG \"\")\n set(TBB_DEF_FILE_PREFIX \"\")\nelseif (APPLE)\n set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,)\n set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH})\n\n # For correct ucontext.h structures layout\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -D_XOPEN_SOURCE)\nelseif (MSVC)\n include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)\n return()\nelse()\n set(TBB_LINK_DEF_FILE_FLAG -Wl,--version-script=)\n set(TBB_DEF_FILE_PREFIX lin${TBB_ARCH})\n set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} $<$>:-ffp-model=precise>)\nendif()\n\n# Depfile options (e.g. -MD) are inserted automatically in some cases.\n# Don't add -MMD to avoid conflicts in such cases.\nif (NOT CMAKE_GENERATOR MATCHES \"Ninja\" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER)\n set(TBB_MMD_FLAG -MMD)\nendif()\n\nset(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror>)\nset(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor)\n\n# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled.\nif (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)\n tbb_remove_compile_flag(-Werror)\nendif()\n\n# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors\nif (CMAKE_SYSTEM_PROCESSOR MATCHES \"(AMD64|amd64|i.86|x86)\" AND NOT EMSCRIPTEN)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>:-mwaitpkg>)\nendif()\n\n# Clang flags to prevent compiler from optimizing out security checks\nset(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -Wformat -Wformat-security -Werror=format-security -fPIC $<$>:-fstack-protector-strong>)\n\n# -z switch is not supported on MacOS\nif (NOT APPLE)\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -Wl,-z,relro,-z,now)\nendif()\n\nset(TBB_COMMON_LINK_LIBS ${CMAKE_DL_LIBS})\n\nif (NOT CMAKE_CXX_FLAGS MATCHES \"_FORTIFY_SOURCE\")\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2>)\nendif ()\n\nif (MINGW)\n list(APPEND TBB_COMMON_COMPILE_FLAGS -U__STRICT_ANSI__)\nendif()\n\nset(TBB_IPO_COMPILE_FLAGS $<$>:-flto>)\nset(TBB_IPO_LINK_FLAGS $<$>:-flto>)\n\n# TBB malloc settings\nset(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions)\n" }, { "path": "src/tbb/cmake/compilers/GNU.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif (MINGW)\n set(TBB_LINK_DEF_FILE_FLAG \"\")\n set(TBB_DEF_FILE_PREFIX \"\")\nelseif (APPLE)\n set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,)\n set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH})\n\n # For correct ucontext.h structures layout\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -D_XOPEN_SOURCE)\nelse()\n set(TBB_LINK_DEF_FILE_FLAG -Wl,--version-script=)\n set(TBB_DEF_FILE_PREFIX lin${TBB_ARCH})\nendif()\n\nset(TBB_WARNING_LEVEL -Wall -Wextra $<$:-Werror> -Wfatal-errors)\nset(TBB_TEST_WARNING_FLAGS -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor)\n\n# Depfile options (e.g. -MD) are inserted automatically in some cases.\n# Don't add -MMD to avoid conflicts in such cases.\nif (NOT CMAKE_GENERATOR MATCHES \"Ninja\" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER)\n set(TBB_MMD_FLAG -MMD)\nendif()\n\n\n# Binutils < 2.31.1 do not support the tpause instruction. When compiling with\n# a modern version of GCC (supporting it) but relying on an outdated assembler,\n# will result in an error reporting \"no such instruction: tpause\".\n# The following code invokes the GNU assembler to extract the version number\n# and convert it to an integer that can be used in the C++ code to compare\n# against, and conditionally disable the __TBB_WAITPKG_INTRINSICS_PRESENT\n# macro if the version is incompatible. Binutils only report the version in the\n# MAJOR.MINOR format, therefore the version checked is >=2.32 (instead of\n# >=2.31.1). Capturing the output in CMake can be done like below. The version\n# information is written to either stdout or stderr. To not make any\n# assumptions, both are captured.\nexecute_process(\n COMMAND ${CMAKE_COMMAND} -E env \"LANG=C\" ${CMAKE_CXX_COMPILER} -xc -c /dev/null -Wa,-v -o/dev/null\n OUTPUT_VARIABLE ASSEMBLER_VERSION_LINE_OUT\n ERROR_VARIABLE ASSEMBLER_VERSION_LINE_ERR\n OUTPUT_STRIP_TRAILING_WHITESPACE\n ERROR_STRIP_TRAILING_WHITESPACE\n)\nset(ASSEMBLER_VERSION_LINE ${ASSEMBLER_VERSION_LINE_OUT}${ASSEMBLER_VERSION_LINE_ERR})\nstring(REGEX REPLACE \".*GNU assembler version ([0-9]+)\\\\.([0-9]+).*\" \"\\\\1\" _tbb_gnu_asm_major_version \"${ASSEMBLER_VERSION_LINE}\")\nstring(REGEX REPLACE \".*GNU assembler version ([0-9]+)\\\\.([0-9]+).*\" \"\\\\2\" _tbb_gnu_asm_minor_version \"${ASSEMBLER_VERSION_LINE}\")\nunset(ASSEMBLER_VERSION_LINE_OUT)\nunset(ASSEMBLER_VERSION_LINE_ERR)\nunset(ASSEMBLER_VERSION_LINE)\nmessage(TRACE \"Extracted GNU assembler version: major=${_tbb_gnu_asm_major_version} minor=${_tbb_gnu_asm_minor_version}\")\n\nmath(EXPR _tbb_gnu_asm_version_number \"${_tbb_gnu_asm_major_version} * 1000 + ${_tbb_gnu_asm_minor_version}\")\nset(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} \"-D__TBB_GNU_ASM_VERSION=${_tbb_gnu_asm_version_number}\")\nmessage(STATUS \"GNU Assembler version: ${_tbb_gnu_asm_major_version}.${_tbb_gnu_asm_minor_version} (${_tbb_gnu_asm_version_number})\")\n\n# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors\nif (CMAKE_SYSTEM_PROCESSOR MATCHES \"(AMD64|amd64|i.86|x86)\" AND NOT EMSCRIPTEN)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$>,$>>:-mwaitpkg>)\nendif()\n\nset(TBB_COMMON_LINK_LIBS ${CMAKE_DL_LIBS})\n\n# Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled.\nif (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)\n tbb_remove_compile_flag(-Werror)\nendif()\n\nif (NOT ${CMAKE_CXX_COMPILER_ID} STREQUAL Intel)\n # gcc 6.0 and later have -flifetime-dse option that controls elimination of stores done outside the object lifetime\n set(TBB_DSE_FLAG $<$>:-flifetime-dse=1>)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-fstack-clash-protection>)\n\n # Suppress GCC 12.x-13.x warning here that to_wait_node(n)->my_is_in_list might have size 0\n set(TBB_COMMON_LINK_FLAGS ${TBB_COMMON_LINK_FLAGS} $<$>,$>:-Wno-stringop-overflow>)\nendif()\n\n# Workaround for heavy tests and too many symbols in debug (rellocation truncated to fit: R_MIPS_CALL16)\nif (\"${CMAKE_SYSTEM_PROCESSOR}\" MATCHES \"mips\")\n set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} -DTBB_TEST_LOW_WORKLOAD $<$:-fPIE -mxgot>)\n set(TBB_TEST_LINK_FLAGS ${TBB_TEST_LINK_FLAGS} $<$:-pie>)\nendif()\n\nset(TBB_IPO_COMPILE_FLAGS $<$>:-flto>)\nset(TBB_IPO_LINK_FLAGS $<$>:-flto>)\n\n\nif (MINGW AND CMAKE_SYSTEM_PROCESSOR MATCHES \"i.86\")\n list (APPEND TBB_COMMON_COMPILE_FLAGS -msse2)\nendif ()\n\n# Gnu flags to prevent compiler from optimizing out security checks\nset(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fno-strict-overflow -fno-delete-null-pointer-checks -fwrapv)\nset(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -Wformat -Wformat-security -Werror=format-security\n -fstack-protector-strong )\n# -z switch is not supported on MacOS and MinGW\nif (NOT APPLE AND NOT MINGW)\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -Wl,-z,relro,-z,now,-z,noexecstack)\nendif()\nif (NOT CMAKE_CXX_FLAGS MATCHES \"_FORTIFY_SOURCE\")\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2> )\nendif ()\n\n# TBB malloc settings\nset(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions)\nset(TBB_OPENMP_FLAG -fopenmp)\n" }, { "path": "src/tbb/cmake/compilers/Intel.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif (MSVC)\n include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)\n set(TBB_WARNING_LEVEL ${TBB_WARNING_LEVEL} /W3)\n set(TBB_OPENMP_FLAG /Qopenmp)\n set(TBB_IPO_COMPILE_FLAGS $<$>:/Qipo>)\n set(TBB_IPO_LINK_FLAGS $<$>:/INCREMENTAL:NO>)\nelseif (APPLE)\n include(${CMAKE_CURRENT_LIST_DIR}/AppleClang.cmake)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fstack-protector -Wformat -Wformat-security\n $<$>:-fno-omit-frame-pointer -qno-opt-report-embed>)\n if (NOT CMAKE_CXX_FLAGS MATCHES \"_FORTIFY_SOURCE\")\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$>:-D_FORTIFY_SOURCE=2>)\n endif ()\n\n set(TBB_OPENMP_FLAG -qopenmp)\n set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>)\nelse()\n include(${CMAKE_CURRENT_LIST_DIR}/GNU.cmake)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$:-falign-stack=maintain-16-byte>)\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -static-intel)\n set(TBB_OPENMP_FLAG -qopenmp)\n set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>)\nendif()\nset(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS})\n" }, { "path": "src/tbb/cmake/compilers/IntelLLVM.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif (WIN32)\n include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)\n set(TBB_OPENMP_FLAG /Qopenmp)\n set(TBB_IPO_COMPILE_FLAGS $<$>:/Qipo>)\n set(TBB_IPO_LINK_FLAGS $<$>:/INCREMENTAL:NO>)\nelse()\n include(${CMAKE_CURRENT_LIST_DIR}/Clang.cmake)\n set(TBB_IPO_COMPILE_FLAGS $<$>:-ipo>)\n # \"--exclude-libs,ALL\" is used to avoid accidental exporting of symbols\n # from statically linked libraries\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -static-intel -Wl,--exclude-libs,ALL)\n set(TBB_OPENMP_FLAG -qopenmp)\nendif()\nset(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS})\n" }, { "path": "src/tbb/cmake/compilers/MSVC.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(TBB_LINK_DEF_FILE_FLAG ${CMAKE_LINK_DEF_FILE_FLAG})\nset(TBB_DEF_FILE_PREFIX win${TBB_ARCH})\n\n# Workaround for CMake issue https://gitlab.kitware.com/cmake/cmake/issues/18317.\n# TODO: consider use of CMP0092 CMake policy.\nstring(REGEX REPLACE \"/W[0-4]\" \"\" CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS}\")\n\nset(TBB_WARNING_LEVEL $<$>:/W4> $<$:/WX>)\n\n# Warning suppression C4324: structure was padded due to alignment specifier\nset(TBB_WARNING_SUPPRESS /wd4324)\n\nset(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} /bigobj)\nif (MSVC_VERSION LESS_EQUAL 1900)\n # Warning suppression C4503 for VS2015 and earlier:\n # decorated name length exceeded, name was truncated.\n # More info can be found at\n # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503\n set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} /wd4503)\nendif()\nset(TBB_LIB_COMPILE_FLAGS -D_CRT_SECURE_NO_WARNINGS /GS)\nset(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /volatile:iso /FS /EHsc)\n\nset(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} /DEPENDENTLOADFLAG:0x2000 /DYNAMICBASE /NXCOMPAT)\n\nif (TBB_ARCH EQUAL 32)\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} /SAFESEH )\nendif()\n\n# Ignore /WX set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled.\nif (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)\n tbb_remove_compile_flag(/WX)\nendif()\n\nif (WINDOWS_STORE OR TBB_WINDOWS_DRIVER)\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D_WIN32_WINNT=0x0A00)\n set(TBB_COMMON_LINK_FLAGS -NODEFAULTLIB:kernel32.lib -INCREMENTAL:NO)\n set(TBB_COMMON_LINK_LIBS OneCore.lib)\nendif()\n\nif (WINDOWS_STORE)\n if (NOT CMAKE_SYSTEM_VERSION EQUAL 10.0)\n message(FATAL_ERROR \"CMAKE_SYSTEM_VERSION must be equal to 10.0\")\n endif()\n\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /ZW /ZW:nostdlib)\n\n # CMake define this extra lib, remove it for this build type\n string(REGEX REPLACE \"WindowsApp.lib\" \"\" CMAKE_CXX_STANDARD_LIBRARIES \"${CMAKE_CXX_STANDARD_LIBRARIES}\")\n\n if (TBB_NO_APPCONTAINER)\n set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -APPCONTAINER:NO)\n endif()\nendif()\n\nif (TBB_WINDOWS_DRIVER)\n # Since this is universal driver disable this variable\n set(CMAKE_SYSTEM_PROCESSOR \"\")\n\n # CMake define list additional libs, remove it for this build type\n set(CMAKE_CXX_STANDARD_LIBRARIES \"\")\n\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D _UNICODE /DUNICODE /DWINAPI_FAMILY=WINAPI_FAMILY_APP /D__WRL_NO_DEFAULT_LIB__)\nendif()\n\nif (CMAKE_CXX_COMPILER_ID MATCHES \"(Clang|IntelLLVM)\")\n if (CMAKE_SYSTEM_PROCESSOR MATCHES \"(x86|AMD64|i.86)\")\n set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm -mwaitpkg)\n endif()\n set(TBB_IPO_COMPILE_FLAGS $<$>:-flto>)\n set(TBB_IPO_LINK_FLAGS $<$>:-flto>)\nelse()\n set(TBB_IPO_COMPILE_FLAGS $<$>:/GL>)\n set(TBB_IPO_LINK_FLAGS $<$>:-LTCG> $<$>:-INCREMENTAL:NO>)\nendif()\n\nset(TBB_OPENMP_FLAG /openmp)\nset(TBB_OPENMP_NO_LINK_FLAG TRUE) # TBB_OPENMP_FLAG will be used only on compilation but not on linkage\n" }, { "path": "src/tbb/cmake/compilers/QCC.cmake", "content": "# Copyright (c) 2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ninclude(${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilers/GNU.cmake)\n\n# Remove dl library not present in QNX systems\nunset(TBB_COMMON_LINK_LIBS)\n" }, { "path": "src/tbb/cmake/config_generation.cmake", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Save current location,\n# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html\nset(_tbb_gen_cfg_path ${CMAKE_CURRENT_LIST_DIR})\n\ninclude(CMakeParseArguments)\n\nfunction(tbb_generate_config)\n set(options HANDLE_SUBDIRS)\n set(oneValueArgs INSTALL_DIR\n SYSTEM_NAME\n LIB_REL_PATH INC_REL_PATH\n VERSION\n TBB_BINARY_VERSION\n TBBMALLOC_BINARY_VERSION\n TBBMALLOC_PROXY_BINARY_VERSION\n TBBBIND_BINARY_VERSION)\n\n cmake_parse_arguments(tbb_gen_cfg \"${options}\" \"${oneValueArgs}\" \"${multiValueArgs}\" ${ARGN})\n\n get_filename_component(config_install_dir ${tbb_gen_cfg_INSTALL_DIR} ABSOLUTE)\n file(MAKE_DIRECTORY ${config_install_dir})\n\n file(TO_CMAKE_PATH \"${tbb_gen_cfg_LIB_REL_PATH}\" TBB_LIB_REL_PATH)\n file(TO_CMAKE_PATH \"${tbb_gen_cfg_INC_REL_PATH}\" TBB_INC_REL_PATH)\n\n set(TBB_VERSION ${tbb_gen_cfg_VERSION})\n\n set(_tbb_pc_lib_name tbb)\n set(_prefix_for_pc_file \"\\${pcfiledir}/../../\")\n set(_includedir_for_pc_file \"\\${prefix}/include\")\n\n set(TBB_COMPONENTS_BIN_VERSION \"\nset(_tbb_bin_version ${tbb_gen_cfg_TBB_BINARY_VERSION})\nset(_tbbmalloc_bin_version ${tbb_gen_cfg_TBBMALLOC_BINARY_VERSION})\nset(_tbbmalloc_proxy_bin_version ${tbb_gen_cfg_TBBMALLOC_PROXY_BINARY_VERSION})\nset(_tbbbind_bin_version ${tbb_gen_cfg_TBBBIND_BINARY_VERSION})\n\")\n\n if (tbb_gen_cfg_SYSTEM_NAME STREQUAL \"Linux\")\n set(TBB_LIB_PREFIX \"lib\")\n set(TBB_LIB_EXT \"so.\\${_\\${_tbb_component}_bin_version}\")\n\n set (TBB_HANDLE_IMPLIB \"\n set (_tbb_release_dll \\${_tbb_release_lib})\n set (_tbb_debug_dll \\${_tbb_debug_lib})\n\")\n if (tbb_gen_cfg_HANDLE_SUBDIRS)\n set(TBB_HANDLE_SUBDIRS \"set(_tbb_subdir gcc4.8)\")\n\n set(_libdir_for_pc_file \"\\${prefix}/lib/intel64/gcc4.8\")\n set(_tbb_pc_extra_libdir \"-L\\${prefix}/lib\")\n configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)\n\n set(_libdir_for_pc_file \"\\${prefix}/lib/ia32/gcc4.8\")\n set(_tbb_pc_extra_libdir \"-L\\${prefix}/lib32\")\n configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY)\n endif()\n elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL \"Darwin\")\n set(TBB_LIB_PREFIX \"lib\")\n set(TBB_LIB_EXT \"\\${_\\${_tbb_component}_bin_version}.dylib\")\n\n set (TBB_HANDLE_IMPLIB \"\n set (_tbb_release_dll \\${_tbb_release_lib})\n set (_tbb_debug_dll \\${_tbb_debug_lib})\n\")\n set(_libdir_for_pc_file \"\\${prefix}/lib\")\n configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)\n elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL \"Windows\")\n set(TBB_LIB_PREFIX \"\")\n set(TBB_LIB_EXT \"lib\")\n set(TBB_COMPILE_DEFINITIONS \"\n INTERFACE_COMPILE_DEFINITIONS \\\"__TBB_NO_IMPLICIT_LINKAGE=1\\\"\")\n \n # .lib - installed to TBB_LIB_REL_PATH (e.g. /lib) and are passed as IMPORTED_IMPLIB_ property to target\n # .dll - installed to /bin or /redist and are passed as IMPORTED_LOCATION_ property to target\n set (TBB_HANDLE_IMPLIB \"\n find_file(_tbb_release_dll\n NAMES \\${_tbb_component}\\${_bin_version}.dll\n PATHS \\${_tbb_root}\n PATH_SUFFIXES \\\"redist/\\${_tbb_intel_arch}/\\${_tbb_subdir}\\\" \\\"bin\\${_tbb_arch_suffix}/\\${_tbb_subdir}\\\" \\\"bin\\${_tbb_arch_suffix}/\\\" \\\"bin\\\"\n NO_DEFAULT_PATH\n )\n\n if (EXISTS \\\"\\${_tbb_debug_lib}\\\")\n find_file(_tbb_debug_dll\n NAMES \\${_tbb_component}\\${_bin_version}_debug.dll\n PATHS \\${_tbb_root}\n PATH_SUFFIXES \\\"redist/\\${_tbb_intel_arch}/\\${_tbb_subdir}\\\" \\\"bin\\${_tbb_arch_suffix}/\\${_tbb_subdir}\\\" \\\"bin\\${_tbb_arch_suffix}/\\\" \\\"bin\\\"\n NO_DEFAULT_PATH\n )\n endif()\n\")\n set(TBB_IMPLIB_RELEASE \"\n IMPORTED_IMPLIB_RELEASE \\\"\\${_tbb_release_lib}\\\"\")\n set(TBB_IMPLIB_DEBUG \"\n IMPORTED_IMPLIB_DEBUG \\\"\\${_tbb_debug_lib}\\\"\")\n\n if (tbb_gen_cfg_HANDLE_SUBDIRS)\n set(TBB_HANDLE_SUBDIRS \"\nset(_tbb_subdir vc14)\nif (WINDOWS_STORE)\n set(_tbb_subdir \\${_tbb_subdir}_uwp)\nendif()\n\")\n set(_tbb_pc_lib_name ${_tbb_pc_lib_name}${TBB_BINARY_VERSION})\n\n set(_libdir_for_pc_file \"\\${prefix}/lib/intel64/vc14\")\n set(_tbb_pc_extra_libdir \"-L\\${prefix}/lib\")\n configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)\n\n set(_libdir_for_pc_file \"\\${prefix}/lib/ia32/vc14\")\n set(_tbb_pc_extra_libdir \"-L\\${prefix}/lib32\")\n configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY)\n endif()\n\n set(TBB_HANDLE_BIN_VERSION \"\n unset(_bin_version)\n if (_tbb_component STREQUAL tbb)\n set(_bin_version \\${_tbb_bin_version})\n endif()\n\")\n else()\n message(FATAL_ERROR \"Unsupported OS name: ${tbb_system_name}\")\n endif()\n\n configure_file(${_tbb_gen_cfg_path}/templates/TBBConfig.cmake.in ${config_install_dir}/TBBConfig.cmake @ONLY)\n configure_file(${_tbb_gen_cfg_path}/templates/TBBConfigVersion.cmake.in ${config_install_dir}/TBBConfigVersion.cmake @ONLY)\nendfunction()\n" }, { "path": "src/tbb/cmake/hwloc_detection.cmake", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlist(APPEND HWLOC_REQUIRED_VERSIONS 1_11 2 2_5)\n\nforeach(hwloc_version ${HWLOC_REQUIRED_VERSIONS})\n if (NOT WIN32)\n set(CMAKE_HWLOC_${hwloc_version}_DLL_PATH STUB)\n endif()\n set(HWLOC_TARGET_NAME HWLOC::hwloc_${hwloc_version})\n\n if (NOT TARGET ${HWLOC_TARGET_NAME} AND\n CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH AND\n CMAKE_HWLOC_${hwloc_version}_DLL_PATH AND\n CMAKE_HWLOC_${hwloc_version}_INCLUDE_PATH\n )\n add_library(${HWLOC_TARGET_NAME} SHARED IMPORTED)\n set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES\n \"${CMAKE_HWLOC_${hwloc_version}_INCLUDE_PATH}\")\n if (WIN32)\n set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES\n IMPORTED_LOCATION \"${CMAKE_HWLOC_${hwloc_version}_DLL_PATH}\"\n IMPORTED_IMPLIB \"${CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH}\")\n else()\n set_target_properties(${HWLOC_TARGET_NAME} PROPERTIES\n IMPORTED_LOCATION \"${CMAKE_HWLOC_${hwloc_version}_LIBRARY_PATH}\")\n endif()\n endif()\n\n if (TARGET ${HWLOC_TARGET_NAME})\n set(HWLOC_TARGET_EXPLICITLY_DEFINED TRUE)\n endif()\nendforeach()\n\nunset(HWLOC_TARGET_NAME)\n\nif (NOT HWLOC_TARGET_EXPLICITLY_DEFINED AND\n NOT TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH\n)\n find_package(PkgConfig QUIET)\n if (PKG_CONFIG_FOUND AND NOT CMAKE_VERSION VERSION_LESS 3.6)\n pkg_search_module(HWLOC hwloc IMPORTED_TARGET)\n if (TARGET PkgConfig::HWLOC)\n if (HWLOC_VERSION VERSION_LESS 2)\n set(TBBBIND_LIBRARY_NAME tbbbind)\n elseif(HWLOC_VERSION VERSION_LESS 2.5)\n set(TBBBIND_LIBRARY_NAME tbbbind_2_0)\n else()\n set(TBBBIND_LIBRARY_NAME tbbbind_2_5)\n endif()\n endif()\n endif()\nendif()\n" }, { "path": "src/tbb/cmake/memcheck.cmake", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\noption(TBB_VALGRIND_MEMCHECK \"Enable scan for memory leaks using Valgrind\" OFF)\n\nif (NOT TBB_VALGRIND_MEMCHECK)\n return()\nendif()\n\nadd_custom_target(memcheck-all\n COMMENT \"Run memcheck on all tests\")\n\nfind_program(VALGRIND_EXE valgrind)\n\nif (NOT VALGRIND_EXE)\n message(FATAL_ERROR \"Valgrind executable is not found, add tool to PATH or turn off TBB_VALGRIND_MEMCHECK\")\nelse()\n message(STATUS \"Found Valgrind to run memory leak scan\")\nendif()\n\nfile(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/memcheck)\n\nfunction(_tbb_run_memcheck test_target subdir)\n set(target_name memcheck-${test_target})\n if(${subdir} STREQUAL \"tbbmalloc\")\n\t# Valgring intercepts all allocation symbols with its own by default,\n\t# so it disables using tbbmalloc. In case of tbbmalloc tests\n\t# intercept allocation symbols only in the default system libraries,\n\t# but not in any other shared library or the executable\n\t# defining public malloc or operator new related functions.\n\tset(option \"--soname-synonyms=somalloc=nouserintercepts\")\n endif()\n add_custom_target(${target_name} \n COMMAND ${VALGRIND_EXE} ${option} --leak-check=full --show-leak-kinds=all --log-file=${CMAKE_BINARY_DIR}/memcheck/${target_name}.log -v $)\n add_dependencies(memcheck-all ${target_name})\nendfunction()\n\nadd_custom_target(memcheck-short\n COMMENT \"Run memcheck scan on specified list\")\n\n# List of reasonable and quick enough tests to use in automated memcheck\nadd_dependencies(memcheck-short \n memcheck-test_allocators\n memcheck-test_arena_constraints\n memcheck-test_dynamic_link\n memcheck-test_concurrent_lru_cache\n memcheck-conformance_concurrent_unordered_map\n memcheck-conformance_concurrent_unordered_set\n memcheck-conformance_concurrent_map\n memcheck-conformance_concurrent_set\n memcheck-conformance_concurrent_priority_queue\n memcheck-conformance_concurrent_vector\n memcheck-conformance_concurrent_queue\n memcheck-conformance_concurrent_hash_map\n memcheck-test_parallel_for\n memcheck-test_parallel_for_each\n memcheck-test_parallel_reduce\n memcheck-test_parallel_sort\n memcheck-test_parallel_invoke\n memcheck-test_parallel_scan\n memcheck-test_parallel_pipeline\n memcheck-test_eh_algorithms\n memcheck-test_task_group\n memcheck-test_task_arena\n memcheck-test_enumerable_thread_specific\n memcheck-test_resumable_tasks\n memcheck-conformance_mutex\n memcheck-test_function_node\n memcheck-test_multifunction_node\n memcheck-test_broadcast_node\n memcheck-test_buffer_node\n memcheck-test_composite_node\n memcheck-test_continue_node\n memcheck-test_eh_flow_graph\n memcheck-test_flow_graph\n memcheck-test_flow_graph_priorities\n memcheck-test_flow_graph_whitebox\n memcheck-test_indexer_node\n memcheck-test_join_node\n memcheck-test_join_node_key_matching\n memcheck-test_join_node_msg_key_matching\n memcheck-test_priority_queue_node\n memcheck-test_sequencer_node\n memcheck-test_split_node\n memcheck-test_tagged_msg\n memcheck-test_overwrite_node\n memcheck-test_write_once_node\n memcheck-test_async_node\n memcheck-test_input_node\n memcheck-test_profiling\n memcheck-test_concurrent_queue_whitebox\n memcheck-test_intrusive_list\n memcheck-test_semaphore\n memcheck-test_environment_whitebox\n memcheck-test_handle_perror\n memcheck-test_hw_concurrency\n memcheck-test_eh_thread\n memcheck-test_global_control\n memcheck-test_task\n memcheck-test_concurrent_monitor\n)\n" }, { "path": "src/tbb/cmake/packaging.cmake", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Note: current implementation uses CMAKE_BUILD_TYPE,\n# this parameter is not defined for multi-config generators.\nset(CPACK_PACKAGE_NAME \"${PROJECT_NAME}\")\nset(CPACK_PACKAGE_VERSION \"${TBB_VERSION}\")\nstring(TOLOWER ${CPACK_PACKAGE_NAME}-${PROJECT_VERSION}-${CMAKE_SYSTEM_NAME}_${TBB_OUTPUT_DIR_BASE}_${CMAKE_BUILD_TYPE} CPACK_PACKAGE_FILE_NAME)\nset(CPACK_GENERATOR ZIP)\n# Note: this is an internal non-documented variable set by CPack \nif (NOT CPack_CMake_INCLUDED)\n include(CPack)\nendif()\n" }, { "path": "src/tbb/cmake/post_install/CMakeLists.txt", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Add code signing as post-install step.\nif (DEFINED TBB_SIGNTOOL)\n file(TO_CMAKE_PATH \"${TBB_SIGNTOOL}\" TBB_SIGNTOOL)\n install(CODE \"\n file(GLOB_RECURSE FILES_TO_SIGN \\${CMAKE_INSTALL_PREFIX}/*${CMAKE_SHARED_LIBRARY_SUFFIX})\n execute_process(COMMAND ${TBB_SIGNTOOL} \\${FILES_TO_SIGN} ${TBB_SIGNTOOL_ARGS})\n\")\nendif()\n" }, { "path": "src/tbb/cmake/python/test_launcher.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfind_package(PythonInterp 3.5 REQUIRED)\n\nfile(GLOB_RECURSE MODULES_LIST \"${PYTHON_MODULE_BUILD_PATH}/*TBB.py*\" )\nlist(LENGTH MODULES_LIST MODULES_COUNT)\n\nif (MODULES_COUNT EQUAL 0)\n message(FATAL_ERROR \"Cannot find oneTBB Python module\")\nelseif (MODULES_COUNT GREATER 1)\n message(WARNING \"Found more than oneTBB Python modules, the only first found module will be tested\")\nendif()\n\nlist(GET MODULES_LIST 0 PYTHON_MODULE)\nget_filename_component(PYTHON_MODULE_PATH ${PYTHON_MODULE} DIRECTORY)\n\nexecute_process(\n COMMAND\n ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${TBB_BINARIES_PATH}\n ${PYTHON_EXECUTABLE} -m tbb test\n WORKING_DIRECTORY ${PYTHON_MODULE_PATH}\n RESULT_VARIABLE CMD_RESULT\n)\nif (CMD_RESULT)\n message(FATAL_ERROR \"Error while test execution: ${cmd} error_code: ${CMD_RESULT}\")\nendif()\n" }, { "path": "src/tbb/cmake/resumable_tasks.cmake", "content": "# Copyright (c) 2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ninclude(CheckSymbolExists)\n\nif (UNIX)\n set(CMAKE_REQUIRED_FLAGS -Wno-deprecated-declarations)\n if (APPLE)\n set(CMAKE_REQUIRED_DEFINITIONS -D_XOPEN_SOURCE)\n endif()\n\n check_symbol_exists(\"getcontext\" \"ucontext.h\" _tbb_have_ucontext)\n if (NOT _tbb_have_ucontext)\n set(TBB_RESUMABLE_TASKS_USE_THREADS \"__TBB_RESUMABLE_TASKS_USE_THREADS=1\")\n endif()\n\n unset(_tbb_have_ucontext)\n unset(CMAKE_REQUIRED_DEFINITIONS)\n unset(CMAKE_REQUIRED_FLAGS)\nendif()\n" }, { "path": "src/tbb/cmake/sanitize.cmake", "content": "# Copyright (c) 2020-2022 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(TBB_SANITIZE ${TBB_SANITIZE} CACHE STRING \"Sanitizer parameter passed to compiler/linker\" FORCE)\n# Possible values of sanitizer parameter for cmake-gui for convenience, user still can use any other value.\nset_property(CACHE TBB_SANITIZE PROPERTY STRINGS \"thread\" \"memory\" \"leak\" \"address -fno-omit-frame-pointer\")\n\nif (NOT TBB_SANITIZE)\n return()\nendif()\n\nset(TBB_SANITIZE_OPTION -fsanitize=${TBB_SANITIZE})\n\n# It is required to add sanitizer option to CMAKE_REQUIRED_LIBRARIES to make check_cxx_compiler_flag working properly:\n# sanitizer option should be passed during the compilation phase as well as during the compilation.\nset(CMAKE_REQUIRED_LIBRARIES \"${TBB_SANITIZE_OPTION} ${CMAKE_REQUIRED_LIBRARIES}\")\n\nstring(MAKE_C_IDENTIFIER ${TBB_SANITIZE_OPTION} FLAG_DISPLAY_NAME)\ncheck_cxx_compiler_flag(${TBB_SANITIZE_OPTION} ${FLAG_DISPLAY_NAME})\nif (NOT ${FLAG_DISPLAY_NAME})\n message(FATAL_ERROR\n \"${TBB_SANITIZE_OPTION} is not supported by compiler ${CMAKE_CXX_COMPILER_ID}:${CMAKE_CXX_COMPILER_VERSION}, \"\n \"please try another compiler or omit TBB_SANITIZE variable\")\nendif()\n\nset(TBB_TESTS_ENVIRONMENT ${TBB_TESTS_ENVIRONMENT}\n \"TSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/cmake/suppressions/tsan.suppressions\"\n \"LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/cmake/suppressions/lsan.suppressions\")\n\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${TBB_SANITIZE_OPTION}\")\nset(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${TBB_SANITIZE_OPTION}\")\nset(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} ${TBB_SANITIZE_OPTION}\")\n" }, { "path": "src/tbb/cmake/scripts/cmake_gen_github_configs.cmake", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ninclude(${CMAKE_CURRENT_LIST_DIR}/../config_generation.cmake)\n\n# TBBConfig in TBB provided packages are expected to be placed into: /lib/cmake/tbb*\nset(TBB_ROOT_REL_PATH \"../../..\")\n\n# Paths relative to TBB root directory\nset(INC_REL_PATH \"include\")\nset(LIB_REL_PATH \"lib\")\n\n# Parse version info\nfile(READ ${CMAKE_CURRENT_LIST_DIR}/../../include/oneapi/tbb/version.h _tbb_version_info)\nstring(REGEX REPLACE \".*#define TBB_VERSION_MAJOR ([0-9]+).*\" \"\\\\1\" _tbb_ver_major \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define TBB_VERSION_MINOR ([0-9]+).*\" \"\\\\1\" _tbb_ver_minor \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define TBB_VERSION_PATCH ([0-9]+).*\" \"\\\\1\" _tbb_ver_patch \"${_tbb_version_info}\")\nstring(REGEX REPLACE \".*#define __TBB_BINARY_VERSION ([0-9]+).*\" \"\\\\1\" TBB_BINARY_VERSION \"${_tbb_version_info}\")\nfile(READ ${CMAKE_CURRENT_LIST_DIR}/../../CMakeLists.txt _tbb_cmakelist)\nstring(REGEX REPLACE \".*TBBMALLOC_BINARY_VERSION ([0-9]+).*\" \"\\\\1\" TBBMALLOC_BINARY_VERSION \"${_tbb_cmakelist}\")\nset(TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION})\nstring(REGEX REPLACE \".*TBBBIND_BINARY_VERSION ([0-9]+).*\" \"\\\\1\" TBBBIND_BINARY_VERSION \"${_tbb_cmakelist}\")\n\nset(COMMON_ARGS\n TBB_ROOT_REL_PATH ${TBB_ROOT_REL_PATH}\n INC_REL_PATH ${INC_REL_PATH}\n LIB_REL_PATH ${LIB_REL_PATH}\n VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch}\n TBB_BINARY_VERSION ${TBB_BINARY_VERSION}\n TBBMALLOC_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION}\n TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_PROXY_BINARY_VERSION}\n TBBBIND_BINARY_VERSION ${TBBBIND_BINARY_VERSION}\n)\n\ntbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/linux SYSTEM_NAME Linux HANDLE_SUBDIRS ${COMMON_ARGS})\ntbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/windows SYSTEM_NAME Windows HANDLE_SUBDIRS ${COMMON_ARGS})\ntbb_generate_config(INSTALL_DIR ${INSTALL_DIR}/darwin SYSTEM_NAME Darwin ${COMMON_ARGS})\nmessage(STATUS \"TBBConfig files were created in ${INSTALL_DIR}\")\n" }, { "path": "src/tbb/cmake/suppressions/lsan.suppressions", "content": "# LSAN suppression for ltdl library known issue.\nleak:libltdl.so\n" }, { "path": "src/tbb/cmake/suppressions/tsan.suppressions", "content": "# TSAN suppression for known issues.\n# Possible data race during ittnotify initialization. Low impact.\nrace:__itt_nullify_all_pointers\nrace:__itt_init_ittlib\n" }, { "path": "src/tbb/cmake/templates/TBBConfig.cmake.in", "content": "# Copyright (c) 2017-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# It defines the following variables:\n# TBB__FOUND\n# TBB_IMPORTED_TARGETS\n#\n# TBBConfigVersion.cmake defines TBB_VERSION\n#\n# Initialize to default values\nif (NOT TBB_IMPORTED_TARGETS)\n set(TBB_IMPORTED_TARGETS \"\")\nendif()\n\nif (NOT TBB_FIND_COMPONENTS)\n set(TBB_FIND_COMPONENTS \"tbb;tbbmalloc;tbbmalloc_proxy\")\n foreach (_tbb_component ${TBB_FIND_COMPONENTS})\n set(TBB_FIND_REQUIRED_${_tbb_component} 1)\n endforeach()\nendif()\n\nget_filename_component(_tbb_root \"${CMAKE_CURRENT_LIST_DIR}\" REALPATH)\nget_filename_component(_tbb_root \"${_tbb_root}/@TBB_ROOT_REL_PATH@\" ABSOLUTE)\n\nset(TBB_INTERFACE_VERSION @TBB_INTERFACE_VERSION@)\n@TBB_COMPONENTS_BIN_VERSION@\n# Add components with internal dependencies: tbbmalloc_proxy -> tbbmalloc\nlist(FIND TBB_FIND_COMPONENTS tbbmalloc_proxy _tbbmalloc_proxy_ix)\nif (NOT _tbbmalloc_proxy_ix EQUAL -1)\n list(APPEND TBB_FIND_COMPONENTS tbbmalloc)\n list(REMOVE_DUPLICATES TBB_FIND_COMPONENTS)\n set(TBB_FIND_REQUIRED_tbbmalloc ${TBB_FIND_REQUIRED_tbbmalloc_proxy})\nendif()\nunset(_tbbmalloc_proxy_ix)\n\nif (CMAKE_SIZEOF_VOID_P STREQUAL \"8\")\n set(_tbb_intel_arch intel64)\nelse ()\n set(_tbb_intel_arch ia32)\n set(_tbb_arch_suffix 32)\nendif()\n\n@TBB_HANDLE_SUBDIRS@\nforeach (_tbb_component ${TBB_FIND_COMPONENTS})\n unset(_tbb_release_dll CACHE)\n unset(_tbb_debug_dll CACHE)\n unset(_tbb_release_lib CACHE)\n unset(_tbb_debug_lib CACHE)\n\n set(TBB_${_tbb_component}_FOUND 0)\n @TBB_HANDLE_BIN_VERSION@\n\n find_library(_tbb_release_lib\n NAMES @TBB_LIB_PREFIX@${_tbb_component}${_bin_version}.@TBB_LIB_EXT@\n PATHS ${_tbb_root}\n PATH_SUFFIXES \"@TBB_LIB_REL_PATH@/${_tbb_intel_arch}/${_tbb_subdir}\" \"@TBB_LIB_REL_PATH@${_tbb_arch_suffix}/${_tbb_subdir}\" \"@TBB_LIB_REL_PATH@${_tbb_arch_suffix}\" \"@TBB_LIB_REL_PATH@\"\n NO_DEFAULT_PATH\n )\n\n if (NOT TBB_FIND_RELEASE_ONLY)\n find_library(_tbb_debug_lib\n NAMES @TBB_LIB_PREFIX@${_tbb_component}${_bin_version}_debug.@TBB_LIB_EXT@\n PATHS ${_tbb_root}\n PATH_SUFFIXES \"@TBB_LIB_REL_PATH@/${_tbb_intel_arch}/${_tbb_subdir}\" \"@TBB_LIB_REL_PATH@${_tbb_arch_suffix}/${_tbb_subdir}\" \"@TBB_LIB_REL_PATH@${_tbb_arch_suffix}\" \"@TBB_LIB_REL_PATH@\"\n NO_DEFAULT_PATH\n )\n endif()\n\n if (EXISTS \"${_tbb_release_lib}\" OR EXISTS \"${_tbb_debug_lib}\")\n if (NOT TARGET TBB::${_tbb_component})\n add_library(TBB::${_tbb_component} SHARED IMPORTED)\n\n get_filename_component(_tbb_include_dir \"${_tbb_root}/@TBB_INC_REL_PATH@\" ABSOLUTE)\n set_target_properties(TBB::${_tbb_component} PROPERTIES\n INTERFACE_INCLUDE_DIRECTORIES \"${_tbb_include_dir}\"@TBB_COMPILE_DEFINITIONS@)\n unset(_tbb_current_realpath)\n unset(_tbb_include_dir)\n\n @TBB_HANDLE_IMPLIB@\n\n if (EXISTS \"${_tbb_release_dll}\")\n set_target_properties(TBB::${_tbb_component} PROPERTIES\n IMPORTED_LOCATION_RELEASE \"${_tbb_release_dll}\"@TBB_IMPLIB_RELEASE@)\n set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE)\n endif()\n\n if (EXISTS \"${_tbb_debug_dll}\")\n set_target_properties(TBB::${_tbb_component} PROPERTIES\n IMPORTED_LOCATION_DEBUG \"${_tbb_debug_dll}\"@TBB_IMPLIB_DEBUG@)\n set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG)\n endif()\n\n # Add internal dependencies for imported targets: TBB::tbbmalloc_proxy -> TBB::tbbmalloc\n if (_tbb_component STREQUAL tbbmalloc_proxy)\n set_target_properties(TBB::tbbmalloc_proxy PROPERTIES INTERFACE_LINK_LIBRARIES TBB::tbbmalloc)\n endif()\n endif()\n list(APPEND TBB_IMPORTED_TARGETS TBB::${_tbb_component})\n set(TBB_${_tbb_component}_FOUND 1)\n elseif (TBB_FIND_REQUIRED AND TBB_FIND_REQUIRED_${_tbb_component})\n message(STATUS \"Missed required oneTBB component: ${_tbb_component}\")\n if (TBB_FIND_RELEASE_ONLY)\n message(STATUS \" ${_tbb_release_lib} must exist.\")\n else()\n message(STATUS \" one or both of:\\n ${_tbb_release_lib}\\n ${_tbb_debug_lib}\\n files must exist.\")\n endif()\n set(TBB_FOUND FALSE)\n endif()\nendforeach()\nlist(REMOVE_DUPLICATES TBB_IMPORTED_TARGETS)\nunset(_tbb_release_dll)\nunset(_tbb_debug_dll)\nunset(_tbb_release_lib)\nunset(_tbb_debug_lib)\nunset(_tbb_root)\nunset(_tbb_intel_arch)\nunset(_tbb_arch_suffix)\n" }, { "path": "src/tbb/cmake/templates/TBBConfigVersion.cmake.in", "content": "# Copyright (c) 2017-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(PACKAGE_VERSION @TBB_VERSION@)\n\nif (\"${PACKAGE_VERSION}\" VERSION_LESS \"${PACKAGE_FIND_VERSION}\")\n set(PACKAGE_VERSION_COMPATIBLE FALSE)\nelse()\n set(PACKAGE_VERSION_COMPATIBLE TRUE)\n if (\"${PACKAGE_VERSION}\" VERSION_EQUAL \"${PACKAGE_FIND_VERSION}\")\n set(PACKAGE_VERSION_EXACT TRUE)\n endif()\nendif()\n" }, { "path": "src/tbb/cmake/test_spec.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\noption(TBB_TEST_SPEC \"Generate test specification (Doxygen)\" OFF)\n\nif (TBB_TEST_SPEC)\n find_package(Doxygen REQUIRED)\n\n set(DOXYGEN_PREDEFINED_MACROS\n \"TBB_USE_EXCEPTIONS \\\n __TBB_RESUMABLE_TASKS \\\n __TBB_HWLOC_PRESENT \\\n __TBB_CPP17_DEDUCTION_GUIDES_PRESENT \\\n __TBB_CPP17_MEMORY_RESOURCE_PRESENT \\\n __TBB_CPP14_GENERIC_LAMBDAS_PRESENT\"\n )\n\n add_custom_target(\n test_spec ALL\n COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile\n COMMENT \"Generating test specification with Doxygen\"\n VERBATIM)\n configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)\nendif()\n" }, { "path": "src/tbb/cmake/toolchains/mips.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Prevent double invocation.\nif (MIPS_TOOLCHAIN_INCLUDED)\n return()\nendif()\nset(MIPS_TOOLCHAIN_INCLUDED TRUE)\n\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_VERSION 1)\nset(CMAKE_SYSTEM_PROCESSOR mips)\n\nset(CMAKE_C_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-gcc)\nset(CMAKE_CXX_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-g++)\nset(CMAKE_LINKER ${CMAKE_FIND_ROOT_PATH}/bin/mips-img-linux-gnu-ld)\n\n# Define result for try_run used in find_package(Threads).\n# In old CMake versions (checked on 3.5) there is invocation of try_run command in FindThreads.cmake module.\n# It can't be executed on host system in case of cross-compilation for MIPS architecture.\n# Define return code for this try_run as 0 since threads are expected to be available on target machine.\nset(THREADS_PTHREAD_ARG \"0\" CACHE STRING \"Result from TRY_RUN\" FORCE)\n\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\n\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -EL -mabi=64 -march=mips64r6 -mcrc -mfp64 -mmt -mtune=mips64r6 -ggdb -ffp-contract=off -mhard-float\" CACHE INTERNAL \"\")\nset(CMAKE_SHARED_LINKER_FLAGS \"${CMAKE_SHARED_LINKER_FLAGS} -mvirt -mxpa\" CACHE INTERNAL \"\")\nset(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} -mvirt -mxpa\" CACHE INTERNAL \"\") # for tests\n" }, { "path": "src/tbb/cmake/toolchains/riscv64.cmake", "content": "# Copyright (c) 2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Prevent double invocation.\nif (RISCV_TOOLCHAIN_INCLUDED)\n return()\nendif()\nset(RISCV_TOOLCHAIN_INCLUDED TRUE)\n\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_VERSION 1)\nset(CMAKE_SYSTEM_PROCESSOR riscv)\n\n# User can use -DCMAKE_FIND_ROOT_PATH to specific toolchain path\nset(CMAKE_C_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-clang)\nset(CMAKE_CXX_COMPILER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-clang++)\nset(CMAKE_LINKER ${CMAKE_FIND_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-ld)\n\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\n\n# Most linux on riscv64 support rv64imafd_zba_zbb extensions\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -march=rv64imafd_zba_zbb -mabi=lp64d \" CACHE INTERNAL \"\")\n" }, { "path": "src/tbb/cmake/utils.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nmacro(tbb_remove_compile_flag flag)\n get_property(_tbb_compile_options DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS)\n list(REMOVE_ITEM _tbb_compile_options ${flag})\n set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS ${_tbb_compile_options})\n unset(_tbb_compile_options)\n if (CMAKE_CXX_FLAGS)\n string(REGEX REPLACE \"(^|[ \\t\\r\\n]+)${flag}($|[ \\t\\r\\n]+)\" \" \" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})\n endif()\nendmacro()\n\nmacro(tbb_install_target target)\n if (TBB_INSTALL)\n install(TARGETS ${target}\n EXPORT TBBTargets\n LIBRARY\n DESTINATION ${CMAKE_INSTALL_LIBDIR}\n NAMELINK_SKIP\n COMPONENT runtime\n RUNTIME\n DESTINATION ${CMAKE_INSTALL_BINDIR}\n COMPONENT runtime\n ARCHIVE\n DESTINATION ${CMAKE_INSTALL_LIBDIR}\n COMPONENT devel\n FRAMEWORK\n DESTINATION ${CMAKE_INSTALL_LIBDIR}\n COMPONENT runtime\n OPTIONAL)\n\n if (BUILD_SHARED_LIBS)\n install(TARGETS ${target}\n LIBRARY\n DESTINATION ${CMAKE_INSTALL_LIBDIR}\n NAMELINK_ONLY\n COMPONENT devel)\n endif()\n if (MSVC AND BUILD_SHARED_LIBS)\n install(FILES $\n DESTINATION ${CMAKE_INSTALL_BINDIR}\n COMPONENT devel\n OPTIONAL)\n endif()\n endif()\nendmacro()\n\nmacro(tbb_handle_ipo target)\n if (TBB_IPO_PROPERTY)\n set_target_properties(${target} PROPERTIES \n INTERPROCEDURAL_OPTIMIZATION TRUE\n INTERPROCEDURAL_OPTIMIZATION_DEBUG FALSE\n )\n elseif (TBB_IPO_FLAGS)\n target_compile_options(${target} PRIVATE ${TBB_IPO_COMPILE_FLAGS})\n if (COMMAND target_link_options)\n target_link_options(${target} PRIVATE ${TBB_IPO_LINK_FLAGS})\n else()\n target_link_libraries(${target} PRIVATE ${TBB_IPO_LINK_FLAGS})\n endif()\n endif()\nendmacro()\n" }, { "path": "src/tbb/cmake/vars_utils.cmake", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\noption(TBB_INSTALL_VARS \"Enable auto-generated vars installation\" OFF)\n\nif (WIN32)\n set(TBB_VARS_TEMPLATE \"windows/env/vars.bat.in\")\nelseif (APPLE)\n set(TBB_VARS_TEMPLATE \"mac/env/vars.sh.in\")\nelse()\n set(TBB_VARS_TEMPLATE \"linux/env/vars.sh.in\")\nendif()\n\nget_filename_component(TBB_VARS_TEMPLATE_NAME ${PROJECT_SOURCE_DIR}/integration/${TBB_VARS_TEMPLATE} NAME)\nstring(REPLACE \".in\" \"\" TBB_VARS_NAME ${TBB_VARS_TEMPLATE_NAME})\n\nmacro(tbb_gen_vars target)\n if (NOT TBB_BUILD_APPLE_FRAMEWORKS)\n set(BIN_PATH $)\n else()\n # For Apple* frameworks, the binaries are placed in a framework bundle. \n # When using an Apple* framework, you refer to the bundle, not the binary inside, so we take the bundle's path and go up one level.\n # This path will then be used to generate the vars file, and the contents of the vars file will use the bundle's parent directory.\n set(BIN_PATH $/..)\n endif()\n if (${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME})\n add_custom_command(TARGET ${target} POST_BUILD COMMAND\n ${CMAKE_COMMAND}\n -DBINARY_DIR=${CMAKE_BINARY_DIR}\n -DSOURCE_DIR=${PROJECT_SOURCE_DIR}\n -DBIN_PATH=${BIN_PATH}\n -DVARS_TEMPLATE=${TBB_VARS_TEMPLATE}\n -DVARS_NAME=${TBB_VARS_NAME}\n -DTBB_INSTALL_VARS=${TBB_INSTALL_VARS}\n -DTBB_CMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}\n -P ${PROJECT_SOURCE_DIR}/integration/cmake/generate_vars.cmake\n )\n endif()\nendmacro(tbb_gen_vars)\n\nif (TBB_INSTALL_VARS)\n install(PROGRAMS \"${CMAKE_BINARY_DIR}/internal_install_vars\"\n DESTINATION env\n RENAME ${TBB_VARS_NAME})\nendif()\n" }, { "path": "src/tbb/include/oneapi/tbb/blocked_range.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_blocked_range_H\n#define __TBB_blocked_range_H\n\n#include \n\n#include \"detail/_range_common.h\"\n#include \"detail/_namespace_injection.h\"\n\n#include \"version.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n/** \\page range_req Requirements on range concept\n Class \\c R implementing the concept of range must define:\n - \\code R::R( const R& ); \\endcode Copy constructor\n - \\code R::~R(); \\endcode Destructor\n - \\code bool R::is_divisible() const; \\endcode True if range can be partitioned into two subranges\n - \\code bool R::empty() const; \\endcode True if range is empty\n - \\code R::R( R& r, split ); \\endcode Split range \\c r into two subranges.\n**/\n\n//! A range over which to iterate.\n/** @ingroup algorithms */\ntemplate\n __TBB_requires(blocked_range_value)\nclass blocked_range {\npublic:\n //! Type of a value\n /** Called a const_iterator for sake of algorithms that need to treat a blocked_range\n as an STL container. */\n using const_iterator = Value;\n\n //! Type for size of a range\n using size_type = std::size_t;\n\n //! Construct range over half-open interval [begin,end), with the given grainsize.\n blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) :\n my_end(end_), my_begin(begin_), my_grainsize(grainsize_)\n {\n __TBB_ASSERT( my_grainsize>0, \"grainsize must be positive\" );\n }\n\n //! Beginning of range.\n const_iterator begin() const { return my_begin; }\n\n //! One past last value in range.\n const_iterator end() const { return my_end; }\n\n //! Size of the range\n /** Unspecified if end()\n __TBB_requires(blocked_range_value &&\n blocked_range_value)\n friend class blocked_range2d;\n\n template\n __TBB_requires(blocked_range_value &&\n blocked_range_value &&\n blocked_range_value)\n friend class blocked_range3d;\n\n template\n __TBB_requires(blocked_range_value)\n friend class blocked_rangeNd_impl;\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::blocked_range;\n// Split types\nusing detail::split;\nusing detail::proportional_split;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_blocked_range_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/blocked_range2d.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_blocked_range2d_H\n#define __TBB_blocked_range2d_H\n\n#include \n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_range_common.h\"\n\n#include \"blocked_range.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! A 2-dimensional range that models the Range concept.\n/** @ingroup algorithms */\ntemplate\n __TBB_requires(blocked_range_value &&\n blocked_range_value)\nclass blocked_range2d {\npublic:\n //! Type for size of an iteration range\n using row_range_type = blocked_range;\n using col_range_type = blocked_range;\n\nprivate:\n row_range_type my_rows;\n col_range_type my_cols;\n\npublic:\n blocked_range2d( RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize,\n ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) :\n my_rows(row_begin,row_end,row_grainsize),\n my_cols(col_begin,col_end,col_grainsize)\n {}\n\n blocked_range2d( RowValue row_begin, RowValue row_end,\n ColValue col_begin, ColValue col_end ) :\n my_rows(row_begin,row_end),\n my_cols(col_begin,col_end)\n {}\n\n //! True if range is empty\n bool empty() const {\n // Range is empty if at least one dimension is empty.\n return my_rows.empty() || my_cols.empty();\n }\n\n //! True if range is divisible into two pieces.\n bool is_divisible() const {\n return my_rows.is_divisible() || my_cols.is_divisible();\n }\n\n blocked_range2d( blocked_range2d& r, split ) :\n my_rows(r.my_rows),\n my_cols(r.my_cols)\n {\n split split_obj;\n do_split(r, split_obj);\n }\n\n blocked_range2d( blocked_range2d& r, proportional_split& proportion ) :\n my_rows(r.my_rows),\n my_cols(r.my_cols)\n {\n do_split(r, proportion);\n }\n\n //! The rows of the iteration space\n const row_range_type& rows() const { return my_rows; }\n\n //! The columns of the iteration space\n const col_range_type& cols() const { return my_cols; }\n\nprivate:\n template \n void do_split( blocked_range2d& r, Split& split_obj ) {\n if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) {\n my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj);\n } else {\n my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj);\n }\n }\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::blocked_range2d;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_blocked_range2d_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/blocked_range3d.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_blocked_range3d_H\n#define __TBB_blocked_range3d_H\n\n#include \n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n\n#include \"blocked_range.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! A 3-dimensional range that models the Range concept.\n/** @ingroup algorithms */\ntemplate\n __TBB_requires(blocked_range_value &&\n blocked_range_value &&\n blocked_range_value)\nclass blocked_range3d {\npublic:\n //! Type for size of an iteration range\n using page_range_type = blocked_range;\n using row_range_type = blocked_range;\n using col_range_type = blocked_range;\n\nprivate:\n page_range_type my_pages;\n row_range_type my_rows;\n col_range_type my_cols;\n\npublic:\n\n blocked_range3d( PageValue page_begin, PageValue page_end,\n RowValue row_begin, RowValue row_end,\n ColValue col_begin, ColValue col_end ) :\n my_pages(page_begin,page_end),\n my_rows(row_begin,row_end),\n my_cols(col_begin,col_end)\n {}\n\n blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize,\n RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize,\n ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) :\n my_pages(page_begin,page_end,page_grainsize),\n my_rows(row_begin,row_end,row_grainsize),\n my_cols(col_begin,col_end,col_grainsize)\n {}\n\n //! True if range is empty\n bool empty() const {\n // Range is empty if at least one dimension is empty.\n return my_pages.empty() || my_rows.empty() || my_cols.empty();\n }\n\n //! True if range is divisible into two pieces.\n bool is_divisible() const {\n return my_pages.is_divisible() || my_rows.is_divisible() || my_cols.is_divisible();\n }\n\n blocked_range3d( blocked_range3d& r, split split_obj ) :\n my_pages(r.my_pages),\n my_rows(r.my_rows),\n my_cols(r.my_cols)\n {\n do_split(r, split_obj);\n }\n\n blocked_range3d( blocked_range3d& r, proportional_split& proportion ) :\n my_pages(r.my_pages),\n my_rows(r.my_rows),\n my_cols(r.my_cols)\n {\n do_split(r, proportion);\n }\n\n //! The pages of the iteration space\n const page_range_type& pages() const { return my_pages; }\n\n //! The rows of the iteration space\n const row_range_type& rows() const { return my_rows; }\n\n //! The columns of the iteration space\n const col_range_type& cols() const { return my_cols; }\n\nprivate:\n template \n void do_split( blocked_range3d& r, Split& split_obj) {\n if ( my_pages.size()*double(my_rows.grainsize()) < my_rows.size()*double(my_pages.grainsize()) ) {\n if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) {\n my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj);\n } else {\n my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj);\n }\n } else {\n if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) {\n my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj);\n } else {\n my_pages.my_begin = page_range_type::do_split(r.my_pages, split_obj);\n }\n }\n }\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::blocked_range3d;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_blocked_range3d_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/blocked_rangeNd.h", "content": "/*\n Copyright (c) 2017-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_blocked_rangeNd_H\n#define __TBB_blocked_rangeNd_H\n\n#if !TBB_PREVIEW_BLOCKED_RANGE_ND\n #error Set TBB_PREVIEW_BLOCKED_RANGE_ND to include blocked_rangeNd.h\n#endif\n\n#include // std::any_of\n#include \n#include \n#include // std::is_same, std::enable_if\n\n#include \"detail/_config.h\"\n#include \"detail/_template_helpers.h\" // index_sequence, make_index_sequence\n#include \"detail/_range_common.h\"\n\n#include \"blocked_range.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n/*\n The blocked_rangeNd_impl uses make_index_sequence to automatically generate a ctor with\n exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity\n to use braced-init-list parameters to initialize each dimension.\n Use of parameters, whose representation is a braced-init-list, but they're not\n std::initializer_list or a reference to one, produces a non-deduced context\n within template argument deduction.\n\n NOTE: blocked_rangeNd must be exactly a templated alias to the blocked_rangeNd_impl\n (and not e.g. a derived class), otherwise it would need to declare its own ctor\n facing the same problem that the impl class solves.\n*/\n\ntemplate>\n __TBB_requires(blocked_range_value)\nclass blocked_rangeNd_impl;\n\ntemplate\n __TBB_requires(blocked_range_value)\nclass blocked_rangeNd_impl> {\npublic:\n //! Type of a value.\n using value_type = Value;\n\nprivate:\n //! Helper type to construct range with N tbb::blocked_range objects.\n template\n using dim_type_helper = tbb::blocked_range;\n\npublic:\n blocked_rangeNd_impl() = delete;\n\n //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range.\n blocked_rangeNd_impl(const dim_type_helper&... args) : my_dims{ {args...} } {}\n\n //! Dimensionality of a range.\n static constexpr unsigned int ndims() { return N; }\n\n //! Range in certain dimension.\n const tbb::blocked_range& dim(unsigned int dimension) const {\n __TBB_ASSERT(dimension < N, \"out of bound\");\n return my_dims[dimension];\n }\n\n //------------------------------------------------------------------------\n // Methods that implement Range concept\n //------------------------------------------------------------------------\n\n //! True if at least one dimension is empty.\n bool empty() const {\n return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) {\n return d.empty();\n });\n }\n\n //! True if at least one dimension is divisible.\n bool is_divisible() const {\n return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) {\n return d.is_divisible();\n });\n }\n\n blocked_rangeNd_impl(blocked_rangeNd_impl& r, proportional_split proportion) : my_dims(r.my_dims) {\n do_split(r, proportion);\n }\n\n blocked_rangeNd_impl(blocked_rangeNd_impl& r, split proportion) : my_dims(r.my_dims) {\n do_split(r, proportion);\n }\n\nprivate:\n static_assert(N != 0, \"zero dimensional blocked_rangeNd can't be constructed\");\n\n //! Ranges in each dimension.\n std::array, N> my_dims;\n\n template\n void do_split(blocked_rangeNd_impl& r, split_type proportion) {\n static_assert((std::is_same::value || std::is_same::value), \"type of split object is incorrect\");\n __TBB_ASSERT(r.is_divisible(), \"can't split not divisible range\");\n\n auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& first, const tbb::blocked_range& second) {\n return (first.size() * second.grainsize() < second.size() * first.grainsize());\n });\n\n auto r_it = r.my_dims.begin() + (my_it - my_dims.begin());\n\n my_it->my_begin = tbb::blocked_range::do_split(*r_it, proportion);\n\n // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to\n // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept\n __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin),\n \"blocked_range has been split incorrectly\");\n }\n};\n\ntemplate\nusing blocked_rangeNd = blocked_rangeNd_impl;\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::blocked_rangeNd;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_blocked_rangeNd_H */\n\n" }, { "path": "src/tbb/include/oneapi/tbb/cache_aligned_allocator.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_cache_aligned_allocator_H\n#define __TBB_cache_aligned_allocator_H\n\n#include \"detail/_utils.h\"\n#include \"detail/_namespace_injection.h\"\n#include \n#include \n\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nTBB_EXPORT void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size);\nTBB_EXPORT void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p);\nTBB_EXPORT std::size_t __TBB_EXPORTED_FUNC cache_line_size();\n}\n\nnamespace d1 {\n\ntemplate\nclass cache_aligned_allocator {\npublic:\n using value_type = T;\n using propagate_on_container_move_assignment = std::true_type;\n\n //! Always defined for TBB containers (supported since C++17 for std containers)\n using is_always_equal = std::true_type;\n\n cache_aligned_allocator() = default;\n template cache_aligned_allocator(const cache_aligned_allocator&) noexcept {}\n\n //! Allocate space for n objects, starting on a cache/sector line.\n __TBB_nodiscard T* allocate(std::size_t n) {\n return static_cast(r1::cache_aligned_allocate(n * sizeof(value_type)));\n }\n\n //! Free block of memory that starts on a cache line\n void deallocate(T* p, std::size_t) {\n r1::cache_aligned_deallocate(p);\n }\n\n //! Largest value for which method allocate might succeed.\n std::size_t max_size() const noexcept {\n return (~std::size_t(0) - r1::cache_line_size()) / sizeof(value_type);\n }\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n using pointer = value_type*;\n using const_pointer = const value_type*;\n using reference = value_type&;\n using const_reference = const value_type&;\n using difference_type = std::ptrdiff_t;\n using size_type = std::size_t;\n template struct rebind {\n using other = cache_aligned_allocator;\n };\n template\n void construct(U *p, Args&&... args)\n { ::new (p) U(std::forward(args)...); }\n void destroy(pointer p) { p->~value_type(); }\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n#endif // TBB_ALLOCATOR_TRAITS_BROKEN\n};\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n template<>\n class cache_aligned_allocator {\n public:\n using pointer = void*;\n using const_pointer = const void*;\n using value_type = void;\n template struct rebind {\n using other = cache_aligned_allocator;\n };\n };\n#endif\n\ntemplate\nbool operator==(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return true; }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate\nbool operator!=(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return false; }\n#endif\n\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n\n//! C++17 memory resource wrapper to ensure cache line size alignment\nclass cache_aligned_resource : public std::pmr::memory_resource {\npublic:\n cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {}\n explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {}\n\n std::pmr::memory_resource* upstream_resource() const {\n return m_upstream;\n }\n\nprivate:\n //! We don't know what memory resource set. Use padding to guarantee alignment\n void* do_allocate(std::size_t bytes, std::size_t alignment) override {\n // TODO: make it common with tbb_allocator.cpp\n std::size_t cache_line_alignment = correct_alignment(alignment);\n std::size_t space = correct_size(bytes) + cache_line_alignment;\n std::uintptr_t base = reinterpret_cast(m_upstream->allocate(space));\n __TBB_ASSERT(base != 0, \"Upstream resource returned nullptr.\");\n\n // Round up to the next cache line (align the base address)\n std::uintptr_t result = (base + cache_line_alignment) & ~(cache_line_alignment - 1);\n __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), \"Can`t store a base pointer to the header\");\n __TBB_ASSERT(space - (result - base) >= bytes, \"Not enough space for the storage\");\n\n // Record where block actually starts.\n (reinterpret_cast(result))[-1] = base;\n return reinterpret_cast(result);\n }\n\n void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override {\n if (ptr) {\n // Recover where block actually starts\n std::uintptr_t base = (reinterpret_cast(ptr))[-1];\n m_upstream->deallocate(reinterpret_cast(base), correct_size(bytes) + correct_alignment(alignment));\n }\n }\n\n bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {\n if (this == &other) { return true; }\n#if __TBB_USE_OPTIONAL_RTTI\n const cache_aligned_resource* other_res = dynamic_cast(&other);\n return other_res && (upstream_resource() == other_res->upstream_resource());\n#else\n return false;\n#endif\n }\n\n std::size_t correct_alignment(std::size_t alignment) {\n __TBB_ASSERT(tbb::detail::is_power_of_two(alignment), \"Alignment is not a power of 2\");\n#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT\n std::size_t cache_line_size = std::hardware_destructive_interference_size;\n#else\n std::size_t cache_line_size = r1::cache_line_size();\n#endif\n return alignment < cache_line_size ? cache_line_size : alignment;\n }\n\n std::size_t correct_size(std::size_t bytes) {\n // To handle the case, when small size requested. There could be not\n // enough space to store the original pointer.\n return bytes < sizeof(std::uintptr_t) ? sizeof(std::uintptr_t) : bytes;\n }\n\n std::pmr::memory_resource* m_upstream;\n};\n\n#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::cache_aligned_allocator;\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\nusing detail::d1::cache_aligned_resource;\n#endif\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_cache_aligned_allocator_H */\n\n" }, { "path": "src/tbb/include/oneapi/tbb/collaborative_call_once.h", "content": "/*\n Copyright (c) 2021-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_collaborative_call_once_H\n#define __TBB_collaborative_call_once_H\n\n#include \"task_arena.h\"\n#include \"task_group.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Suppress warning: structure was padded due to alignment specifier\n // #pragma warning (push)\n // #pragma warning (disable: 4324)\n#endif\n\ntemplate \nclass collaborative_call_stack_task : public task {\n const F& m_func;\n wait_context& m_wait_ctx;\n\n void finalize() {\n m_wait_ctx.release();\n }\n task* execute(d1::execution_data&) override {\n task* res = d2::task_ptr_or_nullptr(m_func);\n finalize();\n return res;\n }\n task* cancel(d1::execution_data&) override {\n finalize();\n return nullptr;\n }\npublic:\n collaborative_call_stack_task(const F& f, wait_context& wctx) : m_func(f), m_wait_ctx(wctx) {}\n};\n\nconstexpr std::uintptr_t collaborative_once_max_references = max_nfs_size;\nconstexpr std::uintptr_t collaborative_once_references_mask = collaborative_once_max_references-1;\n\nclass alignas(max_nfs_size) collaborative_once_runner : no_copy {\n\n struct storage_t {\n task_arena m_arena{ task_arena::attach{} };\n wait_context m_wait_context{1};\n };\n\n std::atomic m_ref_count{0};\n std::atomic m_is_ready{false};\n\n // Storage with task_arena and wait_context must be initialized only by winner thread\n union {\n storage_t m_storage;\n };\n\n template\n void isolated_execute(Fn f) {\n auto func = [f] {\n f();\n // delegate_base requires bool returning functor while isolate_within_arena ignores the result\n return true;\n };\n\n delegated_function delegate(func);\n\n r1::isolate_within_arena(delegate, reinterpret_cast(this));\n }\n\npublic:\n class lifetime_guard : no_copy {\n collaborative_once_runner& m_runner;\n public:\n lifetime_guard(collaborative_once_runner& r) : m_runner(r) {\n m_runner.m_ref_count++;\n }\n ~lifetime_guard() {\n m_runner.m_ref_count--;\n }\n };\n\n collaborative_once_runner() {}\n\n ~collaborative_once_runner() {\n spin_wait_until_eq(m_ref_count, 0, std::memory_order_acquire);\n if (m_is_ready.load(std::memory_order_relaxed)) {\n m_storage.~storage_t();\n }\n }\n\n std::uintptr_t to_bits() {\n return reinterpret_cast(this);\n }\n\n static collaborative_once_runner* from_bits(std::uintptr_t bits) {\n __TBB_ASSERT( (bits & collaborative_once_references_mask) == 0, \"invalid pointer, last log2(max_nfs_size) bits must be zero\" );\n return reinterpret_cast(bits);\n }\n\n template \n void run_once(F&& f) {\n __TBB_ASSERT(!m_is_ready.load(std::memory_order_relaxed), \"storage with task_arena and wait_context is already initialized\");\n // Initialize internal state\n new(&m_storage) storage_t();\n m_storage.m_arena.execute([&] {\n isolated_execute([&] {\n task_group_context context{ task_group_context::bound,\n task_group_context::default_traits | task_group_context::concurrent_wait };\n\n collaborative_call_stack_task t{ std::forward(f), m_storage.m_wait_context };\n\n // Set the ready flag after entering the execute body to prevent\n // moonlighting threads from occupying all slots inside the arena.\n m_is_ready.store(true, std::memory_order_release);\n execute_and_wait(t, context, m_storage.m_wait_context, context);\n });\n });\n }\n\n void assist() noexcept {\n // Do not join the arena until the winner thread takes the slot\n spin_wait_while_eq(m_is_ready, false);\n m_storage.m_arena.execute([&] {\n isolated_execute([&] {\n // We do not want to get an exception from user functor on moonlighting threads.\n // The exception is handled with the winner thread\n task_group_context stub_context;\n wait(m_storage.m_wait_context, stub_context);\n });\n });\n }\n\n};\n\nclass collaborative_once_flag : no_copy {\n enum state : std::uintptr_t {\n uninitialized,\n done,\n#if TBB_USE_ASSERT\n dead\n#endif\n };\n std::atomic m_state{ state::uninitialized };\n\n template \n friend void collaborative_call_once(collaborative_once_flag& flag, Fn&& f, Args&&... args);\n\n void set_completion_state(std::uintptr_t runner_bits, std::uintptr_t desired) {\n std::uintptr_t expected = runner_bits;\n do {\n expected = runner_bits;\n // Possible inefficiency: when we start waiting,\n // some moonlighting threads might continue coming that will prolong our waiting.\n // Fortunately, there are limited number of threads on the system so wait time is limited.\n spin_wait_until_eq(m_state, expected);\n } while (!m_state.compare_exchange_strong(expected, desired));\n }\n\n template \n void do_collaborative_call_once(Fn&& f) {\n std::uintptr_t expected = m_state.load(std::memory_order_acquire);\n collaborative_once_runner runner;\n\n do {\n if (expected == state::uninitialized && m_state.compare_exchange_strong(expected, runner.to_bits())) {\n // Winner thread\n runner.run_once([&] {\n try_call([&] {\n std::forward(f)();\n }).on_exception([&] {\n // Reset the state to uninitialized to allow other threads to try initialization again\n set_completion_state(runner.to_bits(), state::uninitialized);\n });\n // We successfully executed functor\n set_completion_state(runner.to_bits(), state::done);\n });\n break;\n } else {\n // Moonlighting thread: we need to add a reference to the state to prolong runner lifetime.\n // However, the maximum number of references are limited with runner alignment.\n // So, we use CAS loop and spin_wait to guarantee that references never exceed \"max_value\".\n do {\n auto max_value = expected | collaborative_once_references_mask;\n expected = spin_wait_while_eq(m_state, max_value);\n // \"expected > state::done\" prevents storing values, when state is uninitialized or done\n } while (expected > state::done && !m_state.compare_exchange_strong(expected, expected + 1));\n\n if (auto shared_runner = collaborative_once_runner::from_bits(expected & ~collaborative_once_references_mask)) {\n collaborative_once_runner::lifetime_guard guard{*shared_runner};\n m_state.fetch_sub(1);\n\n // The moonlighting threads are not expected to handle exceptions from user functor.\n // Therefore, no exception is expected from assist().\n shared_runner->assist();\n }\n }\n __TBB_ASSERT(m_state.load(std::memory_order_relaxed) != state::dead,\n \"collaborative_once_flag has been prematurely destroyed\");\n } while (expected != state::done);\n }\n\n#if TBB_USE_ASSERT\npublic:\n ~collaborative_once_flag() {\n m_state.store(state::dead, std::memory_order_relaxed);\n }\n#endif\n};\n\n\ntemplate \nvoid collaborative_call_once(collaborative_once_flag& flag, Fn&& fn, Args&&... args) {\n __TBB_ASSERT(flag.m_state.load(std::memory_order_relaxed) != collaborative_once_flag::dead,\n \"collaborative_once_flag has been prematurely destroyed\");\n if (flag.m_state.load(std::memory_order_acquire) != collaborative_once_flag::done) {\n #if __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN\n // Using stored_pack to suppress bug in GCC 4.8\n // with parameter pack expansion in lambda\n auto stored_pack = save_pack(std::forward(args)...);\n auto func = [&] { call(std::forward(fn), std::move(stored_pack)); };\n #else\n auto func = [&] { fn(std::forward(args)...); };\n #endif\n flag.do_collaborative_call_once(func);\n }\n}\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning (pop) // 4324 warning\n#endif\n\n} // namespace d1\n} // namespace detail\n\nusing detail::d1::collaborative_call_once;\nusing detail::d1::collaborative_once_flag;\n} // namespace tbb\n\n#endif // __TBB_collaborative_call_once_H\n" }, { "path": "src/tbb/include/oneapi/tbb/combinable.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_combinable_H\n#define __TBB_combinable_H\n\n#include \"detail/_namespace_injection.h\"\n\n#include \"enumerable_thread_specific.h\"\n#include \"cache_aligned_allocator.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n/** \\name combinable **/\n//@{\n//! Thread-local storage with optional reduction\n/** @ingroup containers */\ntemplate \nclass combinable {\n using my_alloc = typename tbb::cache_aligned_allocator;\n using my_ets_type = typename tbb::enumerable_thread_specific;\n my_ets_type my_ets;\n\npublic:\n combinable() = default;\n\n template \n explicit combinable(Finit _finit) : my_ets(_finit) { }\n\n void clear() { my_ets.clear(); }\n\n T& local() { return my_ets.local(); }\n\n T& local(bool& exists) { return my_ets.local(exists); }\n\n // combine_func_t has signature T(T,T) or T(const T&, const T&)\n template \n T combine(CombineFunc f_combine) { return my_ets.combine(f_combine); }\n\n // combine_func_t has signature void(T) or void(const T&)\n template \n void combine_each(CombineFunc f_combine) { my_ets.combine_each(f_combine); }\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::combinable;\n} // inline namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_combinable_H */\n\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_hash_map.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_hash_map_H\n#define __TBB_concurrent_hash_map_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_allocator_traits.h\"\n#include \"detail/_containers_helpers.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_hash_compare.h\"\n#include \"detail/_range_common.h\"\n#include \"tbb_allocator.h\"\n#include \"spin_rw_mutex.h\"\n\n#include \n#include \n#include \n#include \n#include // Need std::pair\n#include // Need std::memset\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS && __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \nconcept ch_map_rw_scoped_lockable = rw_scoped_lockable &&\n\trequires(const typename Mutex::scoped_lock& sl) {\n\t\t{ sl.is_writer() } -> std::convertible_to;\n};\n#endif\n\ntemplate \nstruct hash_map_node_base : no_copy {\n using mutex_type = MutexType;\n // Scoped lock type for mutex\n using scoped_type = typename MutexType::scoped_lock;\n // Next node in chain\n hash_map_node_base* next;\n mutex_type mutex;\n};\n\n// Incompleteness flag value\nstatic void* const rehash_req_flag = reinterpret_cast(std::size_t(3));\n// Rehashed empty bucket flag\nstatic void* const empty_rehashed_flag = reinterpret_cast(std::size_t(0));\n\ntemplate \nbool rehash_required( hash_map_node_base* node_ptr ) {\n return reinterpret_cast(node_ptr) == rehash_req_flag;\n}\n\n#if TBB_USE_ASSERT\ntemplate \nbool empty_rehashed( hash_map_node_base* node_ptr ) {\n return reinterpret_cast(node_ptr) == empty_rehashed_flag;\n}\n#endif\n\n// base class of concurrent_hash_map\n\ntemplate \nclass hash_map_base {\npublic:\n using size_type = std::size_t;\n using hashcode_type = std::size_t;\n using segment_index_type = std::size_t;\n using node_base = hash_map_node_base;\n\n struct bucket : no_copy {\n using mutex_type = MutexType;\n using scoped_type = typename mutex_type::scoped_lock;\n\n bucket() : node_list(nullptr) {}\n bucket( node_base* ptr ) : node_list(ptr) {}\n\n mutex_type mutex;\n std::atomic node_list;\n };\n\n using allocator_type = Allocator;\n using allocator_traits_type = tbb::detail::allocator_traits;\n using bucket_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using bucket_allocator_traits = tbb::detail::allocator_traits;\n\n // Count of segments in the first block\n static constexpr size_type embedded_block = 1;\n // Count of segments in the first block\n static constexpr size_type embedded_buckets = 1 << embedded_block;\n // Count of segments in the first block\n static constexpr size_type first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096\n // Size of a pointer / table size\n static constexpr size_type pointers_per_table = sizeof(segment_index_type) * 8; // one segment per bit\n\n using segment_ptr_type = bucket*;\n using atomic_segment_type = std::atomic;\n using segments_table_type = atomic_segment_type[pointers_per_table];\n\n hash_map_base( const allocator_type& alloc ) : my_allocator(alloc), my_mask(embedded_buckets - 1), my_size(0) {\n for (size_type i = 0; i != embedded_buckets; ++i) {\n my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed);\n }\n\n for (size_type segment_index = 0; segment_index < pointers_per_table; ++segment_index) {\n auto argument = segment_index < embedded_block ? my_embedded_segment + segment_base(segment_index) : nullptr;\n my_table[segment_index].store(argument, std::memory_order_relaxed);\n }\n\n __TBB_ASSERT( embedded_block <= first_block, \"The first block number must include embedded blocks\");\n }\n\n // segment index of given index in the array\n static segment_index_type segment_index_of( size_type index ) {\n return segment_index_type(tbb::detail::log2( index|1 ));\n }\n\n // the first array index of given segment\n static segment_index_type segment_base( segment_index_type k ) {\n return (segment_index_type(1) << k & ~segment_index_type(1));\n }\n\n // segment size except for k == 0\n static size_type segment_size( segment_index_type k ) {\n return size_type(1) << k; // fake value for k==0\n }\n\n // true if ptr is valid pointer\n static bool is_valid( void* ptr ) {\n return reinterpret_cast(ptr) > uintptr_t(63);\n }\n\n template \n void init_buckets_impl( segment_ptr_type ptr, size_type sz, const Args&... args ) {\n for (size_type i = 0; i < sz; ++i) {\n bucket_allocator_traits::construct(my_allocator, ptr + i, args...);\n }\n }\n\n // Initialize buckets\n void init_buckets( segment_ptr_type ptr, size_type sz, bool is_initial ) {\n if (is_initial) {\n init_buckets_impl(ptr, sz);\n } else {\n init_buckets_impl(ptr, sz, reinterpret_cast(rehash_req_flag));\n }\n }\n\n // Add node n to bucket b\n static void add_to_bucket( bucket* b, node_base* n ) {\n __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr);\n n->next = b->node_list.load(std::memory_order_relaxed);\n b->node_list.store(n, std::memory_order_relaxed); // its under lock and flag is set\n }\n\n const bucket_allocator_type& get_allocator() const {\n return my_allocator;\n }\n\n bucket_allocator_type& get_allocator() {\n return my_allocator;\n }\n\n // Enable segment\n void enable_segment( segment_index_type k, bool is_initial = false ) {\n __TBB_ASSERT( k, \"Zero segment must be embedded\" );\n size_type sz;\n __TBB_ASSERT( !is_valid(my_table[k].load(std::memory_order_relaxed)), \"Wrong concurrent assignment\");\n if (k >= first_block) {\n sz = segment_size(k);\n segment_ptr_type ptr = nullptr;\n try_call( [&] {\n ptr = bucket_allocator_traits::allocate(my_allocator, sz);\n } ).on_exception( [&] {\n my_table[k].store(nullptr, std::memory_order_relaxed);\n });\n\n __TBB_ASSERT(ptr, nullptr);\n init_buckets(ptr, sz, is_initial);\n my_table[k].store(ptr, std::memory_order_release);\n sz <<= 1;// double it to get entire capacity of the container\n } else { // the first block\n __TBB_ASSERT( k == embedded_block, \"Wrong segment index\" );\n sz = segment_size(first_block);\n segment_ptr_type ptr = nullptr;\n try_call( [&] {\n ptr = bucket_allocator_traits::allocate(my_allocator, sz - embedded_buckets);\n } ).on_exception( [&] {\n my_table[k].store(nullptr, std::memory_order_relaxed);\n });\n\n __TBB_ASSERT(ptr, nullptr);\n init_buckets(ptr, sz - embedded_buckets, is_initial);\n ptr -= segment_base(embedded_block);\n for(segment_index_type i = embedded_block; i < first_block; i++) // calc the offsets\n my_table[i].store(ptr + segment_base(i), std::memory_order_release);\n }\n my_mask.store(sz-1, std::memory_order_release);\n }\n\n void delete_segment( segment_index_type s ) {\n segment_ptr_type buckets_ptr = my_table[s].load(std::memory_order_relaxed);\n size_type sz = segment_size( s ? s : 1 );\n\n size_type deallocate_size = 0;\n\n if (s >= first_block) { // the first segment or the next\n deallocate_size = sz;\n } else if (s == embedded_block && embedded_block != first_block) {\n deallocate_size = segment_size(first_block) - embedded_buckets;\n }\n\n for (size_type i = 0; i < deallocate_size; ++i) {\n bucket_allocator_traits::destroy(my_allocator, buckets_ptr + i);\n }\n if (deallocate_size != 0) {\n bucket_allocator_traits::deallocate(my_allocator, buckets_ptr, deallocate_size);\n }\n\n if (s >= embedded_block) my_table[s].store(nullptr, std::memory_order_relaxed);\n }\n\n // Get bucket by (masked) hashcode\n bucket *get_bucket( hashcode_type h ) const noexcept {\n segment_index_type s = segment_index_of( h );\n h -= segment_base(s);\n segment_ptr_type seg = my_table[s].load(std::memory_order_acquire);\n __TBB_ASSERT( is_valid(seg), \"hashcode must be cut by valid mask for allocated segments\" );\n return &seg[h];\n }\n\n // detail serial rehashing helper\n void mark_rehashed_levels( hashcode_type h ) noexcept {\n segment_index_type s = segment_index_of( h );\n while (segment_ptr_type seg = my_table[++s].load(std::memory_order_relaxed))\n if (rehash_required(seg[h].node_list.load(std::memory_order_relaxed))) {\n seg[h].node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_relaxed);\n mark_rehashed_levels( h + ((hashcode_type)1<node_list.load(std::memory_order_acquire))) {\n return true;\n }\n }\n return false;\n }\n\n // Insert a node and check for load factor. @return segment index to enable.\n segment_index_type insert_new_node( bucket *b, node_base *n, hashcode_type mask ) {\n size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted\n add_to_bucket( b, n );\n // check load factor\n if( sz >= mask ) { // TODO: add custom load_factor\n segment_index_type new_seg = tbb::detail::log2( mask+1 ); //optimized segment_index_of\n __TBB_ASSERT( is_valid(my_table[new_seg-1].load(std::memory_order_relaxed)), \"new allocations must not publish new mask until segment has allocated\");\n static const segment_ptr_type is_allocating = segment_ptr_type(2);\n segment_ptr_type disabled = nullptr;\n if (!(my_table[new_seg].load(std::memory_order_acquire))\n && my_table[new_seg].compare_exchange_strong(disabled, is_allocating))\n return new_seg; // The value must be processed\n }\n return 0;\n }\n\n // Prepare enough segments for number of buckets\n void reserve(size_type buckets) {\n if( !buckets-- ) return;\n bool is_initial = !my_size.load(std::memory_order_relaxed);\n for (size_type m = my_mask.load(std::memory_order_relaxed); buckets > m;\n m = my_mask.load(std::memory_order_relaxed))\n {\n enable_segment( segment_index_of( m+1 ), is_initial );\n }\n }\n\n // Swap hash_map_bases\n void internal_swap_content(hash_map_base &table) {\n using std::swap;\n swap_atomics_relaxed(my_mask, table.my_mask);\n swap_atomics_relaxed(my_size, table.my_size);\n\n for(size_type i = 0; i < embedded_buckets; i++) {\n auto temp = my_embedded_segment[i].node_list.load(std::memory_order_relaxed);\n my_embedded_segment[i].node_list.store(table.my_embedded_segment[i].node_list.load(std::memory_order_relaxed),\n std::memory_order_relaxed);\n table.my_embedded_segment[i].node_list.store(temp, std::memory_order_relaxed);\n }\n for(size_type i = embedded_block; i < pointers_per_table; i++) {\n auto temp = my_table[i].load(std::memory_order_relaxed);\n my_table[i].store(table.my_table[i].load(std::memory_order_relaxed),\n std::memory_order_relaxed);\n table.my_table[i].store(temp, std::memory_order_relaxed);\n }\n }\n\n void internal_move(hash_map_base&& other) {\n my_mask.store(other.my_mask.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_mask.store(embedded_buckets - 1, std::memory_order_relaxed);\n\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_size.store(0, std::memory_order_relaxed);\n\n for (size_type i = 0; i < embedded_buckets; ++i) {\n my_embedded_segment[i].node_list.store(other.my_embedded_segment[i].node_list, std::memory_order_relaxed);\n other.my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed);\n }\n\n for (size_type i = embedded_block; i < pointers_per_table; ++i) {\n my_table[i].store(other.my_table[i].load(std::memory_order_relaxed),\n std::memory_order_relaxed);\n other.my_table[i].store(nullptr, std::memory_order_relaxed);\n }\n }\n\nprotected:\n bucket_allocator_type my_allocator;\n // Hash mask = sum of allocated segment sizes - 1\n std::atomic my_mask;\n // Size of container in stored items\n std::atomic my_size; // It must be in separate cache line from my_mask due to performance effects\n // Zero segment\n bucket my_embedded_segment[embedded_buckets];\n // Segment pointers table. Also prevents false sharing between my_mask and my_size\n segments_table_type my_table;\n};\n\ntemplate \nclass hash_map_range;\n\n// Meets requirements of a forward iterator for STL\n// Value is either the T or const T type of the container.\ntemplate \nclass hash_map_iterator {\n using map_type = Container;\n using node = typename Container::node;\n using map_base = typename Container::base_type;\n using node_base = typename map_base::node_base;\n using bucket = typename map_base::bucket;\npublic:\n using value_type = Value;\n using size_type = typename Container::size_type;\n using difference_type = typename Container::difference_type;\n using pointer = value_type*;\n using reference = value_type&;\n using iterator_category = std::forward_iterator_tag;\n\n // Construct undefined iterator\n hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {}\n hash_map_iterator( const hash_map_iterator& other ) :\n my_map(other.my_map),\n my_index(other.my_index),\n my_bucket(other.my_bucket),\n my_node(other.my_node)\n {}\n\n hash_map_iterator& operator=( const hash_map_iterator& other ) {\n my_map = other.my_map;\n my_index = other.my_index;\n my_bucket = other.my_bucket;\n my_node = other.my_node;\n return *this;\n }\n\n Value& operator*() const {\n __TBB_ASSERT( map_base::is_valid(my_node), \"iterator uninitialized or at end of container?\" );\n return my_node->value();\n }\n\n Value* operator->() const {return &operator*();}\n\n hash_map_iterator& operator++() {\n my_node = static_cast( my_node->next );\n if( !my_node ) advance_to_next_bucket();\n return *this;\n }\n\n // Post increment\n hash_map_iterator operator++(int) {\n hash_map_iterator old(*this);\n operator++();\n return old;\n }\nprivate:\n template \n friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j );\n\n template \n friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j );\n\n template \n friend ptrdiff_t operator-( const hash_map_iterator& i, const hash_map_iterator& j );\n\n template \n friend class hash_map_iterator;\n\n template \n friend class hash_map_range;\n\n void advance_to_next_bucket() { // TODO?: refactor to iterator_base class\n size_t k = my_index+1;\n __TBB_ASSERT( my_bucket, \"advancing an invalid iterator?\");\n while (k <= my_map->my_mask.load(std::memory_order_relaxed)) {\n // Following test uses 2's-complement wizardry\n if( k&(k-2) ) // not the beginning of a segment\n ++my_bucket;\n else my_bucket = my_map->get_bucket( k );\n node_base *n = my_bucket->node_list.load(std::memory_order_relaxed);\n if( map_base::is_valid(n) ) {\n my_node = static_cast(n);\n my_index = k;\n return;\n }\n ++k;\n }\n my_bucket = nullptr; my_node = nullptr; my_index = k; // the end\n }\n\n template \n __TBB_requires(tbb::detail::hash_compare &&\n ch_map_rw_scoped_lockable)\n#else\n >\n __TBB_requires(tbb::detail::hash_compare)\n#endif\n friend class concurrent_hash_map;\n\n hash_map_iterator( const Container &map, std::size_t index, const bucket *b, node_base *n ) :\n my_map(&map), my_index(index), my_bucket(b), my_node(static_cast(n))\n {\n if( b && !map_base::is_valid(n) )\n advance_to_next_bucket();\n }\n\n // concurrent_hash_map over which we are iterating.\n const Container *my_map;\n // Index in hash table for current item\n size_t my_index;\n // Pointer to bucket\n const bucket* my_bucket;\n // Pointer to node that has current item\n node* my_node;\n};\n\ntemplate \nbool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) {\n return i.my_node == j.my_node && i.my_map == j.my_map;\n}\n\ntemplate \nbool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) {\n return i.my_node != j.my_node || i.my_map != j.my_map;\n}\n\n// Range class used with concurrent_hash_map\ntemplate \nclass hash_map_range {\n using map_type = typename Iterator::map_type;\npublic:\n // Type for size of a range\n using size_type = std::size_t;\n using value_type = typename Iterator::value_type;\n using reference = typename Iterator::reference;\n using difference_type = typename Iterator::difference_type;\n using iterator = Iterator;\n\n // True if range is empty.\n bool empty() const { return my_begin == my_end; }\n\n // True if range can be partitioned into two subranges.\n bool is_divisible() const {\n return my_midpoint != my_end;\n }\n\n // Split range.\n hash_map_range( hash_map_range& r, split ) :\n my_end(r.my_end),\n my_grainsize(r.my_grainsize)\n {\n r.my_end = my_begin = r.my_midpoint;\n __TBB_ASSERT( !empty(), \"Splitting despite the range is not divisible\" );\n __TBB_ASSERT( !r.empty(), \"Splitting despite the range is not divisible\" );\n set_midpoint();\n r.set_midpoint();\n }\n\n // Init range with container and grainsize specified\n hash_map_range( const map_type &map, size_type grainsize_ = 1 ) :\n my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list.load(std::memory_order_relaxed) ) ),\n my_end( Iterator( map, map.my_mask.load(std::memory_order_relaxed) + 1, nullptr, nullptr ) ),\n my_grainsize( grainsize_ )\n {\n __TBB_ASSERT( grainsize_>0, \"grainsize must be positive\" );\n set_midpoint();\n }\n\n Iterator begin() const { return my_begin; }\n Iterator end() const { return my_end; }\n // The grain size for this range.\n size_type grainsize() const { return my_grainsize; }\n\nprivate:\n Iterator my_begin;\n Iterator my_end;\n mutable Iterator my_midpoint;\n size_t my_grainsize;\n // Set my_midpoint to point approximately half way between my_begin and my_end.\n void set_midpoint() const;\n template friend class hash_map_range;\n};\n\ntemplate \nvoid hash_map_range::set_midpoint() const {\n // Split by groups of nodes\n size_t m = my_end.my_index-my_begin.my_index;\n if( m > my_grainsize ) {\n m = my_begin.my_index + m/2u;\n auto b = my_begin.my_map->get_bucket(m);\n my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list.load(std::memory_order_relaxed));\n } else {\n my_midpoint = my_end;\n }\n __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index,\n \"my_begin is after my_midpoint\" );\n __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index,\n \"my_midpoint is after my_end\" );\n __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end,\n \"[my_begin, my_midpoint) range should not be empty\" );\n}\n\ntemplate ,\n typename Allocator = tbb_allocator>\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n , typename MutexType = spin_rw_mutex\n >\n __TBB_requires(tbb::detail::hash_compare &&\n ch_map_rw_scoped_lockable)\n#else\n >\n __TBB_requires(tbb::detail::hash_compare)\n#endif\nclass concurrent_hash_map\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n : protected hash_map_base\n#else\n : protected hash_map_base\n#endif\n{\n template \n friend class hash_map_iterator;\n\n template \n friend class hash_map_range;\n using allocator_traits_type = tbb::detail::allocator_traits;\n\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n using base_type = hash_map_base;\n#else\n using base_type = hash_map_base;\n#endif\npublic:\n using key_type = Key;\n using mapped_type = T;\n // type_identity is needed to disable implicit deduction guides for std::initializer_list constructors\n // and copy/move constructor with explicit allocator argument\n using allocator_type = tbb::detail::type_identity_t;\n using hash_compare_type = tbb::detail::type_identity_t;\n using value_type = std::pair;\n using size_type = typename base_type::size_type;\n using difference_type = std::ptrdiff_t;\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n using mutex_type = MutexType;\n#endif\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using reference = value_type&;\n using const_reference = const value_type&;\n using iterator = hash_map_iterator;\n using const_iterator = hash_map_iterator;\n using range_type = hash_map_range;\n using const_range_type = hash_map_range;\n\nprotected:\n static_assert(std::is_same::value,\n \"value_type of the container must be the same as its allocator's\");\n\n friend class const_accessor;\n class node;\n using segment_index_type = typename base_type::segment_index_type;\n using segment_ptr_type = typename base_type::segment_ptr_type;\n using node_base = typename base_type::node_base;\n using bucket = typename base_type::bucket;\n using hashcode_type = typename base_type::hashcode_type;\n using bucket_allocator_type = typename base_type::bucket_allocator_type;\n using node_allocator_type = typename base_type::allocator_traits_type::template rebind_alloc;\n using node_allocator_traits = tbb::detail::allocator_traits;\n hash_compare_type my_hash_compare;\n\n class node : public node_base {\n public:\n node() {}\n ~node() {}\n pointer storage() { return &my_value; }\n value_type& value() { return *storage(); }\n private:\n union {\n value_type my_value;\n };\n };\n\n void delete_node( node_base *n ) {\n node_allocator_type node_allocator(this->get_allocator());\n node_allocator_traits::destroy(node_allocator, static_cast(n)->storage());\n node_allocator_traits::destroy(node_allocator, static_cast(n));\n node_allocator_traits::deallocate(node_allocator, static_cast(n), 1);\n }\n\n template \n static node* create_node(bucket_allocator_type& allocator, Args&&... args) {\n node_allocator_type node_allocator(allocator);\n node* node_ptr = node_allocator_traits::allocate(node_allocator, 1);\n auto guard = make_raii_guard([&] {\n node_allocator_traits::destroy(node_allocator, node_ptr);\n node_allocator_traits::deallocate(node_allocator, node_ptr, 1);\n });\n\n node_allocator_traits::construct(node_allocator, node_ptr);\n node_allocator_traits::construct(node_allocator, node_ptr->storage(), std::forward(args)...);\n guard.dismiss();\n return node_ptr;\n }\n\n static node* allocate_node_copy_construct(bucket_allocator_type& allocator, const Key &key, const T * t){\n return create_node(allocator, key, *t);\n }\n\n static node* allocate_node_move_construct(bucket_allocator_type& allocator, const Key &key, const T * t){\n return create_node(allocator, key, std::move(*const_cast(t)));\n }\n\n template \n static node* allocate_node_default_construct(bucket_allocator_type& allocator, const K &key, const T * ){\n // Emplace construct an empty T object inside the pair\n return create_node(allocator, std::piecewise_construct,\n std::forward_as_tuple(key), std::forward_as_tuple());\n }\n\n static node* do_not_allocate_node(bucket_allocator_type& , const Key &, const T * ){\n __TBB_ASSERT(false,\"this dummy function should not be called\");\n return nullptr;\n }\n\n template \n node *search_bucket( const K &key, bucket *b ) const {\n node *n = static_cast( b->node_list.load(std::memory_order_relaxed) );\n while (this->is_valid(n) && !my_hash_compare.equal(key, n->value().first))\n n = static_cast( n->next );\n __TBB_ASSERT(!rehash_required(n), \"Search can be executed only for rehashed bucket\");\n return n;\n }\n\n // bucket accessor is to find, rehash, acquire a lock, and access a bucket\n class bucket_accessor : public bucket::scoped_type {\n bucket *my_b;\n public:\n bucket_accessor( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) { acquire( base, h, writer ); }\n // find a bucket by masked hashcode, optionally rehash, and acquire the lock\n inline void acquire( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) {\n my_b = base->get_bucket( h );\n // TODO: actually, notification is unnecessary here, just hiding double-check\n if (rehash_required(my_b->node_list.load(std::memory_order_acquire))\n && bucket::scoped_type::try_acquire( my_b->mutex, /*write=*/true ) )\n {\n if (rehash_required(my_b->node_list.load(std::memory_order_relaxed))) base->rehash_bucket(my_b, h); // recursive rehashing\n }\n else bucket::scoped_type::acquire( my_b->mutex, writer );\n __TBB_ASSERT(!rehash_required(my_b->node_list.load(std::memory_order_relaxed)), nullptr);\n }\n\n // get bucket pointer\n bucket *operator() () { return my_b; }\n };\n\n // TODO refactor to hash_base\n void rehash_bucket( bucket *b_new, const hashcode_type hash ) {\n __TBB_ASSERT( hash > 1, \"The lowermost buckets can't be rehashed\" );\n b_new->node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_release); // mark rehashed\n hashcode_type mask = (hashcode_type(1) << tbb::detail::log2(hash)) - 1; // get parent mask from the topmost bit\n bucket_accessor b_old( this, hash & mask );\n\n mask = (mask<<1) | 1; // get full mask for new bucket\n __TBB_ASSERT( (mask&(mask+1))==0 && (hash & mask) == hash, nullptr );\n restart:\n node_base* prev = nullptr;\n node_base* curr = b_old()->node_list.load(std::memory_order_acquire);\n while (this->is_valid(curr)) {\n hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first);\n\n if ((curr_node_hash & mask) == hash) {\n if (!b_old.is_writer()) {\n if (!b_old.upgrade_to_writer()) {\n goto restart; // node ptr can be invalid due to concurrent erase\n }\n }\n node_base* next = curr->next;\n // exclude from b_old\n if (prev == nullptr) {\n b_old()->node_list.store(curr->next, std::memory_order_relaxed);\n } else {\n prev->next = curr->next;\n }\n this->add_to_bucket(b_new, curr);\n curr = next;\n } else {\n prev = curr;\n curr = curr->next;\n }\n }\n }\n\n template \n using hash_compare_is_transparent = dependent_bool, U>;\n\npublic:\n\n class accessor;\n // Combines data access, locking, and garbage collection.\n class const_accessor : private node::scoped_type /*which derived from no_copy*/ {\n#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n friend class concurrent_hash_map;\n#else\n friend class concurrent_hash_map;\n#endif\n friend class accessor;\n public:\n // Type of value\n using value_type = const typename concurrent_hash_map::value_type;\n\n // True if result is empty.\n bool empty() const { return !my_node; }\n\n // Set to null\n void release() {\n if( my_node ) {\n node::scoped_type::release();\n my_node = nullptr;\n }\n }\n\n // Return reference to associated value in hash table.\n const_reference operator*() const {\n __TBB_ASSERT( my_node, \"attempt to dereference empty accessor\" );\n return my_node->value();\n }\n\n // Return pointer to associated value in hash table.\n const_pointer operator->() const {\n return &operator*();\n }\n\n // Create empty result\n const_accessor() : my_node(nullptr), my_hash() {}\n\n // Destroy result after releasing the underlying reference.\n ~const_accessor() {\n my_node = nullptr; // scoped lock's release() is called in its destructor\n }\n protected:\n bool is_writer() { return node::scoped_type::is_writer(); }\n node *my_node;\n hashcode_type my_hash;\n };\n\n // Allows write access to elements and combines data access, locking, and garbage collection.\n class accessor: public const_accessor {\n public:\n // Type of value\n using value_type = typename concurrent_hash_map::value_type;\n\n // Return reference to associated value in hash table.\n reference operator*() const {\n __TBB_ASSERT( this->my_node, \"attempt to dereference empty accessor\" );\n return this->my_node->value();\n }\n\n // Return pointer to associated value in hash table.\n pointer operator->() const {\n return &operator*();\n }\n };\n\n explicit concurrent_hash_map( const hash_compare_type& compare, const allocator_type& a = allocator_type() )\n : base_type(a)\n , my_hash_compare(compare)\n {}\n\n concurrent_hash_map() : concurrent_hash_map(hash_compare_type()) {}\n\n explicit concurrent_hash_map( const allocator_type& a )\n : concurrent_hash_map(hash_compare_type(), a)\n {}\n\n // Construct empty table with n preallocated buckets. This number serves also as initial concurrency level.\n concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() )\n : concurrent_hash_map(a)\n {\n this->reserve(n);\n }\n\n concurrent_hash_map( size_type n, const hash_compare_type& compare, const allocator_type& a = allocator_type() )\n : concurrent_hash_map(compare, a)\n {\n this->reserve(n);\n }\n\n // Copy constructor\n concurrent_hash_map( const concurrent_hash_map &table )\n : concurrent_hash_map(node_allocator_traits::select_on_container_copy_construction(table.get_allocator()))\n {\n try_call( [&] {\n internal_copy(table);\n }).on_exception( [&] {\n this->clear();\n });\n }\n\n concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a)\n : concurrent_hash_map(a)\n {\n try_call( [&] {\n internal_copy(table);\n }).on_exception( [&] {\n this->clear();\n });\n }\n\n // Move constructor\n concurrent_hash_map( concurrent_hash_map &&table )\n : concurrent_hash_map(std::move(table.get_allocator()))\n {\n this->internal_move(std::move(table));\n }\n\n // Move constructor\n concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a )\n : concurrent_hash_map(a)\n {\n using is_equal_type = typename node_allocator_traits::is_always_equal;\n internal_move_construct_with_allocator(std::move(table), a, is_equal_type());\n }\n\n // Construction with copying iteration range and given allocator instance\n template \n concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() )\n : concurrent_hash_map(a)\n {\n try_call( [&] {\n internal_copy(first, last, std::distance(first, last));\n }).on_exception( [&] {\n this->clear();\n });\n }\n\n template \n concurrent_hash_map( I first, I last, const hash_compare_type& compare, const allocator_type& a = allocator_type() )\n : concurrent_hash_map(compare, a)\n {\n try_call( [&] {\n internal_copy(first, last, std::distance(first, last));\n }).on_exception( [&] {\n this->clear();\n });\n }\n\n concurrent_hash_map( std::initializer_list il, const hash_compare_type& compare = hash_compare_type(), const allocator_type& a = allocator_type() )\n : concurrent_hash_map(compare, a)\n {\n try_call( [&] {\n internal_copy(il.begin(), il.end(), il.size());\n }).on_exception( [&] {\n this->clear();\n });\n }\n\n concurrent_hash_map( std::initializer_list il, const allocator_type& a )\n : concurrent_hash_map(il, hash_compare_type(), a) {}\n\n // Assignment\n concurrent_hash_map& operator=( const concurrent_hash_map &table ) {\n if( this != &table ) {\n clear();\n copy_assign_allocators(this->my_allocator, table.my_allocator);\n internal_copy(table);\n }\n return *this;\n }\n\n // Move Assignment\n concurrent_hash_map& operator=( concurrent_hash_map &&table ) {\n if( this != &table ) {\n using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment;\n using is_equal_type = typename node_allocator_traits::is_always_equal;\n move_assign_allocators(this->my_allocator, table.my_allocator);\n internal_move_assign(std::move(table), tbb::detail::disjunction());\n }\n return *this;\n }\n\n // Assignment\n concurrent_hash_map& operator=( std::initializer_list il ) {\n clear();\n internal_copy(il.begin(), il.end(), il.size());\n return *this;\n }\n\n // Rehashes and optionally resizes the whole table.\n /** Useful to optimize performance before or after concurrent operations.\n Also enables using of find() and count() concurrent methods in serial context. */\n void rehash(size_type sz = 0) {\n this->reserve(sz); // TODO: add reduction of number of buckets as well\n hashcode_type mask = this->my_mask.load(std::memory_order_relaxed);\n hashcode_type b = (mask+1)>>1; // size or first index of the last segment\n __TBB_ASSERT((b&(b-1))==0, nullptr); // zero or power of 2\n bucket *bp = this->get_bucket( b ); // only the last segment should be scanned for rehashing\n for(; b <= mask; b++, bp++ ) {\n node_base *n = bp->node_list.load(std::memory_order_relaxed);\n __TBB_ASSERT( this->is_valid(n) || empty_rehashed(n) || rehash_required(n), \"Broken internal structure\" );\n __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, \"concurrent or unexpectedly terminated operation during rehash() execution\" );\n if (rehash_required(n)) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one\n hashcode_type h = b; bucket *b_old = bp;\n do {\n __TBB_ASSERT( h > 1, \"The lowermost buckets can't be rehashed\" );\n hashcode_type m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit\n b_old = this->get_bucket( h &= m );\n } while( rehash_required(b_old->node_list.load(std::memory_order_relaxed)) );\n // now h - is index of the root rehashed bucket b_old\n this->mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments\n node_base* prev = nullptr;\n node_base* curr = b_old->node_list.load(std::memory_order_relaxed);\n while (this->is_valid(curr)) {\n hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first);\n\n if ((curr_node_hash & mask) != h) { // should be rehashed\n node_base* next = curr->next;\n // exclude from b_old\n if (prev == nullptr) {\n b_old->node_list.store(curr->next, std::memory_order_relaxed);\n } else {\n prev->next = curr->next;\n }\n bucket *b_new = this->get_bucket(curr_node_hash & mask);\n __TBB_ASSERT(!rehash_required(b_new->node_list.load(std::memory_order_relaxed)), \"hash() function changed for key in table or internal error\");\n this->add_to_bucket(b_new, curr);\n curr = next;\n } else {\n prev = curr;\n curr = curr->next;\n }\n }\n }\n }\n }\n\n // Clear table\n void clear() {\n hashcode_type m = this->my_mask.load(std::memory_order_relaxed);\n __TBB_ASSERT((m&(m+1))==0, \"data structure is invalid\");\n this->my_size.store(0, std::memory_order_relaxed);\n segment_index_type s = this->segment_index_of( m );\n __TBB_ASSERT( s+1 == this->pointers_per_table || !this->my_table[s+1].load(std::memory_order_relaxed), \"wrong mask or concurrent grow\" );\n do {\n __TBB_ASSERT(this->is_valid(this->my_table[s].load(std::memory_order_relaxed)), \"wrong mask or concurrent grow\" );\n segment_ptr_type buckets_ptr = this->my_table[s].load(std::memory_order_relaxed);\n size_type sz = this->segment_size( s ? s : 1 );\n for( segment_index_type i = 0; i < sz; i++ )\n for( node_base *n = buckets_ptr[i].node_list.load(std::memory_order_relaxed);\n this->is_valid(n); n = buckets_ptr[i].node_list.load(std::memory_order_relaxed) )\n {\n buckets_ptr[i].node_list.store(n->next, std::memory_order_relaxed);\n delete_node( n );\n }\n this->delete_segment(s);\n } while(s-- > 0);\n this->my_mask.store(this->embedded_buckets - 1, std::memory_order_relaxed);\n }\n\n // Clear table and destroy it.\n ~concurrent_hash_map() { clear(); }\n\n //------------------------------------------------------------------------\n // Parallel algorithm support\n //------------------------------------------------------------------------\n range_type range( size_type grainsize=1 ) {\n return range_type( *this, grainsize );\n }\n const_range_type range( size_type grainsize=1 ) const {\n return const_range_type( *this, grainsize );\n }\n\n //------------------------------------------------------------------------\n // STL support - not thread-safe methods\n //------------------------------------------------------------------------\n iterator begin() { return iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); }\n const_iterator begin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); }\n const_iterator cbegin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); }\n iterator end() { return iterator( *this, 0, nullptr, nullptr ); }\n const_iterator end() const { return const_iterator( *this, 0, nullptr, nullptr ); }\n const_iterator cend() const { return const_iterator( *this, 0, nullptr, nullptr ); }\n std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); }\n std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); }\n\n template \n typename std::enable_if::value,\n std::pair>::type equal_range( const K& key ) {\n return internal_equal_range(key, end());\n }\n\n template \n typename std::enable_if::value,\n std::pair>::type equal_range( const K& key ) const {\n return internal_equal_range(key, end());\n }\n\n // Number of items in table.\n size_type size() const { return this->my_size.load(std::memory_order_acquire); }\n\n // True if size()==0.\n __TBB_nodiscard bool empty() const { return size() == 0; }\n\n // Upper bound on size.\n size_type max_size() const {\n return allocator_traits_type::max_size(base_type::get_allocator());\n }\n\n // Returns the current number of buckets\n size_type bucket_count() const { return this->my_mask.load(std::memory_order_relaxed) + 1; }\n\n // return allocator object\n allocator_type get_allocator() const { return base_type::get_allocator(); }\n\n // swap two instances. Iterators are invalidated\n void swap(concurrent_hash_map& table) {\n using pocs_type = typename node_allocator_traits::propagate_on_container_swap;\n using is_equal_type = typename node_allocator_traits::is_always_equal;\n swap_allocators(this->my_allocator, table.my_allocator);\n internal_swap(table, tbb::detail::disjunction());\n }\n\n //------------------------------------------------------------------------\n // concurrent map operations\n //------------------------------------------------------------------------\n\n // Return count of items (0 or 1)\n size_type count( const Key &key ) const {\n return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node);\n }\n\n template \n typename std::enable_if::value,\n size_type>::type count( const K& key ) const {\n return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node);\n }\n\n // Find item and acquire a read lock on the item.\n /** Return true if item is found, false otherwise. */\n bool find( const_accessor &result, const Key &key ) const {\n result.release();\n return const_cast(this)->lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node );\n }\n\n // Find item and acquire a write lock on the item.\n /** Return true if item is found, false otherwise. */\n bool find( accessor &result, const Key &key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node);\n }\n\n template \n typename std::enable_if::value,\n bool>::type find( const_accessor& result, const K& key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node);\n }\n\n template \n typename std::enable_if::value,\n bool>::type find( accessor& result, const K& key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node);\n }\n\n // Insert item (if not already present) and acquire a read lock on the item.\n /** Returns true if item is new. */\n bool insert( const_accessor &result, const Key &key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct<>);\n }\n\n // Insert item (if not already present) and acquire a write lock on the item.\n /** Returns true if item is new. */\n bool insert( accessor &result, const Key &key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct<>);\n }\n\n template \n typename std::enable_if::value &&\n std::is_constructible::value,\n bool>::type insert( const_accessor& result, const K& key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct);\n }\n\n template \n typename std::enable_if::value &&\n std::is_constructible::value,\n bool>::type insert( accessor& result, const K& key ) {\n result.release();\n return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct);\n }\n\n // Insert item by copying if there is no such key present already and acquire a read lock on the item.\n /** Returns true if item is new. */\n bool insert( const_accessor &result, const value_type &value ) {\n result.release();\n return lookup(value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct);\n }\n\n // Insert item by copying if there is no such key present already and acquire a write lock on the item.\n /** Returns true if item is new. */\n bool insert( accessor &result, const value_type &value ) {\n result.release();\n return lookup(value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct);\n }\n\n // Insert item by copying if there is no such key present already\n /** Returns true if item is inserted. */\n bool insert( const value_type &value ) {\n return lookup(value.first, &value.second, nullptr, /*write=*/false, &allocate_node_copy_construct);\n }\n\n // Insert item by copying if there is no such key present already and acquire a read lock on the item.\n /** Returns true if item is new. */\n bool insert( const_accessor &result, value_type && value ) {\n return generic_move_insert(result, std::move(value));\n }\n\n // Insert item by copying if there is no such key present already and acquire a write lock on the item.\n /** Returns true if item is new. */\n bool insert( accessor &result, value_type && value ) {\n return generic_move_insert(result, std::move(value));\n }\n\n // Insert item by copying if there is no such key present already\n /** Returns true if item is inserted. */\n bool insert( value_type && value ) {\n return generic_move_insert(accessor_not_used(), std::move(value));\n }\n\n // Insert item by copying if there is no such key present already and acquire a read lock on the item.\n /** Returns true if item is new. */\n template \n bool emplace( const_accessor &result, Args&&... args ) {\n return generic_emplace(result, std::forward(args)...);\n }\n\n // Insert item by copying if there is no such key present already and acquire a write lock on the item.\n /** Returns true if item is new. */\n template \n bool emplace( accessor &result, Args&&... args ) {\n return generic_emplace(result, std::forward(args)...);\n }\n\n // Insert item by copying if there is no such key present already\n /** Returns true if item is inserted. */\n template \n bool emplace( Args&&... args ) {\n return generic_emplace(accessor_not_used(), std::forward(args)...);\n }\n\n // Insert range [first, last)\n template \n void insert( I first, I last ) {\n for ( ; first != last; ++first )\n insert( *first );\n }\n\n // Insert initializer list\n void insert( std::initializer_list il ) {\n insert( il.begin(), il.end() );\n }\n\n // Erase item.\n /** Return true if item was erased by particularly this call. */\n bool erase( const Key &key ) {\n return internal_erase(key);\n }\n\n template \n typename std::enable_if::value,\n bool>::type erase( const K& key ) {\n return internal_erase(key);\n }\n\n // Erase item by const_accessor.\n /** Return true if item was erased by particularly this call. */\n bool erase( const_accessor& item_accessor ) {\n return exclude( item_accessor );\n }\n\n // Erase item by accessor.\n /** Return true if item was erased by particularly this call. */\n bool erase( accessor& item_accessor ) {\n return exclude( item_accessor );\n }\n\nprotected:\n template \n node* allocate_node_helper( const K& key, const T* t, AllocateNodeType allocate_node, std::true_type ) {\n return allocate_node(base_type::get_allocator(), key, t);\n }\n\n template \n node* allocate_node_helper( const K&, const T*, AllocateNodeType, std::false_type ) {\n __TBB_ASSERT(false, \"allocate_node_helper with std::false_type should never been called\");\n return nullptr;\n }\n\n // Insert or find item and optionally acquire a lock on the item.\n template \n bool lookup( const K &key, const T *t, const_accessor *result, bool write, AllocateNodeType allocate_node, node *tmp_n = nullptr)\n {\n __TBB_ASSERT( !result || !result->my_node, nullptr );\n bool return_value;\n hashcode_type const h = my_hash_compare.hash( key );\n hashcode_type m = this->my_mask.load(std::memory_order_acquire);\n segment_index_type grow_segment = 0;\n node *n;\n restart:\n {//lock scope\n __TBB_ASSERT((m&(m+1))==0, \"data structure is invalid\");\n return_value = false;\n // get bucket\n bucket_accessor b( this, h & m );\n // find a node\n n = search_bucket( key, b() );\n if( OpInsert ) {\n // [opt] insert a key\n if( !n ) {\n if( !tmp_n ) {\n tmp_n = allocate_node_helper(key, t, allocate_node, std::integral_constant{});\n }\n while ( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion\n // Rerun search list, in case another thread inserted the intem during the upgrade\n n = search_bucket(key, b());\n if (this->is_valid(n)) { // unfortunately, it did\n if (!b.downgrade_to_reader()) {\n // If the lock was downgraded with reacquiring the mutex\n // Rerun search list in case another thread removed the item during the downgrade\n n = search_bucket(key, b());\n if (!this->is_valid(n)) {\n // Unfortunately, it did\n // We need to try upgrading to writer again\n continue;\n }\n }\n goto exists;\n }\n }\n\n if( this->check_mask_race(h, m) )\n goto restart; // b.release() is done in ~b().\n // insert and set flag to grow the container\n grow_segment = this->insert_new_node( b(), n = tmp_n, m );\n tmp_n = nullptr;\n return_value = true;\n }\n } else { // find or count\n if( !n ) {\n if( this->check_mask_race( h, m ) )\n goto restart; // b.release() is done in ~b(). TODO: replace by continue\n return false;\n }\n return_value = true;\n }\n exists:\n if( !result ) goto check_growth;\n // TODO: the following seems as generic/regular operation\n // acquire the item\n if( !result->try_acquire( n->mutex, write ) ) {\n for( tbb::detail::atomic_backoff backoff(true);; ) {\n if( result->try_acquire( n->mutex, write ) ) break;\n if( !backoff.bounded_pause() ) {\n // the wait takes really long, restart the operation\n b.release();\n __TBB_ASSERT( !OpInsert || !return_value, \"Can't acquire new item in locked bucket?\" );\n yield();\n m = this->my_mask.load(std::memory_order_acquire);\n goto restart;\n }\n }\n }\n }//lock scope\n result->my_node = n;\n result->my_hash = h;\n check_growth:\n // [opt] grow the container\n if( grow_segment ) {\n this->enable_segment( grow_segment );\n }\n if( tmp_n ) // if OpInsert only\n delete_node( tmp_n );\n return return_value;\n }\n\n struct accessor_not_used { void release(){}};\n friend const_accessor* accessor_location( accessor_not_used const& ){ return nullptr;}\n friend const_accessor* accessor_location( const_accessor & a ) { return &a;}\n\n friend bool is_write_access_needed( accessor const& ) { return true;}\n friend bool is_write_access_needed( const_accessor const& ) { return false;}\n friend bool is_write_access_needed( accessor_not_used const& ) { return false;}\n\n template \n bool generic_move_insert( Accessor && result, value_type && value ) {\n result.release();\n return lookup(value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct);\n }\n\n template \n bool generic_emplace( Accessor && result, Args &&... args ) {\n result.release();\n node * node_ptr = create_node(base_type::get_allocator(), std::forward(args)...);\n return lookup(node_ptr->value().first, nullptr, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr);\n }\n\n // delete item by accessor\n bool exclude( const_accessor &item_accessor ) {\n __TBB_ASSERT( item_accessor.my_node, nullptr );\n node_base *const exclude_node = item_accessor.my_node;\n hashcode_type const hash = item_accessor.my_hash;\n hashcode_type mask = this->my_mask.load(std::memory_order_acquire);\n do {\n // get bucket\n bucket_accessor b( this, hash & mask, /*writer=*/true );\n node_base* prev = nullptr;\n node_base* curr = b()->node_list.load(std::memory_order_relaxed);\n\n while (curr && curr != exclude_node) {\n prev = curr;\n curr = curr->next;\n }\n\n if (curr == nullptr) { // someone else was first\n if (this->check_mask_race(hash, mask))\n continue;\n item_accessor.release();\n return false;\n }\n __TBB_ASSERT( curr == exclude_node, nullptr );\n // remove from container\n if (prev == nullptr) {\n b()->node_list.store(curr->next, std::memory_order_relaxed);\n } else {\n prev->next = curr->next;\n }\n\n this->my_size--;\n break;\n } while(true);\n if (!item_accessor.is_writer()) { // need to get exclusive lock\n item_accessor.upgrade_to_writer(); // return value means nothing here\n }\n\n item_accessor.release();\n delete_node(exclude_node); // Only one thread can delete it\n return true;\n }\n\n template \n bool internal_erase( const K& key ) {\n node_base *erase_node;\n hashcode_type const hash = my_hash_compare.hash(key);\n hashcode_type mask = this->my_mask.load(std::memory_order_acquire);\n restart:\n {//lock scope\n // get bucket\n bucket_accessor b( this, hash & mask );\n search:\n node_base* prev = nullptr;\n erase_node = b()->node_list.load(std::memory_order_relaxed);\n while (this->is_valid(erase_node) && !my_hash_compare.equal(key, static_cast(erase_node)->value().first ) ) {\n prev = erase_node;\n erase_node = erase_node->next;\n }\n\n if (erase_node == nullptr) { // not found, but mask could be changed\n if (this->check_mask_race(hash, mask))\n goto restart;\n return false;\n } else if (!b.is_writer() && !b.upgrade_to_writer()) {\n if (this->check_mask_race(hash, mask)) // contended upgrade, check mask\n goto restart;\n goto search;\n }\n\n // remove from container\n if (prev == nullptr) {\n b()->node_list.store(erase_node->next, std::memory_order_relaxed);\n } else {\n prev->next = erase_node->next;\n }\n this->my_size--;\n }\n {\n typename node::scoped_type item_locker( erase_node->mutex, /*write=*/true );\n }\n // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor!\n delete_node(erase_node); // Only one thread can delete it due to write lock on the bucket\n return true;\n }\n\n // Returns an iterator for an item defined by the key, or for the next item after it (if upper==true)\n template \n std::pair internal_equal_range( const K& key, I end_ ) const {\n hashcode_type h = my_hash_compare.hash( key );\n hashcode_type m = this->my_mask.load(std::memory_order_relaxed);\n __TBB_ASSERT((m&(m+1))==0, \"data structure is invalid\");\n h &= m;\n bucket *b = this->get_bucket( h );\n while (rehash_required(b->node_list.load(std::memory_order_relaxed))) {\n m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit\n b = this->get_bucket( h &= m );\n }\n node *n = search_bucket( key, b );\n if( !n )\n return std::make_pair(end_, end_);\n iterator lower(*this, h, b, n), upper(lower);\n return std::make_pair(lower, ++upper);\n }\n\n // Copy \"source\" to *this, where *this must start out empty.\n void internal_copy( const concurrent_hash_map& source ) {\n hashcode_type mask = source.my_mask.load(std::memory_order_relaxed);\n if( this->my_mask.load(std::memory_order_relaxed) == mask ) { // optimized version\n this->reserve(source.my_size.load(std::memory_order_relaxed)); // TODO: load_factor?\n bucket *dst = nullptr, *src = nullptr;\n bool rehashing_required = false;\n for( hashcode_type k = 0; k <= mask; k++ ) {\n if( k & (k-2) ) ++dst,src++; // not the beginning of a segment\n else { dst = this->get_bucket( k ); src = source.get_bucket( k ); }\n __TBB_ASSERT(!rehash_required(dst->node_list.load(std::memory_order_relaxed)), \"Invalid bucket in destination table\");\n node *n = static_cast( src->node_list.load(std::memory_order_relaxed) );\n if (rehash_required(n)) { // source is not rehashed, items are in previous buckets\n rehashing_required = true;\n dst->node_list.store(reinterpret_cast(rehash_req_flag), std::memory_order_relaxed);\n } else for(; n; n = static_cast( n->next ) ) {\n node* node_ptr = create_node(base_type::get_allocator(), n->value().first, n->value().second);\n this->add_to_bucket( dst, node_ptr);\n this->my_size.fetch_add(1, std::memory_order_relaxed);\n }\n }\n if( rehashing_required ) rehash();\n } else internal_copy(source.begin(), source.end(), source.my_size.load(std::memory_order_relaxed));\n }\n\n template \n void internal_copy( I first, I last, size_type reserve_size ) {\n this->reserve(reserve_size); // TODO: load_factor?\n hashcode_type m = this->my_mask.load(std::memory_order_relaxed);\n for(; first != last; ++first) {\n hashcode_type h = my_hash_compare.hash( (*first).first );\n bucket *b = this->get_bucket( h & m );\n __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), \"Invalid bucket in destination table\");\n node* node_ptr = create_node(base_type::get_allocator(), (*first).first, (*first).second);\n this->add_to_bucket( b, node_ptr );\n ++this->my_size; // TODO: replace by non-atomic op\n }\n }\n\n void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type&,\n /*is_always_equal=*/std::true_type )\n {\n this->internal_move(std::move(other));\n }\n\n void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type& a,\n /*is_always_equal=*/std::false_type )\n {\n if (a == other.get_allocator()){\n this->internal_move(std::move(other));\n } else {\n try_call( [&] {\n internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()),\n other.size());\n }).on_exception( [&] {\n this->clear();\n });\n }\n }\n\n void internal_move_assign( concurrent_hash_map&& other,\n /*is_always_equal || POCMA = */std::true_type)\n {\n this->internal_move(std::move(other));\n }\n\n void internal_move_assign(concurrent_hash_map&& other, /*is_always_equal=*/ std::false_type) {\n if (this->my_allocator == other.my_allocator) {\n this->internal_move(std::move(other));\n } else {\n //do per element move\n internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()),\n other.size());\n }\n }\n\n void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::true_type) {\n this->internal_swap_content(other);\n }\n\n void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::false_type) {\n __TBB_ASSERT(this->my_allocator == other.my_allocator, nullptr);\n this->internal_swap_content(other);\n }\n\n // Fast find when no concurrent erasure is used. For internal use inside TBB only!\n /** Return pointer to item with given key, or nullptr if no such item exists.\n Must not be called concurrently with erasure operations. */\n const_pointer internal_fast_find( const Key& key ) const {\n hashcode_type h = my_hash_compare.hash( key );\n hashcode_type m = this->my_mask.load(std::memory_order_acquire);\n node *n;\n restart:\n __TBB_ASSERT((m&(m+1))==0, \"data structure is invalid\");\n bucket *b = this->get_bucket( h & m );\n // TODO: actually, notification is unnecessary here, just hiding double-check\n if (rehash_required(b->node_list.load(std::memory_order_acquire)))\n {\n typename bucket::scoped_type lock;\n if( lock.try_acquire( b->mutex, /*write=*/true ) ) {\n if (rehash_required(b->node_list.load(std::memory_order_relaxed)))\n const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing\n }\n else lock.acquire( b->mutex, /*write=*/false );\n __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr);\n }\n n = search_bucket( key, b );\n if( n )\n return n->storage();\n else if( this->check_mask_race( h, m ) )\n goto restart;\n return nullptr;\n }\n};\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate >,\n typename Alloc = tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_hash_map( It, It, HashCompare = HashCompare(), Alloc = Alloc() )\n-> concurrent_hash_map, iterator_mapped_t, HashCompare, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_hash_map( It, It, Alloc )\n-> concurrent_hash_map, iterator_mapped_t, d1::tbb_hash_compare>, Alloc>;\n\ntemplate >,\n typename Alloc = tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_hash_map( std::initializer_list>, HashCompare = HashCompare(), Alloc = Alloc() )\n-> concurrent_hash_map, T, HashCompare, Alloc>;\n\ntemplate >>\nconcurrent_hash_map( std::initializer_list>, Alloc )\n-> concurrent_hash_map, T, d1::tbb_hash_compare>, Alloc>;\n\n#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */\n\ntemplate \ninline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) {\n if(a.size() != b.size()) return false;\n typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end());\n typename concurrent_hash_map::const_iterator j, j_end(b.end());\n for(; i != i_end; ++i) {\n j = b.equal_range(i->first).first;\n if( j == j_end || !(i->second == j->second) ) return false;\n }\n return true;\n}\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate \ninline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b)\n{ return !(a == b); }\n#endif // !__TBB_CPP20_COMPARISONS_PRESENT\n\ntemplate \ninline void swap(concurrent_hash_map &a, concurrent_hash_map &b)\n{ a.swap( b ); }\n\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\n using detail::split;\n using detail::d2::concurrent_hash_map;\n using detail::d1::tbb_hash_compare;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_concurrent_hash_map_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_lru_cache.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_lru_cache_H\n#define __TBB_concurrent_lru_cache_H\n\n#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE\n #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h\n#endif\n\n#include \"detail/_assert.h\"\n#include \"detail/_aggregator.h\"\n\n#include // for std::map\n#include // for std::list\n#include // for std::make_pair\n#include // for std::find\n#include // for std::atomic\n\nnamespace tbb {\n\nnamespace detail {\nnamespace d1 {\n\n//-----------------------------------------------------------------------------\n// Concurrent LRU cache\n//-----------------------------------------------------------------------------\n\ntemplate\nclass concurrent_lru_cache : no_assign {\n// incapsulated helper classes\nprivate:\n struct handle_object;\n struct storage_map_value_type;\n\n struct aggregator_operation;\n struct retrieve_aggregator_operation;\n struct signal_end_of_usage_aggregator_operation;\n\n// typedefs\npublic:\n using key_type = KeyT;\n using value_type = ValT;\n using pointer = ValT*;\n using reference = ValT&;\n using const_pointer = const ValT*;\n using const_reference = const ValT&;\n\n using value_function_type = KeyToValFunctorT;\n using handle = handle_object;\nprivate:\n using lru_cache_type = concurrent_lru_cache;\n\n using storage_map_type = std::map;\n using storage_map_iterator_type = typename storage_map_type::iterator;\n using storage_map_pointer_type = typename storage_map_type::pointer;\n using storage_map_reference_type = typename storage_map_type::reference;\n\n using history_list_type = std::list;\n using history_list_iterator_type = typename history_list_type::iterator;\n\n using aggregator_operation_type = aggregator_operation;\n using aggregator_function_type = aggregating_functor;\n using aggregator_type = aggregator;\n\n friend class aggregating_functor;\n\n// fields\nprivate:\n value_function_type my_value_function;\n aggregator_type my_aggregator;\n\n storage_map_type my_storage_map; // storage map for used objects\n history_list_type my_history_list; // history list for unused objects\n const std::size_t my_history_list_capacity; // history list's allowed capacity\n\n// interface\npublic:\n\n concurrent_lru_cache(value_function_type value_function, std::size_t cache_capacity)\n : my_value_function(value_function), my_history_list_capacity(cache_capacity) {\n my_aggregator.initialize_handler(aggregator_function_type(this));\n }\n\n handle operator[](key_type key) {\n retrieve_aggregator_operation op(key);\n my_aggregator.execute(&op);\n\n if (op.is_new_value_needed()) {\n op.result().second.my_value = my_value_function(key);\n op.result().second.my_is_ready.store(true, std::memory_order_release);\n } else {\n spin_wait_while_eq(op.result().second.my_is_ready, false);\n }\n\n return handle(*this, op.result());\n }\n\nprivate:\n\n void handle_operations(aggregator_operation* op_list) {\n while (op_list) {\n op_list->cast_and_handle(*this);\n aggregator_operation* prev_op = op_list;\n op_list = op_list->next;\n\n (prev_op->status).store(1, std::memory_order_release);\n }\n }\n\n void signal_end_of_usage(storage_map_reference_type map_record_ref) {\n signal_end_of_usage_aggregator_operation op(map_record_ref);\n my_aggregator.execute(&op);\n }\n\n void signal_end_of_usage_serial(storage_map_reference_type map_record_ref) {\n storage_map_iterator_type map_it = my_storage_map.find(map_record_ref.first);\n\n __TBB_ASSERT(map_it != my_storage_map.end(),\n \"cache should not return past-end iterators to outer world\");\n __TBB_ASSERT(&(*map_it) == &map_record_ref,\n \"dangling reference has been returned to outside world: data race?\");\n __TBB_ASSERT(std::find(my_history_list.begin(), my_history_list.end(), map_it) == my_history_list.end(),\n \"object in use should not be in list of unused objects \");\n\n // if it was the last reference, put it to the LRU history\n if (! --(map_it->second.my_ref_counter)) {\n // if the LRU history is full, evict the oldest items to get space\n if (my_history_list.size() >= my_history_list_capacity) {\n if (my_history_list_capacity == 0) {\n // Since LRU history capacity is zero, there is no need to keep the element in history\n my_storage_map.erase(map_it);\n return;\n }\n std::size_t number_of_elements_to_evict = 1 + my_history_list.size() - my_history_list_capacity;\n\n for (std::size_t i = 0; i < number_of_elements_to_evict; ++i) {\n storage_map_iterator_type map_it_to_evict = my_history_list.back();\n\n __TBB_ASSERT(map_it_to_evict->second.my_ref_counter == 0,\n \"item to be evicted should not have a live references\");\n\n // TODO: can we use forward_list instead of list? pop_front / insert_after last\n my_history_list.pop_back();\n my_storage_map.erase(map_it_to_evict);\n }\n }\n\n // TODO: can we use forward_list instead of list? pop_front / insert_after last\n my_history_list.push_front(map_it);\n map_it->second.my_history_list_iterator = my_history_list.begin();\n }\n }\n\n storage_map_reference_type retrieve_serial(key_type key, bool& is_new_value_needed) {\n storage_map_iterator_type map_it = my_storage_map.find(key);\n\n if (map_it == my_storage_map.end()) {\n map_it = my_storage_map.emplace_hint(\n map_it, std::piecewise_construct, std::make_tuple(key), std::make_tuple(value_type(), 0, my_history_list.end(), false));\n is_new_value_needed = true;\n } else {\n history_list_iterator_type list_it = map_it->second.my_history_list_iterator;\n if (list_it != my_history_list.end()) {\n __TBB_ASSERT(map_it->second.my_ref_counter == 0,\n \"item to be evicted should not have a live references\");\n\n // Item is going to be used. Therefore it is not a subject for eviction,\n // so we remove it from LRU history.\n my_history_list.erase(list_it);\n map_it->second.my_history_list_iterator = my_history_list.end();\n }\n }\n\n ++(map_it->second.my_ref_counter);\n return *map_it;\n }\n};\n\n//-----------------------------------------------------------------------------\n// Value type for storage map in concurrent LRU cache\n//-----------------------------------------------------------------------------\n\ntemplate\nstruct concurrent_lru_cache::storage_map_value_type {\n//typedefs\npublic:\n using ref_counter_type = std::size_t;\n\n// fields\npublic:\n value_type my_value;\n ref_counter_type my_ref_counter;\n history_list_iterator_type my_history_list_iterator;\n std::atomic my_is_ready;\n\n// interface\npublic:\n storage_map_value_type(\n value_type const& value, ref_counter_type ref_counter,\n history_list_iterator_type history_list_iterator, bool is_ready)\n : my_value(value), my_ref_counter(ref_counter),\n my_history_list_iterator(history_list_iterator), my_is_ready(is_ready) {}\n};\n\n//-----------------------------------------------------------------------------\n// Handle object for operator[] in concurrent LRU cache\n//-----------------------------------------------------------------------------\n\ntemplate\nstruct concurrent_lru_cache::handle_object {\n// fields\nprivate:\n lru_cache_type* my_lru_cache_ptr;\n storage_map_pointer_type my_map_record_ptr;\n\n// interface\npublic:\n handle_object()\n : my_lru_cache_ptr(nullptr), my_map_record_ptr(nullptr) {}\n handle_object(lru_cache_type& lru_cache_ref, storage_map_reference_type map_record_ref)\n : my_lru_cache_ptr(&lru_cache_ref), my_map_record_ptr(&map_record_ref) {}\n\n handle_object(handle_object&) = delete;\n void operator=(handle_object&) = delete;\n\n handle_object(handle_object&& other)\n : my_lru_cache_ptr(other.my_lru_cache_ptr), my_map_record_ptr(other.my_map_record_ptr) {\n\n __TBB_ASSERT(\n (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) ||\n (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr),\n \"invalid state of moving object?\");\n\n other.my_lru_cache_ptr = nullptr;\n other.my_map_record_ptr = nullptr;\n }\n\n handle_object& operator=(handle_object&& other) {\n __TBB_ASSERT(\n (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) ||\n (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr),\n \"invalid state of moving object?\");\n\n if (my_lru_cache_ptr)\n my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr);\n\n my_lru_cache_ptr = other.my_lru_cache_ptr;\n my_map_record_ptr = other.my_map_record_ptr;\n other.my_lru_cache_ptr = nullptr;\n other.my_map_record_ptr = nullptr;\n\n return *this;\n }\n\n ~handle_object() {\n if (my_lru_cache_ptr)\n my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr);\n }\n\n operator bool() const {\n return (my_lru_cache_ptr && my_map_record_ptr);\n }\n\n value_type& value() {\n __TBB_ASSERT(my_lru_cache_ptr, \"get value from already moved object?\");\n __TBB_ASSERT(my_map_record_ptr, \"get value from an invalid or already moved object?\");\n\n return my_map_record_ptr->second.my_value;\n }\n};\n\n//-----------------------------------------------------------------------------\n// Aggregator operation for aggregator type in concurrent LRU cache\n//-----------------------------------------------------------------------------\n\ntemplate\nstruct concurrent_lru_cache::aggregator_operation\n : aggregated_operation {\n// incapsulated helper classes\npublic:\n enum class op_type { retrieve, signal_end_of_usage };\n\n// fields\nprivate:\n op_type my_op;\n\n// interface\npublic:\n aggregator_operation(op_type op) : my_op(op) {}\n\n // TODO: aggregator_operation can be implemented\n // - as a statically typed variant type or CRTP? (static, dependent on the use case)\n // - or use pointer to function and apply_visitor (dynamic)\n // - or use virtual functions (dynamic)\n void cast_and_handle(lru_cache_type& lru_cache_ref) {\n if (my_op == op_type::retrieve)\n static_cast(this)->handle(lru_cache_ref);\n else\n static_cast(this)->handle(lru_cache_ref);\n }\n};\n\ntemplate\nstruct concurrent_lru_cache::retrieve_aggregator_operation\n : aggregator_operation, private no_assign {\npublic:\n key_type my_key;\n storage_map_pointer_type my_map_record_ptr;\n bool my_is_new_value_needed;\n\npublic:\n retrieve_aggregator_operation(key_type key)\n : aggregator_operation(aggregator_operation::op_type::retrieve),\n my_key(key), my_map_record_ptr(nullptr), my_is_new_value_needed(false) {}\n\n void handle(lru_cache_type& lru_cache_ref) {\n my_map_record_ptr = &lru_cache_ref.retrieve_serial(my_key, my_is_new_value_needed);\n }\n\n storage_map_reference_type result() {\n __TBB_ASSERT(my_map_record_ptr, \"Attempt to call result() before calling handle()\");\n return *my_map_record_ptr;\n }\n\n bool is_new_value_needed() { return my_is_new_value_needed; }\n};\n\ntemplate\nstruct concurrent_lru_cache::signal_end_of_usage_aggregator_operation\n : aggregator_operation, private no_assign {\n\nprivate:\n storage_map_reference_type my_map_record_ref;\n\npublic:\n signal_end_of_usage_aggregator_operation(storage_map_reference_type map_record_ref)\n : aggregator_operation(aggregator_operation::op_type::signal_end_of_usage),\n my_map_record_ref(map_record_ref) {}\n\n void handle(lru_cache_type& lru_cache_ref) {\n lru_cache_ref.signal_end_of_usage_serial(my_map_record_ref);\n }\n};\n\n// TODO: if we have guarantees that KeyToValFunctorT always have\n// ValT as a return type and KeyT as an argument type\n// we can deduce template parameters of concurrent_lru_cache\n// by pattern matching on KeyToValFunctorT\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d1::concurrent_lru_cache;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_lru_cache_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_map.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_map_H\n#define __TBB_concurrent_map_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_concurrent_skip_list.h\"\n#include \"tbb_allocator.h\"\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate\nstruct map_traits {\n static constexpr std::size_t max_level = RandomGenerator::max_level;\n using random_level_generator_type = RandomGenerator;\n using key_type = Key;\n using mapped_type = Value;\n using compare_type = KeyCompare;\n using value_type = std::pair;\n using reference = value_type&;\n using const_reference = const value_type&;\n using allocator_type = Allocator;\n\n static constexpr bool allow_multimapping = AllowMultimapping;\n\n class value_compare {\n public:\n bool operator()(const value_type& lhs, const value_type& rhs) const {\n return comp(lhs.first, rhs.first);\n }\n\n protected:\n value_compare(compare_type c) : comp(c) {}\n\n friend struct map_traits;\n\n compare_type comp;\n };\n\n static value_compare value_comp(compare_type comp) { return value_compare(comp); }\n\n static const key_type& get_key(const_reference val) {\n return val.first;\n }\n}; // struct map_traits\n\ntemplate \nclass concurrent_multimap;\n\ntemplate , typename Allocator = tbb::tbb_allocator>>\nclass concurrent_map : public concurrent_skip_list, Allocator, false>> {\n using base_type = concurrent_skip_list, Allocator, false>>;\npublic:\n using key_type = Key;\n using mapped_type = Value;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using key_compare = Compare;\n using value_compare = typename base_type::value_compare;\n using allocator_type = Allocator;\n\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n\n using node_type = typename base_type::node_type;\n\n // Include constructors of base type\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_map() = default;\n concurrent_map( const concurrent_map& ) = default;\n concurrent_map( const concurrent_map& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_map( concurrent_map&& ) = default;\n concurrent_map( concurrent_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_map& operator=( const concurrent_map& ) = default;\n concurrent_map& operator=( concurrent_map&& ) = default;\n\n concurrent_map& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n // Observers\n mapped_type& at(const key_type& key) {\n iterator it = this->find(key);\n\n if (it == this->end()) {\n throw_exception(exception_id::invalid_key);\n }\n return it->second;\n }\n\n const mapped_type& at(const key_type& key) const {\n return const_cast(this)->at(key);\n }\n\n mapped_type& operator[](const key_type& key) {\n iterator it = this->find(key);\n\n if (it == this->end()) {\n it = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first;\n }\n return it->second;\n }\n\n mapped_type& operator[](key_type&& key) {\n iterator it = this->find(key);\n\n if (it == this->end()) {\n it = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first;\n }\n return it->second;\n }\n\n using base_type::insert;\n\n template \n typename std::enable_if::value,\n std::pair>::type insert( P&& value )\n {\n return this->emplace(std::forward

(value));\n }\n\n template \n typename std::enable_if::value,\n iterator>::type insert( const_iterator hint, P&& value )\n {\n return this->emplace_hint(hint, std::forward

(value));\n }\n\n template\n void merge(concurrent_map& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_map&& source) {\n this->internal_merge(std::move(source));\n }\n\n template\n void merge(concurrent_multimap& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_multimap&& source) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_map\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_map( It, It, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_map, iterator_mapped_t, Comp, Alloc>;\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_map( std::initializer_list>, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_map, T, Comp, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_map( It, It, Alloc )\n-> concurrent_map, iterator_mapped_t,\n std::less>, Alloc>;\n\ntemplate >>\nconcurrent_map( std::initializer_list>, Alloc )\n-> concurrent_map, T, std::less>, Alloc>;\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_map& lhs,\n concurrent_map& rhs )\n{\n lhs.swap(rhs);\n}\n\ntemplate , typename Allocator = tbb::tbb_allocator>>\nclass concurrent_multimap : public concurrent_skip_list, Allocator, true>> {\n using base_type = concurrent_skip_list, Allocator, true>>;\npublic:\n using key_type = Key;\n using mapped_type = Value;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using key_compare = Compare;\n using value_compare = typename base_type::value_compare;\n using allocator_type = Allocator;\n\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n\n using node_type = typename base_type::node_type;\n\n // Include constructors of base_type\n using base_type::base_type;\n using base_type::insert;\n\n // Required for implicit deduction guides\n concurrent_multimap() = default;\n concurrent_multimap( const concurrent_multimap& ) = default;\n concurrent_multimap( const concurrent_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_multimap( concurrent_multimap&& ) = default;\n concurrent_multimap( concurrent_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_multimap& operator=( const concurrent_multimap& ) = default;\n concurrent_multimap& operator=( concurrent_multimap&& ) = default;\n\n concurrent_multimap& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template \n typename std::enable_if::value,\n std::pair>::type insert( P&& value )\n {\n return this->emplace(std::forward

(value));\n }\n\n template \n typename std::enable_if::value,\n iterator>::type insert( const_iterator hint, P&& value )\n {\n return this->emplace_hint(hint, std::forward

(value));\n }\n\n template\n void merge(concurrent_multimap& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_multimap&& source) {\n this->internal_merge(std::move(source));\n }\n\n template\n void merge(concurrent_map& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_map&& source) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_multimap\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_multimap( It, It, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_multimap, iterator_mapped_t, Comp, Alloc>;\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_multimap( std::initializer_list>, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_multimap, T, Comp, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_multimap( It, It, Alloc )\n-> concurrent_multimap, iterator_mapped_t,\n std::less>, Alloc>;\n\ntemplate >>\nconcurrent_multimap( std::initializer_list>, Alloc )\n-> concurrent_multimap, T, std::less>, Alloc>;\n\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_multimap& lhs,\n concurrent_multimap& rhs )\n{\n lhs.swap(rhs);\n}\n\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d2::concurrent_map;\nusing detail::d2::concurrent_multimap;\nusing detail::split;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_map_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_priority_queue.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_priority_queue_H\n#define __TBB_concurrent_priority_queue_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_aggregator.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_allocator_traits.h\"\n#include \"detail/_range_common.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_containers_helpers.h\"\n#include \"cache_aligned_allocator.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\ntemplate , typename Allocator = cache_aligned_allocator>\nclass concurrent_priority_queue {\npublic:\n using value_type = T;\n using reference = T&;\n using const_reference = const T&;\n\n using size_type = std::size_t;\n using difference_type = std::ptrdiff_t;\n\n using allocator_type = Allocator;\n\n concurrent_priority_queue() : concurrent_priority_queue(allocator_type{}) {}\n\n explicit concurrent_priority_queue( const allocator_type& alloc )\n : mark(0), my_size(0), my_compare(), data(alloc)\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n explicit concurrent_priority_queue( const Compare& compare, const allocator_type& alloc = allocator_type() )\n : mark(0), my_size(0), my_compare(compare), data(alloc)\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n explicit concurrent_priority_queue( size_type init_capacity, const allocator_type& alloc = allocator_type() )\n : mark(0), my_size(0), my_compare(), data(alloc)\n {\n data.reserve(init_capacity);\n my_aggregator.initialize_handler(functor{this});\n }\n\n explicit concurrent_priority_queue( size_type init_capacity, const Compare& compare, const allocator_type& alloc = allocator_type() )\n : mark(0), my_size(0), my_compare(compare), data(alloc)\n {\n data.reserve(init_capacity);\n my_aggregator.initialize_handler(functor{this});\n }\n\n template \n concurrent_priority_queue( InputIterator begin, InputIterator end, const Compare& compare, const allocator_type& alloc = allocator_type() )\n : mark(0), my_compare(compare), data(begin, end, alloc)\n {\n my_aggregator.initialize_handler(functor{this});\n heapify();\n my_size.store(data.size(), std::memory_order_relaxed);\n }\n\n template \n concurrent_priority_queue( InputIterator begin, InputIterator end, const allocator_type& alloc = allocator_type() )\n : concurrent_priority_queue(begin, end, Compare(), alloc) {}\n\n concurrent_priority_queue( std::initializer_list init, const Compare& compare, const allocator_type& alloc = allocator_type() )\n : concurrent_priority_queue(init.begin(), init.end(), compare, alloc) {}\n\n concurrent_priority_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() )\n : concurrent_priority_queue(init, Compare(), alloc) {}\n\n concurrent_priority_queue( const concurrent_priority_queue& other )\n : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare),\n data(other.data)\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n concurrent_priority_queue( const concurrent_priority_queue& other, const allocator_type& alloc )\n : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare),\n data(other.data, alloc)\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n concurrent_priority_queue( concurrent_priority_queue&& other )\n : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare),\n data(std::move(other.data))\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n concurrent_priority_queue( concurrent_priority_queue&& other, const allocator_type& alloc )\n : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare),\n data(std::move(other.data), alloc)\n {\n my_aggregator.initialize_handler(functor{this});\n }\n\n concurrent_priority_queue& operator=( const concurrent_priority_queue& other ) {\n if (this != &other) {\n data = other.data;\n mark = other.mark;\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n }\n return *this;\n }\n\n concurrent_priority_queue& operator=( concurrent_priority_queue&& other ) {\n if (this != &other) {\n // TODO: check if exceptions from std::vector::operator=(vector&&) should be handled separately\n data = std::move(other.data);\n mark = other.mark;\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n }\n return *this;\n }\n\n concurrent_priority_queue& operator=( std::initializer_list init ) {\n assign(init.begin(), init.end());\n return *this;\n }\n\n template \n void assign( InputIterator begin, InputIterator end ) {\n data.assign(begin, end);\n mark = 0;\n my_size.store(data.size(), std::memory_order_relaxed);\n heapify();\n }\n\n void assign( std::initializer_list init ) {\n assign(init.begin(), init.end());\n }\n\n /* Returned value may not reflect results of pending operations.\n This operation reads shared data and will trigger a race condition. */\n __TBB_nodiscard bool empty() const { return size() == 0; }\n\n // Returns the current number of elements contained in the queue\n /* Returned value may not reflect results of pending operations.\n This operation reads shared data and will trigger a race condition. */\n size_type size() const { return my_size.load(std::memory_order_relaxed); }\n\n /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */\n void push( const value_type& value ) {\n cpq_operation op_data(value, PUSH_OP);\n my_aggregator.execute(&op_data);\n if (op_data.status == FAILED)\n throw_exception(exception_id::bad_alloc);\n }\n\n /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */\n void push( value_type&& value ) {\n cpq_operation op_data(value, PUSH_RVALUE_OP);\n my_aggregator.execute(&op_data);\n if (op_data.status == FAILED)\n throw_exception(exception_id::bad_alloc);\n }\n\n /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */\n template \n void emplace( Args&&... args ) {\n // TODO: support uses allocator construction in this place\n push(value_type(std::forward(args)...));\n }\n\n // Gets a reference to and removes highest priority element\n /* If a highest priority element was found, sets elem and returns true,\n otherwise returns false.\n This operation can be safely used concurrently with other push, try_pop or emplace operations. */\n bool try_pop( value_type& value ) {\n cpq_operation op_data(value, POP_OP);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n // This operation affects the whole container => it is not thread-safe\n void clear() {\n data.clear();\n mark = 0;\n my_size.store(0, std::memory_order_relaxed);\n }\n\n // This operation affects the whole container => it is not thread-safe\n void swap( concurrent_priority_queue& other ) {\n if (this != &other) {\n using std::swap;\n swap(data, other.data);\n swap(mark, other.mark);\n\n size_type sz = my_size.load(std::memory_order_relaxed);\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_size.store(sz, std::memory_order_relaxed);\n }\n }\n\n allocator_type get_allocator() const { return data.get_allocator(); }\nprivate:\n enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP};\n enum operation_status {WAIT = 0, SUCCEEDED, FAILED};\n\n class cpq_operation : public aggregated_operation {\n public:\n operation_type type;\n union {\n value_type* elem;\n size_type sz;\n };\n cpq_operation( const value_type& value, operation_type t )\n : type(t), elem(const_cast(&value)) {}\n }; // class cpq_operation\n\n class functor {\n concurrent_priority_queue* my_cpq;\n public:\n functor() : my_cpq(nullptr) {}\n functor( concurrent_priority_queue* cpq ) : my_cpq(cpq) {}\n\n void operator()(cpq_operation* op_list) {\n __TBB_ASSERT(my_cpq != nullptr, \"Invalid functor\");\n my_cpq->handle_operations(op_list);\n }\n }; // class functor\n\n void handle_operations( cpq_operation* op_list ) {\n call_itt_notify(acquired, this);\n cpq_operation* tmp, *pop_list = nullptr;\n __TBB_ASSERT(mark == data.size(), nullptr);\n\n // First pass processes all constant (amortized; reallocation may happen) time pushes and pops.\n while(op_list) {\n // ITT note: &(op_list->status) tag is used to cover accesses to op_list\n // node. This thread is going to handle the operation, and so will acquire it\n // and perform the associated operation w/o triggering a race condition; the\n // thread that created the operation is waiting on the status field, so when\n // this thread is done with the operation, it will perform a\n // store_with_release to give control back to the waiting thread in\n // aggregator::insert_operation.\n // TODO: enable\n call_itt_notify(acquired, &(op_list->status));\n __TBB_ASSERT(op_list->type != INVALID_OP, nullptr);\n\n tmp = op_list;\n op_list = op_list->next.load(std::memory_order_relaxed);\n if (tmp->type == POP_OP) {\n if (mark < data.size() &&\n my_compare(data[0], data.back()))\n {\n // there are newly pushed elems and the last one is higher than top\n *(tmp->elem) = std::move(data.back());\n my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed);\n tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release);\n\n data.pop_back();\n __TBB_ASSERT(mark <= data.size(), nullptr);\n } else { // no convenient item to pop; postpone\n tmp->next.store(pop_list, std::memory_order_relaxed);\n pop_list = tmp;\n }\n } else { // PUSH_OP or PUSH_RVALUE_OP\n __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, \"Unknown operation\");\n#if TBB_USE_EXCEPTIONS\n try\n#endif\n {\n if (tmp->type == PUSH_OP) {\n push_back_helper(*(tmp->elem));\n } else {\n data.push_back(std::move(*(tmp->elem)));\n }\n my_size.store(my_size.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release);\n }\n#if TBB_USE_EXCEPTIONS\n catch(...) {\n tmp->status.store(uintptr_t(FAILED), std::memory_order_release);\n }\n#endif\n }\n }\n\n // Second pass processes pop operations\n while(pop_list) {\n tmp = pop_list;\n pop_list = pop_list->next.load(std::memory_order_relaxed);\n __TBB_ASSERT(tmp->type == POP_OP, nullptr);\n if (data.empty()) {\n tmp->status.store(uintptr_t(FAILED), std::memory_order_release);\n } else {\n __TBB_ASSERT(mark <= data.size(), nullptr);\n if (mark < data.size() &&\n my_compare(data[0], data.back()))\n {\n // there are newly pushed elems and the last one is higher than top\n *(tmp->elem) = std::move(data.back());\n my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed);\n tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release);\n data.pop_back();\n } else { // extract top and push last element down heap\n *(tmp->elem) = std::move(data[0]);\n my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed);\n tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release);\n reheap();\n }\n }\n }\n\n // heapify any leftover pushed elements before doing the next\n // batch of operations\n if (mark < data.size()) heapify();\n __TBB_ASSERT(mark == data.size(), nullptr);\n call_itt_notify(releasing, this);\n }\n\n // Merge unsorted elements into heap\n void heapify() {\n if (!mark && data.size() > 0) mark = 1;\n for (; mark < data.size(); ++mark) {\n // for each unheapified element under size\n size_type cur_pos = mark;\n value_type to_place = std::move(data[mark]);\n do { // push to_place up the heap\n size_type parent = (cur_pos - 1) >> 1;\n if (!my_compare(data[parent], to_place))\n break;\n data[cur_pos] = std::move(data[parent]);\n cur_pos = parent;\n } while(cur_pos);\n data[cur_pos] = std::move(to_place);\n }\n }\n\n // Re-heapify after an extraction\n // Re-heapify by pushing last element down the heap from the root.\n void reheap() {\n size_type cur_pos = 0, child = 1;\n\n while(child < mark) {\n size_type target = child;\n if (child + 1 < mark && my_compare(data[child], data[child + 1]))\n ++target;\n // target now has the higher priority child\n if (my_compare(data[target], data.back()))\n break;\n data[cur_pos] = std::move(data[target]);\n cur_pos = target;\n child = (cur_pos << 1) + 1;\n }\n if (cur_pos != data.size() - 1)\n data[cur_pos] = std::move(data.back());\n data.pop_back();\n if (mark > data.size()) mark = data.size();\n }\n\n void push_back_helper( const T& value ) {\n push_back_helper_impl(value, std::is_copy_constructible{});\n }\n\n void push_back_helper_impl( const T& value, /*is_copy_constructible = */std::true_type ) {\n data.push_back(value);\n }\n\n void push_back_helper_impl( const T&, /*is_copy_constructible = */std::false_type ) {\n __TBB_ASSERT(false, \"error: calling tbb::concurrent_priority_queue.push(const value_type&) for move-only type\");\n }\n\n using aggregator_type = aggregator;\n\n aggregator_type my_aggregator;\n // Padding added to avoid false sharing\n char padding1[max_nfs_size - sizeof(aggregator_type)];\n // The point at which unsorted elements begin\n size_type mark;\n std::atomic my_size;\n Compare my_compare;\n\n // Padding added to avoid false sharing\n char padding2[max_nfs_size - (2*sizeof(size_type)) - sizeof(Compare)];\n //! Storage for the heap of elements in queue, plus unheapified elements\n /** data has the following structure:\n\n binary unheapified\n heap elements\n ____|_______|____\n | | |\n v v v\n [_|...|_|_|...|_| |...| ]\n 0 ^ ^ ^\n | | |__capacity\n | |__my_size\n |__mark\n\n Thus, data stores the binary heap starting at position 0 through\n mark-1 (it may be empty). Then there are 0 or more elements\n that have not yet been inserted into the heap, in positions\n mark through my_size-1. */\n\n using vector_type = std::vector;\n vector_type data;\n\n friend bool operator==( const concurrent_priority_queue& lhs,\n const concurrent_priority_queue& rhs )\n {\n return lhs.data == rhs.data;\n }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\n friend bool operator!=( const concurrent_priority_queue& lhs,\n const concurrent_priority_queue& rhs )\n {\n return !(lhs == rhs);\n }\n#endif\n}; // class concurrent_priority_queue\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate >,\n typename Alloc = tbb::cache_aligned_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_priority_queue( It, It, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_priority_queue, Comp, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_priority_queue( It, It, Alloc )\n-> concurrent_priority_queue, std::less>, Alloc>;\n\ntemplate ,\n typename Alloc = tbb::cache_aligned_allocator,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_priority_queue( std::initializer_list, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_priority_queue;\n\ntemplate >>\nconcurrent_priority_queue( std::initializer_list, Alloc )\n-> concurrent_priority_queue, Alloc>;\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_priority_queue& lhs,\n concurrent_priority_queue& rhs )\n{\n lhs.swap(rhs);\n}\n\n} // namespace d1\n} // namespace detail\ninline namespace v1 {\nusing detail::d1::concurrent_priority_queue;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_priority_queue_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_queue.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_queue_H\n#define __TBB_concurrent_queue_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_concurrent_queue_base.h\"\n#include \"detail/_allocator_traits.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_containers_helpers.h\"\n#include \"cache_aligned_allocator.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nstd::pair internal_try_pop_impl(void* dst, QueueRep& queue, Allocator& alloc ) {\n ticket_type ticket{};\n do {\n // Basically, we need to read `head_counter` before `tail_counter`. To achieve it we build happens-before on `head_counter`\n ticket = queue.head_counter.load(std::memory_order_acquire);\n do {\n if (static_cast(queue.tail_counter.load(std::memory_order_relaxed) - ticket) <= 0) { // queue is empty\n // Queue is empty\n return { false, ticket };\n }\n // Queue had item with ticket k when we looked. Attempt to get that item.\n // Another thread snatched the item, retry.\n } while (!queue.head_counter.compare_exchange_strong(ticket, ticket + 1));\n } while (!queue.choose(ticket).pop(dst, ticket, queue, alloc));\n return { true, ticket };\n}\n\n// A high-performance thread-safe non-blocking concurrent queue.\n// Multiple threads may each push and pop concurrently.\n// Assignment construction is not allowed.\ntemplate >\nclass concurrent_queue {\n using allocator_traits_type = tbb::detail::allocator_traits;\n using queue_representation_type = concurrent_queue_rep;\n using queue_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using queue_allocator_traits = tbb::detail::allocator_traits;\npublic:\n using size_type = std::size_t;\n using value_type = T;\n using reference = T&;\n using const_reference = const T&;\n using difference_type = std::ptrdiff_t;\n\n using allocator_type = Allocator;\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using iterator = concurrent_queue_iterator;\n using const_iterator = concurrent_queue_iterator;\n\n concurrent_queue() : concurrent_queue(allocator_type()) {}\n\n explicit concurrent_queue(const allocator_type& a) :\n my_allocator(a), my_queue_representation(nullptr)\n {\n my_queue_representation = static_cast(r1::cache_aligned_allocate(sizeof(queue_representation_type)));\n queue_allocator_traits::construct(my_allocator, my_queue_representation);\n\n __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), \"alignment error\" );\n }\n\n template \n concurrent_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) :\n concurrent_queue(a)\n {\n for (; begin != end; ++begin)\n push(*begin);\n }\n\n concurrent_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ) :\n concurrent_queue(init.begin(), init.end(), alloc)\n {}\n\n concurrent_queue(const concurrent_queue& src, const allocator_type& a) :\n concurrent_queue(a)\n {\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item);\n }\n\n concurrent_queue(const concurrent_queue& src) :\n concurrent_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator()))\n {\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item);\n }\n\n // Move constructors\n concurrent_queue(concurrent_queue&& src) :\n concurrent_queue(std::move(src.my_allocator))\n {\n internal_swap(src);\n }\n\n concurrent_queue(concurrent_queue&& src, const allocator_type& a) :\n concurrent_queue(a)\n {\n // checking that memory allocated by one instance of allocator can be deallocated\n // with another\n if (my_allocator == src.my_allocator) {\n internal_swap(src);\n } else {\n // allocators are different => performing per-element move\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item);\n src.clear();\n }\n }\n\n // Destroy queue\n ~concurrent_queue() {\n clear();\n my_queue_representation->clear(my_allocator);\n queue_allocator_traits::destroy(my_allocator, my_queue_representation);\n r1::cache_aligned_deallocate(my_queue_representation);\n }\n\n concurrent_queue& operator=( const concurrent_queue& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment\n if (my_queue_representation != other.my_queue_representation) {\n clear();\n my_allocator = other.my_allocator;\n my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item);\n }\n return *this;\n }\n\n concurrent_queue& operator=( concurrent_queue&& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment\n if (my_queue_representation != other.my_queue_representation) {\n clear();\n if (my_allocator == other.my_allocator) {\n internal_swap(other);\n } else {\n my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item);\n other.clear();\n my_allocator = std::move(other.my_allocator);\n }\n }\n return *this;\n }\n\n concurrent_queue& operator=( std::initializer_list init ) {\n assign(init);\n return *this;\n }\n\n template \n void assign( InputIterator first, InputIterator last ) {\n concurrent_queue src(first, last);\n clear();\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item);\n }\n\n void assign( std::initializer_list init ) {\n assign(init.begin(), init.end());\n }\n\n void swap ( concurrent_queue& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_swap\n __TBB_ASSERT(my_allocator == other.my_allocator, \"unequal allocators\");\n internal_swap(other);\n }\n\n // Enqueue an item at tail of queue.\n void push(const T& value) {\n internal_push(value);\n }\n\n void push(T&& value) {\n internal_push(std::move(value));\n }\n\n template \n void emplace( Args&&... args ) {\n internal_push(std::forward(args)...);\n }\n\n // Attempt to dequeue an item from head of queue.\n /** Does not wait for item to become available.\n Returns true if successful; false otherwise. */\n bool try_pop( T& result ) {\n return internal_try_pop(&result);\n }\n\n // Return the number of items in the queue; thread unsafe\n size_type unsafe_size() const {\n std::ptrdiff_t size = my_queue_representation->size();\n return size < 0 ? 0 : size_type(size);\n }\n\n // Equivalent to size()==0.\n __TBB_nodiscard bool empty() const {\n return my_queue_representation->empty();\n }\n\n // Clear the queue. not thread-safe.\n void clear() {\n my_queue_representation->clear(my_allocator);\n }\n\n // Return allocator object\n allocator_type get_allocator() const { return my_allocator; }\n\n //------------------------------------------------------------------------\n // The iterators are intended only for debugging. They are slow and not thread safe.\n //------------------------------------------------------------------------\n\n iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); }\n iterator unsafe_end() { return iterator(); }\n const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); }\n const_iterator unsafe_end() const { return const_iterator(); }\n const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); }\n const_iterator unsafe_cend() const { return const_iterator(); }\n\nprivate:\n void internal_swap(concurrent_queue& src) {\n using std::swap;\n swap(my_queue_representation, src.my_queue_representation);\n }\n\n template \n void internal_push( Args&&... args ) {\n ticket_type k = my_queue_representation->tail_counter++;\n my_queue_representation->choose(k).push(k, *my_queue_representation, my_allocator, std::forward(args)...);\n }\n\n bool internal_try_pop( void* dst ) {\n return internal_try_pop_impl(dst, *my_queue_representation, my_allocator).first;\n }\n\n template \n friend class concurrent_queue_iterator;\n\n static void copy_construct_item(T* location, const void* src) {\n // TODO: use allocator_traits for copy construction\n new (location) value_type(*static_cast(src));\n // queue_allocator_traits::construct(my_allocator, location, *static_cast(src));\n }\n\n static void move_construct_item(T* location, const void* src) {\n // TODO: use allocator_traits for move construction\n new (location) value_type(std::move(*static_cast(const_cast(src))));\n }\n\n queue_allocator_type my_allocator;\n queue_representation_type* my_queue_representation;\n\n friend void swap( concurrent_queue& lhs, concurrent_queue& rhs ) {\n lhs.swap(rhs);\n }\n\n friend bool operator==( const concurrent_queue& lhs, const concurrent_queue& rhs ) {\n return lhs.unsafe_size() == rhs.unsafe_size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin());\n }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\n friend bool operator!=( const concurrent_queue& lhs, const concurrent_queue& rhs ) {\n return !(lhs == rhs);\n }\n#endif // __TBB_CPP20_COMPARISONS_PRESENT\n}; // class concurrent_queue\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n// Deduction guide for the constructor from two iterators\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_queue( It, It, Alloc = Alloc() )\n-> concurrent_queue, Alloc>;\n\n#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */\n\nclass concurrent_monitor;\n\n// The concurrent monitor tags for concurrent_bounded_queue.\nstatic constexpr std::size_t cbq_slots_avail_tag = 0;\nstatic constexpr std::size_t cbq_items_avail_tag = 1;\n} // namespace d2\n\n\nnamespace r1 {\n class concurrent_monitor;\n\n TBB_EXPORT std::uint8_t* __TBB_EXPORTED_FUNC allocate_bounded_queue_rep( std::size_t queue_rep_size );\n TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_bounded_queue_rep( std::uint8_t* mem, std::size_t queue_rep_size );\n TBB_EXPORT void __TBB_EXPORTED_FUNC abort_bounded_queue_monitors( concurrent_monitor* monitors );\n TBB_EXPORT void __TBB_EXPORTED_FUNC notify_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag\n , std::size_t ticket );\n TBB_EXPORT void __TBB_EXPORTED_FUNC wait_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag,\n std::ptrdiff_t target, d1::delegate_base& predicate );\n} // namespace r1\n\n\nnamespace d2 {\n// A high-performance thread-safe blocking concurrent bounded queue.\n// Supports boundedness and blocking semantics.\n// Multiple threads may each push and pop concurrently.\n// Assignment construction is not allowed.\ntemplate >\nclass concurrent_bounded_queue {\n using allocator_traits_type = tbb::detail::allocator_traits;\n using queue_representation_type = concurrent_queue_rep;\n using queue_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using queue_allocator_traits = tbb::detail::allocator_traits;\n\n template \n void internal_wait(r1::concurrent_monitor* monitors, std::size_t monitor_tag, std::ptrdiff_t target, FuncType pred) {\n d1::delegated_function func(pred);\n r1::wait_bounded_queue_monitor(monitors, monitor_tag, target, func);\n }\npublic:\n using size_type = std::ptrdiff_t;\n using value_type = T;\n using reference = T&;\n using const_reference = const T&;\n using difference_type = std::ptrdiff_t;\n\n using allocator_type = Allocator;\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using iterator = concurrent_queue_iterator;\n using const_iterator = concurrent_queue_iterator ;\n\n concurrent_bounded_queue() : concurrent_bounded_queue(allocator_type()) {}\n\n explicit concurrent_bounded_queue( const allocator_type& a ) :\n my_allocator(a), my_capacity(0), my_abort_counter(0), my_queue_representation(nullptr)\n {\n my_queue_representation = reinterpret_cast(\n r1::allocate_bounded_queue_rep(sizeof(queue_representation_type)));\n my_monitors = reinterpret_cast(my_queue_representation + 1);\n queue_allocator_traits::construct(my_allocator, my_queue_representation);\n my_capacity = std::size_t(-1) / (queue_representation_type::item_size > 1 ? queue_representation_type::item_size : 2);\n\n __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), \"alignment error\" );\n __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), \"alignment error\" );\n }\n\n template \n concurrent_bounded_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type() ) :\n concurrent_bounded_queue(a)\n {\n for (; begin != end; ++begin)\n push(*begin);\n }\n\n concurrent_bounded_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ):\n concurrent_bounded_queue(init.begin(), init.end(), alloc)\n {}\n\n concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a ) :\n concurrent_bounded_queue(a)\n {\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item);\n }\n\n concurrent_bounded_queue( const concurrent_bounded_queue& src ) :\n concurrent_bounded_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator()))\n {\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item);\n }\n\n // Move constructors\n concurrent_bounded_queue( concurrent_bounded_queue&& src ) :\n concurrent_bounded_queue(std::move(src.my_allocator))\n {\n internal_swap(src);\n }\n\n concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) :\n concurrent_bounded_queue(a)\n {\n // checking that memory allocated by one instance of allocator can be deallocated\n // with another\n if (my_allocator == src.my_allocator) {\n internal_swap(src);\n } else {\n // allocators are different => performing per-element move\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item);\n src.clear();\n }\n }\n\n // Destroy queue\n ~concurrent_bounded_queue() {\n clear();\n my_queue_representation->clear(my_allocator);\n queue_allocator_traits::destroy(my_allocator, my_queue_representation);\n r1::deallocate_bounded_queue_rep(reinterpret_cast(my_queue_representation),\n sizeof(queue_representation_type));\n }\n\n concurrent_bounded_queue& operator=( const concurrent_bounded_queue& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment\n if (my_queue_representation != other.my_queue_representation) {\n clear();\n my_allocator = other.my_allocator;\n my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item);\n }\n return *this;\n }\n\n concurrent_bounded_queue& operator=( concurrent_bounded_queue&& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment\n if (my_queue_representation != other.my_queue_representation) {\n clear();\n if (my_allocator == other.my_allocator) {\n internal_swap(other);\n } else {\n my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item);\n other.clear();\n my_allocator = std::move(other.my_allocator);\n }\n }\n return *this;\n }\n\n concurrent_bounded_queue& operator=( std::initializer_list init ) {\n assign(init);\n return *this;\n }\n\n template \n void assign( InputIterator first, InputIterator last ) {\n concurrent_bounded_queue src(first, last);\n clear();\n my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item);\n }\n\n void assign( std::initializer_list init ) {\n assign(init.begin(), init.end());\n }\n\n void swap ( concurrent_bounded_queue& other ) {\n //TODO: implement support for std::allocator_traits::propagate_on_container_swap\n __TBB_ASSERT(my_allocator == other.my_allocator, \"unequal allocators\");\n internal_swap(other);\n }\n\n // Enqueue an item at tail of queue.\n void push( const T& value ) {\n internal_push(value);\n }\n\n void push( T&& value ) {\n internal_push(std::move(value));\n }\n\n // Enqueue an item at tail of queue if queue is not already full.\n // Does not wait for queue to become not full.\n // Returns true if item is pushed; false if queue was already full.\n bool try_push( const T& value ) {\n return internal_push_if_not_full(value);\n }\n\n bool try_push( T&& value ) {\n return internal_push_if_not_full(std::move(value));\n }\n\n template \n void emplace( Args&&... args ) {\n internal_push(std::forward(args)...);\n }\n\n template \n bool try_emplace( Args&&... args ) {\n return internal_push_if_not_full(std::forward(args)...);\n }\n\n // Attempt to dequeue an item from head of queue.\n void pop( T& result ) {\n internal_pop(&result);\n }\n\n /** Does not wait for item to become available.\n Returns true if successful; false otherwise. */\n bool try_pop( T& result ) {\n return internal_pop_if_present(&result);\n }\n\n void abort() {\n internal_abort();\n }\n\n // Return the number of items in the queue; thread unsafe\n std::ptrdiff_t size() const {\n return my_queue_representation->size();\n }\n\n void set_capacity( size_type new_capacity ) {\n std::ptrdiff_t c = new_capacity < 0 ? infinite_capacity : new_capacity;\n my_capacity = c;\n }\n\n size_type capacity() const {\n return my_capacity;\n }\n\n // Equivalent to size()==0.\n __TBB_nodiscard bool empty() const {\n return my_queue_representation->empty();\n }\n\n // Clear the queue. not thread-safe.\n void clear() {\n my_queue_representation->clear(my_allocator);\n }\n\n // Return allocator object\n allocator_type get_allocator() const { return my_allocator; }\n\n //------------------------------------------------------------------------\n // The iterators are intended only for debugging. They are slow and not thread safe.\n //------------------------------------------------------------------------\n\n iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); }\n iterator unsafe_end() { return iterator(); }\n const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); }\n const_iterator unsafe_end() const { return const_iterator(); }\n const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); }\n const_iterator unsafe_cend() const { return const_iterator(); }\n\nprivate:\n void internal_swap( concurrent_bounded_queue& src ) {\n std::swap(my_queue_representation, src.my_queue_representation);\n std::swap(my_monitors, src.my_monitors);\n }\n\n static constexpr std::ptrdiff_t infinite_capacity = std::ptrdiff_t(~size_type(0) / 2);\n\n template \n void internal_push( Args&&... args ) {\n unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed);\n ticket_type ticket = my_queue_representation->tail_counter++;\n std::ptrdiff_t target = ticket - my_capacity;\n\n if (static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target) { // queue is full\n auto pred = [&] {\n if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) {\n throw_exception(exception_id::user_abort);\n }\n\n return static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target;\n };\n\n try_call( [&] {\n internal_wait(my_monitors, cbq_slots_avail_tag, target, pred);\n }).on_exception( [&] {\n my_queue_representation->choose(ticket).abort_push(ticket, *my_queue_representation, my_allocator);\n });\n\n }\n __TBB_ASSERT((static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) > target), nullptr);\n my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...);\n r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket);\n }\n\n template \n bool internal_push_if_not_full( Args&&... args ) {\n ticket_type ticket = my_queue_representation->tail_counter.load(std::memory_order_relaxed);\n do {\n if (static_cast(ticket - my_queue_representation->head_counter.load(std::memory_order_relaxed)) >= my_capacity) {\n // Queue is full\n return false;\n }\n // Queue had empty slot with ticket k when we looked. Attempt to claim that slot.\n // Another thread claimed the slot, so retry.\n } while (!my_queue_representation->tail_counter.compare_exchange_strong(ticket, ticket + 1));\n\n my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...);\n r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket);\n return true;\n }\n\n void internal_pop( void* dst ) {\n std::ptrdiff_t target;\n // This loop is a single pop operation; abort_counter should not be re-read inside\n unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed);\n\n do {\n target = my_queue_representation->head_counter++;\n if (static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target) {\n auto pred = [&] {\n if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) {\n throw_exception(exception_id::user_abort);\n }\n\n return static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target;\n };\n\n try_call( [&] {\n internal_wait(my_monitors, cbq_items_avail_tag, target, pred);\n }).on_exception( [&] {\n my_queue_representation->head_counter--;\n });\n }\n __TBB_ASSERT(static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) > target, nullptr);\n } while (!my_queue_representation->choose(target).pop(dst, target, *my_queue_representation, my_allocator));\n\n r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, target);\n }\n\n bool internal_pop_if_present( void* dst ) {\n bool present{};\n ticket_type ticket{};\n std::tie(present, ticket) = internal_try_pop_impl(dst, *my_queue_representation, my_allocator);\n\n if (present) {\n r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, ticket);\n }\n return present;\n }\n\n void internal_abort() {\n ++my_abort_counter;\n r1::abort_bounded_queue_monitors(my_monitors);\n }\n\n static void copy_construct_item(T* location, const void* src) {\n // TODO: use allocator_traits for copy construction\n new (location) value_type(*static_cast(src));\n }\n\n static void move_construct_item(T* location, const void* src) {\n // TODO: use allocator_traits for move construction\n new (location) value_type(std::move(*static_cast(const_cast(src))));\n }\n\n template \n friend class concurrent_queue_iterator;\n\n queue_allocator_type my_allocator;\n std::ptrdiff_t my_capacity;\n std::atomic my_abort_counter;\n queue_representation_type* my_queue_representation;\n\n r1::concurrent_monitor* my_monitors;\n\n friend void swap( concurrent_bounded_queue& lhs, concurrent_bounded_queue& rhs ) {\n lhs.swap(rhs);\n }\n\n friend bool operator==( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) {\n return lhs.size() == rhs.size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin());\n }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\n friend bool operator!=( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) {\n return !(lhs == rhs);\n }\n#endif // __TBB_CPP20_COMPARISONS_PRESENT\n}; // class concurrent_bounded_queue\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n// Deduction guide for the constructor from two iterators\ntemplate >>\nconcurrent_bounded_queue( It, It, Alloc = Alloc() )\n-> concurrent_bounded_queue, Alloc>;\n\n#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */\n\n} //namespace d2\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d2::concurrent_queue;\nusing detail::d2::concurrent_bounded_queue;\nusing detail::r1::user_abort;\nusing detail::r1::bad_last_alloc;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_queue_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_set.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_set_H\n#define __TBB_concurrent_set_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_concurrent_skip_list.h\"\n#include \"tbb_allocator.h\"\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate\nstruct set_traits {\n static constexpr std::size_t max_level = RandomGenerator::max_level;\n using random_level_generator_type = RandomGenerator;\n using key_type = Key;\n using value_type = key_type;\n using compare_type = KeyCompare;\n using value_compare = compare_type;\n using reference = value_type&;\n using const_reference = const value_type&;\n using allocator_type = Allocator;\n\n static constexpr bool allow_multimapping = AllowMultimapping;\n\n static const key_type& get_key(const_reference val) {\n return val;\n }\n\n static value_compare value_comp(compare_type comp) { return comp; }\n}; // struct set_traits\n\ntemplate \nclass concurrent_multiset;\n\ntemplate , typename Allocator = tbb::tbb_allocator>\nclass concurrent_set : public concurrent_skip_list, Allocator, false>> {\n using base_type = concurrent_skip_list, Allocator, false>>;\npublic:\n using key_type = Key;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using key_compare = Compare;\n using value_compare = typename base_type::value_compare;\n using allocator_type = Allocator;\n\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n\n using node_type = typename base_type::node_type;\n\n // Include constructors of base_type\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_set() = default;\n concurrent_set( const concurrent_set& ) = default;\n concurrent_set( const concurrent_set& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_set( concurrent_set&& ) = default;\n concurrent_set( concurrent_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_set& operator=( const concurrent_set& ) = default;\n concurrent_set& operator=( concurrent_set&& ) = default;\n\n concurrent_set& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template\n void merge(concurrent_set& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_set&& source) {\n this->internal_merge(std::move(source));\n }\n\n template\n void merge(concurrent_multiset& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_multiset&& source) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_set\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_set( It, It, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_set, Comp, Alloc>;\n\ntemplate ,\n typename Alloc = tbb::tbb_allocator,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_set( std::initializer_list, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_set;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_set( It, It, Alloc )\n-> concurrent_set,\n std::less>, Alloc>;\n\ntemplate >>\nconcurrent_set( std::initializer_list, Alloc )\n-> concurrent_set, Alloc>;\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_set& lhs,\n concurrent_set& rhs )\n{\n lhs.swap(rhs);\n}\n\ntemplate , typename Allocator = tbb::tbb_allocator>\nclass concurrent_multiset : public concurrent_skip_list, Allocator, true>> {\n using base_type = concurrent_skip_list, Allocator, true>>;\npublic:\n using key_type = Key;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using key_compare = Compare;\n using value_compare = typename base_type::value_compare;\n using allocator_type = Allocator;\n\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n\n using node_type = typename base_type::node_type;\n\n // Include constructors of base_type;\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_multiset() = default;\n concurrent_multiset( const concurrent_multiset& ) = default;\n concurrent_multiset( const concurrent_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_multiset( concurrent_multiset&& ) = default;\n concurrent_multiset( concurrent_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_multiset& operator=( const concurrent_multiset& ) = default;\n concurrent_multiset& operator=( concurrent_multiset&& ) = default;\n\n concurrent_multiset& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template\n void merge(concurrent_set& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_set&& source) {\n this->internal_merge(std::move(source));\n }\n\n template\n void merge(concurrent_multiset& source) {\n this->internal_merge(source);\n }\n\n template\n void merge(concurrent_multiset&& source) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_multiset\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_multiset( It, It, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_multiset, Comp, Alloc>;\n\ntemplate ,\n typename Alloc = tbb::tbb_allocator,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_multiset( std::initializer_list, Comp = Comp(), Alloc = Alloc() )\n-> concurrent_multiset;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_multiset( It, It, Alloc )\n-> concurrent_multiset, std::less>, Alloc>;\n\ntemplate >>\nconcurrent_multiset( std::initializer_list, Alloc )\n-> concurrent_multiset, Alloc>;\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_multiset& lhs,\n concurrent_multiset& rhs )\n{\n lhs.swap(rhs);\n}\n\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d2::concurrent_set;\nusing detail::d2::concurrent_multiset;\nusing detail::split;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_set_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_unordered_map.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_unordered_map_H\n#define __TBB_concurrent_unordered_map_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_concurrent_unordered_base.h\"\n#include \"tbb_allocator.h\"\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nstruct concurrent_unordered_map_traits {\n using value_type = std::pair;\n using key_type = Key;\n using allocator_type = Allocator;\n using hash_compare_type = d1::hash_compare;\n static constexpr bool allow_multimapping = AllowMultimapping;\n\n static constexpr const key_type& get_key( const value_type& value ) {\n return value.first;\n }\n}; // struct concurrent_unordered_map_traits\n\ntemplate \nclass concurrent_unordered_multimap;\n\ntemplate , typename KeyEqual = std::equal_to,\n typename Allocator = tbb::tbb_allocator> >\nclass concurrent_unordered_map\n : public concurrent_unordered_base>\n{\n using traits_type = concurrent_unordered_map_traits;\n using base_type = concurrent_unordered_base;\npublic:\n using key_type = typename base_type::key_type;\n using mapped_type = T;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using hasher = typename base_type::hasher;\n using key_equal = typename base_type::key_equal;\n using allocator_type = typename base_type::allocator_type;\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n using local_iterator = typename base_type::local_iterator;\n using const_local_iterator = typename base_type::const_local_iterator;\n using node_type = typename base_type::node_type;\n\n // Include constructors of base type\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_unordered_map() = default;\n concurrent_unordered_map( const concurrent_unordered_map& ) = default;\n concurrent_unordered_map( const concurrent_unordered_map& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_unordered_map( concurrent_unordered_map&& ) = default;\n concurrent_unordered_map( concurrent_unordered_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_unordered_map& operator=( const concurrent_unordered_map& ) = default;\n concurrent_unordered_map& operator=( concurrent_unordered_map&& ) = default;\n\n concurrent_unordered_map& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n // Observers\n mapped_type& operator[]( const key_type& key ) {\n iterator where = this->find(key);\n\n if (where == this->end()) {\n where = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first;\n }\n return where->second;\n }\n\n mapped_type& operator[]( key_type&& key ) {\n iterator where = this->find(key);\n\n if (where == this->end()) {\n where = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first;\n }\n return where->second;\n }\n\n mapped_type& at( const key_type& key ) {\n iterator where = this->find(key);\n\n if (where == this->end()) {\n throw_exception(exception_id::invalid_key);\n }\n return where->second;\n }\n\n const mapped_type& at( const key_type& key ) const {\n const_iterator where = this->find(key);\n\n if (where == this->end()) {\n throw_exception(exception_id::out_of_range);\n }\n return where->second;\n }\n\n using base_type::insert;\n\n template\n typename std::enable_if::value,\n std::pair>::type insert( P&& value ) {\n return this->emplace(std::forward

(value));\n }\n\n template\n typename std::enable_if::value,\n iterator>::type insert( const_iterator hint, P&& value ) {\n return this->emplace_hint(hint, std::forward

(value));\n }\n\n template \n void merge( concurrent_unordered_map& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_map&& source ) {\n this->internal_merge(std::move(source));\n }\n\n template \n void merge( concurrent_unordered_multimap& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_multimap&& source ) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_unordered_map\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_map( It, It, std::size_t = {},\n Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_map, iterator_mapped_t, Hash, KeyEq, Alloc>;\n\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_map( std::initializer_list>, std::size_t = {},\n Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_map, T, Hash, KeyEq, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_unordered_map( It, It, std::size_t, Alloc )\n-> concurrent_unordered_map, iterator_mapped_t,\n std::hash>,\n std::equal_to>, Alloc>;\n\n// TODO: investigate if a deduction guide for concurrent_unordered_map(It, It, Alloc) is needed\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_map( It, It, std::size_t, Hash, Alloc )\n-> concurrent_unordered_map, iterator_mapped_t,\n Hash, std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_map( std::initializer_list>, std::size_t, Alloc )\n-> concurrent_unordered_map, T, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_map( std::initializer_list>, Alloc )\n-> concurrent_unordered_map, T, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_map( std::initializer_list>, std::size_t, Hash, Alloc )\n-> concurrent_unordered_map, T, Hash,\n std::equal_to>, Alloc>;\n\n#if __APPLE__ && __TBB_CLANG_VERSION == 100000\n// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0\n// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances.\n// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides\n// The issue reproduces only on this version of the compiler\ntemplate \nconcurrent_unordered_map( concurrent_unordered_map, Alloc )\n-> concurrent_unordered_map;\n#endif\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_unordered_map& lhs,\n concurrent_unordered_map& rhs ) {\n lhs.swap(rhs);\n}\n\ntemplate , typename KeyEqual = std::equal_to,\n typename Allocator = tbb::tbb_allocator> >\nclass concurrent_unordered_multimap\n : public concurrent_unordered_base>\n{\n using traits_type = concurrent_unordered_map_traits;\n using base_type = concurrent_unordered_base;\npublic:\n using key_type = typename base_type::key_type;\n using mapped_type = T;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using hasher = typename base_type::hasher;\n using key_equal = typename base_type::key_equal;\n using allocator_type = typename base_type::allocator_type;\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n using local_iterator = typename base_type::local_iterator;\n using const_local_iterator = typename base_type::const_local_iterator;\n using node_type = typename base_type::node_type;\n\n // Include constructors of base type\n using base_type::base_type;\n using base_type::insert;\n\n // Required for implicit deduction guides\n concurrent_unordered_multimap() = default;\n concurrent_unordered_multimap( const concurrent_unordered_multimap& ) = default;\n concurrent_unordered_multimap( const concurrent_unordered_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_unordered_multimap( concurrent_unordered_multimap&& ) = default;\n concurrent_unordered_multimap( concurrent_unordered_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_unordered_multimap& operator=( const concurrent_unordered_multimap& ) = default;\n concurrent_unordered_multimap& operator=( concurrent_unordered_multimap&& ) = default;\n\n concurrent_unordered_multimap& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template \n typename std::enable_if::value,\n std::pair>::type insert( P&& value ) {\n return this->emplace(std::forward

(value));\n }\n\n template\n typename std::enable_if::value,\n iterator>::type insert( const_iterator hint, P&& value ) {\n return this->emplace_hint(hint, std::forward(value));\n }\n\n template \n void merge( concurrent_unordered_map& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_map&& source ) {\n this->internal_merge(std::move(source));\n }\n\n template \n void merge( concurrent_unordered_multimap& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_multimap&& source ) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_unordered_multimap\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multimap( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_multimap, iterator_mapped_t, Hash, KeyEq, Alloc>;\n\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multimap( std::initializer_list>, std::size_t = {},\n Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_multimap, T, Hash, KeyEq, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_unordered_multimap( It, It, std::size_t, Alloc )\n-> concurrent_unordered_multimap, iterator_mapped_t,\n std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multimap( It, It, std::size_t, Hash, Alloc )\n-> concurrent_unordered_multimap, iterator_mapped_t, Hash,\n std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_multimap( std::initializer_list>, std::size_t, Alloc )\n-> concurrent_unordered_multimap, T, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_multimap( std::initializer_list>, Alloc )\n-> concurrent_unordered_multimap, T, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multimap( std::initializer_list>, std::size_t, Hash, Alloc )\n-> concurrent_unordered_multimap, T, Hash,\n std::equal_to>, Alloc>;\n\n#if __APPLE__ && __TBB_CLANG_VERSION == 100000\n// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0\n// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances.\n// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides\n// The issue reproduces only on this version of the compiler\ntemplate \nconcurrent_unordered_multimap( concurrent_unordered_multimap, Alloc )\n-> concurrent_unordered_multimap;\n#endif\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_unordered_multimap& lhs,\n concurrent_unordered_multimap& rhs ) {\n lhs.swap(rhs);\n}\n\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d2::concurrent_unordered_map;\nusing detail::d2::concurrent_unordered_multimap;\nusing detail::split;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_unordered_map_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_unordered_set.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_unordered_set_H\n#define __TBB_concurrent_unordered_set_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_concurrent_unordered_base.h\"\n#include \"tbb_allocator.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nstruct concurrent_unordered_set_traits {\n using key_type = Key;\n using value_type = key_type;\n using allocator_type = Allocator;\n using hash_compare_type = d1::hash_compare;\n static constexpr bool allow_multimapping = AllowMultimapping;\n\n static constexpr const key_type& get_key( const value_type& value ) {\n return value;\n }\n}; // class concurrent_unordered_set_traits\n\ntemplate \nclass concurrent_unordered_multiset;\n\ntemplate , typename KeyEqual = std::equal_to,\n typename Allocator = tbb::tbb_allocator>\nclass concurrent_unordered_set\n : public concurrent_unordered_base>\n{\n using traits_type = concurrent_unordered_set_traits;\n using base_type = concurrent_unordered_base;\npublic:\n using key_type = typename base_type::key_type;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using hasher = typename base_type::hasher;\n using key_equal = typename base_type::key_equal;\n using allocator_type = typename base_type::allocator_type;\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n using local_iterator = typename base_type::local_iterator;\n using const_local_iterator = typename base_type::const_local_iterator;\n using node_type = typename base_type::node_type;\n\n // Include constructors of base_type;\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_unordered_set() = default;\n concurrent_unordered_set( const concurrent_unordered_set& ) = default;\n concurrent_unordered_set( const concurrent_unordered_set& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_unordered_set( concurrent_unordered_set&& ) = default;\n concurrent_unordered_set( concurrent_unordered_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_unordered_set& operator=( const concurrent_unordered_set& ) = default;\n concurrent_unordered_set& operator=( concurrent_unordered_set&& ) = default;\n\n concurrent_unordered_set& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template \n void merge( concurrent_unordered_set& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_set&& source ) {\n this->internal_merge(std::move(source));\n }\n\n template \n void merge( concurrent_unordered_multiset& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_multiset&& source ) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_unordered_set\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_set( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_set, Hash, KeyEq, Alloc>;\n\ntemplate ,\n typename KeyEq = std::equal_to,\n typename Alloc = tbb::tbb_allocator,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_set( std::initializer_list, std::size_t = {},\n Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_set;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_unordered_set( It, It, std::size_t, Alloc )\n-> concurrent_unordered_set, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_set( It, It, std::size_t, Hash, Alloc )\n-> concurrent_unordered_set, Hash, std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_set( std::initializer_list, std::size_t, Alloc )\n-> concurrent_unordered_set, std::equal_to, Alloc>;\n\ntemplate >>\nconcurrent_unordered_set( std::initializer_list, Alloc )\n-> concurrent_unordered_set, std::equal_to, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_set( std::initializer_list, std::size_t, Hash, Alloc )\n-> concurrent_unordered_set, Alloc>;\n\n#if __APPLE__ && __TBB_CLANG_VERSION == 100000\n// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0\n// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances.\n// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides\n// The issue reproduces only on this version of the compiler\ntemplate \nconcurrent_unordered_set( concurrent_unordered_set, Alloc )\n-> concurrent_unordered_set;\n#endif\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_unordered_set& lhs,\n concurrent_unordered_set& rhs ) {\n lhs.swap(rhs);\n}\n\ntemplate , typename KeyEqual = std::equal_to,\n typename Allocator = tbb::tbb_allocator>\nclass concurrent_unordered_multiset\n : public concurrent_unordered_base>\n{\n using traits_type = concurrent_unordered_set_traits;\n using base_type = concurrent_unordered_base;\npublic:\n using key_type = typename base_type::key_type;\n using value_type = typename base_type::value_type;\n using size_type = typename base_type::size_type;\n using difference_type = typename base_type::difference_type;\n using hasher = typename base_type::hasher;\n using key_equal = typename base_type::key_equal;\n using allocator_type = typename base_type::allocator_type;\n using reference = typename base_type::reference;\n using const_reference = typename base_type::const_reference;\n using pointer = typename base_type::pointer;\n using const_pointer = typename base_type::const_pointer;\n using iterator = typename base_type::iterator;\n using const_iterator = typename base_type::const_iterator;\n using local_iterator = typename base_type::local_iterator;\n using const_local_iterator = typename base_type::const_local_iterator;\n using node_type = typename base_type::node_type;\n\n // Include constructors of base_type;\n using base_type::base_type;\n\n // Required for implicit deduction guides\n concurrent_unordered_multiset() = default;\n concurrent_unordered_multiset( const concurrent_unordered_multiset& ) = default;\n concurrent_unordered_multiset( const concurrent_unordered_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {}\n concurrent_unordered_multiset( concurrent_unordered_multiset&& ) = default;\n concurrent_unordered_multiset( concurrent_unordered_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {}\n // Required to respect the rule of 5\n concurrent_unordered_multiset& operator=( const concurrent_unordered_multiset& ) = default;\n concurrent_unordered_multiset& operator=( concurrent_unordered_multiset&& ) = default;\n\n concurrent_unordered_multiset& operator=( std::initializer_list il ) {\n base_type::operator= (il);\n return *this;\n }\n\n template \n void merge( concurrent_unordered_set& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_set&& source ) {\n this->internal_merge(std::move(source));\n }\n\n template \n void merge( concurrent_unordered_multiset& source ) {\n this->internal_merge(source);\n }\n\n template \n void merge( concurrent_unordered_multiset&& source ) {\n this->internal_merge(std::move(source));\n }\n}; // class concurrent_unordered_multiset\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate >,\n typename KeyEq = std::equal_to>,\n typename Alloc = tbb::tbb_allocator>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multiset( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_multiset, Hash, KeyEq, Alloc>;\n\ntemplate ,\n typename KeyEq = std::equal_to,\n typename Alloc = tbb::tbb_allocator,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multiset( std::initializer_list, std::size_t = {},\n Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() )\n-> concurrent_unordered_multiset;\n\ntemplate >,\n typename = std::enable_if_t>>\nconcurrent_unordered_multiset( It, It, std::size_t, Alloc )\n-> concurrent_unordered_multiset, std::hash>,\n std::equal_to>, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multiset( It, It, std::size_t, Hash, Alloc )\n-> concurrent_unordered_multiset, Hash, std::equal_to>, Alloc>;\n\ntemplate >>\nconcurrent_unordered_multiset( std::initializer_list, std::size_t, Alloc )\n-> concurrent_unordered_multiset, std::equal_to, Alloc>;\n\ntemplate >>\nconcurrent_unordered_multiset( std::initializer_list, Alloc )\n-> concurrent_unordered_multiset, std::equal_to, Alloc>;\n\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_unordered_multiset( std::initializer_list, std::size_t, Hash, Alloc )\n-> concurrent_unordered_multiset, Alloc>;\n\n#if __APPLE__ && __TBB_CLANG_VERSION == 100000\n// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0\n// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances.\n// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides\n// The issue reproduces only on this version of the compiler\ntemplate \nconcurrent_unordered_multiset( concurrent_unordered_multiset, Alloc )\n-> concurrent_unordered_multiset;\n#endif\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\ntemplate \nvoid swap( concurrent_unordered_multiset& lhs,\n concurrent_unordered_multiset& rhs ) {\n lhs.swap(rhs);\n}\n\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\n\nusing detail::d2::concurrent_unordered_set;\nusing detail::d2::concurrent_unordered_multiset;\nusing detail::split;\n\n} // inline namespace v1\n} // namespace tbb\n\n#endif // __TBB_concurrent_unordered_set_H\n" }, { "path": "src/tbb/include/oneapi/tbb/concurrent_vector.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_vector_H\n#define __TBB_concurrent_vector_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_allocator_traits.h\"\n#include \"detail/_segment_table.h\"\n#include \"detail/_containers_helpers.h\"\n#include \"blocked_range.h\"\n#include \"cache_aligned_allocator.h\"\n\n#include \n#include // std::move_if_noexcept\n#include \n#if __TBB_CPP20_COMPARISONS_PRESENT\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\ntemplate \nclass vector_iterator {\n using vector_type = Vector;\n\npublic:\n using value_type = Value;\n using size_type = typename vector_type::size_type;\n using difference_type = typename vector_type::difference_type;\n using pointer = value_type*;\n using reference = value_type&;\n using iterator_category = std::random_access_iterator_tag;\n\n template \n friend vector_iterator operator+( typename vector_iterator::difference_type, const vector_iterator& );\n\n template \n friend typename vector_iterator::difference_type operator-( const vector_iterator&, const vector_iterator& );\n\n template \n friend bool operator==( const vector_iterator&, const vector_iterator& );\n\n template \n friend bool operator<( const vector_iterator&, const vector_iterator& );\n\n template \n friend class vector_iterator;\n\n template \n friend class concurrent_vector;\n\nprivate:\n vector_iterator( const vector_type& vector, size_type index, value_type* item = nullptr )\n : my_vector(const_cast(&vector)), my_index(index), my_item(item)\n {}\n\npublic:\n vector_iterator() : my_vector(nullptr), my_index(~size_type(0)), my_item(nullptr)\n {}\n\n vector_iterator( const vector_iterator& other )\n : my_vector(other.my_vector), my_index(other.my_index), my_item(other.my_item)\n {}\n\n vector_iterator& operator=( const vector_iterator& other ) {\n my_vector = other.my_vector;\n my_index = other.my_index;\n my_item = other.my_item;\n return *this;\n }\n\n vector_iterator operator+( difference_type offset ) const {\n return vector_iterator(*my_vector, my_index + offset);\n }\n\n vector_iterator& operator+=( difference_type offset ) {\n my_index += offset;\n my_item = nullptr;\n return *this;\n }\n\n vector_iterator operator-( difference_type offset ) const {\n return vector_iterator(*my_vector, my_index - offset);\n }\n\n vector_iterator& operator-=( difference_type offset ) {\n my_index -= offset;\n my_item = nullptr;\n return *this;\n }\n\n reference operator*() const {\n value_type *item = my_item;\n if (item == nullptr) {\n item = &my_vector->internal_subscript(my_index);\n } else {\n __TBB_ASSERT(item == &my_vector->internal_subscript(my_index), \"corrupt cache\");\n }\n return *item;\n }\n\n pointer operator->() const { return &(operator*()); }\n\n reference operator[]( difference_type k ) const {\n return my_vector->internal_subscript(my_index + k);\n }\n\n vector_iterator& operator++() {\n ++my_index;\n if (my_item != nullptr) {\n if (vector_type::is_first_element_in_segment(my_index)) {\n // If the iterator crosses a segment boundary, the pointer become invalid\n // as possibly next segment is in another memory location\n my_item = nullptr;\n } else {\n ++my_item;\n }\n }\n return *this;\n }\n\n vector_iterator operator++(int) {\n vector_iterator result = *this;\n ++(*this);\n return result;\n }\n\n vector_iterator& operator--() {\n __TBB_ASSERT(my_index > 0, \"operator--() applied to iterator already at beginning of concurrent_vector\");\n --my_index;\n if (my_item != nullptr) {\n if (vector_type::is_first_element_in_segment(my_index)) {\n // If the iterator crosses a segment boundary, the pointer become invalid\n // as possibly next segment is in another memory location\n my_item = nullptr;\n } else {\n --my_item;\n }\n }\n return *this;\n }\n\n vector_iterator operator--(int) {\n vector_iterator result = *this;\n --(*this);\n return result;\n }\n\nprivate:\n // concurrent_vector over which we are iterating.\n vector_type* my_vector;\n\n // Index into the vector\n size_type my_index;\n\n // Caches my_vector *it;\n // If my_item == nullptr cached value is not available use internal_subscript(my_index)\n mutable value_type* my_item;\n}; // class vector_iterator\n\ntemplate \nvector_iterator operator+( typename vector_iterator::difference_type offset,\n const vector_iterator& v )\n{\n return vector_iterator(*v.my_vector, v.my_index + offset);\n}\n\ntemplate \ntypename vector_iterator::difference_type operator-( const vector_iterator& i,\n const vector_iterator& j )\n{\n using difference_type = typename vector_iterator::difference_type;\n return static_cast(i.my_index) - static_cast(j.my_index);\n}\n\ntemplate \nbool operator==( const vector_iterator& i, const vector_iterator& j ) {\n return i.my_vector == j.my_vector && i.my_index == j.my_index;\n}\n\ntemplate \nbool operator!=( const vector_iterator& i, const vector_iterator& j ) {\n return !(i == j);\n}\n\ntemplate \nbool operator<( const vector_iterator& i, const vector_iterator& j ) {\n return i.my_index < j.my_index;\n}\n\ntemplate \nbool operator>( const vector_iterator& i, const vector_iterator& j ) {\n return j < i;\n}\n\ntemplate \nbool operator>=( const vector_iterator& i, const vector_iterator& j ) {\n return !(i < j);\n}\n\ntemplate \nbool operator<=( const vector_iterator& i, const vector_iterator& j ) {\n return !(j < i);\n}\n\nstatic constexpr std::size_t embedded_table_num_segments = 3;\n\ntemplate >\nclass concurrent_vector\n : private segment_table, embedded_table_num_segments>\n{\n using self_type = concurrent_vector;\n using base_type = segment_table;\n\n friend class segment_table;\n\n template \n class generic_range_type : public tbb::blocked_range {\n using base_type = tbb::blocked_range;\n public:\n using value_type = T;\n using reference = T&;\n using const_reference = const T&;\n using iterator = Iterator;\n using difference_type = std::ptrdiff_t;\n\n using base_type::base_type;\n\n template\n generic_range_type( const generic_range_type& r) : blocked_range(r.begin(), r.end(), r.grainsize()) {}\n generic_range_type( generic_range_type& r, split ) : blocked_range(r, split()) {}\n }; // class generic_range_type\n\n static_assert(std::is_same::value,\n \"value_type of the container must be the same as its allocator's\");\n using allocator_traits_type = tbb::detail::allocator_traits;\n // Segment table for concurrent_vector can be extended\n static constexpr bool allow_table_extending = true;\n static constexpr bool is_noexcept_assignment = allocator_traits_type::propagate_on_container_move_assignment::value ||\n allocator_traits_type::is_always_equal::value;\n static constexpr bool is_noexcept_swap = allocator_traits_type::propagate_on_container_swap::value ||\n allocator_traits_type::is_always_equal::value;\n\npublic:\n using value_type = T;\n using allocator_type = Allocator;\n using size_type = std::size_t;\n using difference_type = std::ptrdiff_t;\n using reference = value_type&;\n using const_reference = const value_type&;\n\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using iterator = vector_iterator;\n using const_iterator = vector_iterator;\n using reverse_iterator = std::reverse_iterator;\n using const_reverse_iterator = std::reverse_iterator;\n\n using range_type = generic_range_type;\n using const_range_type = generic_range_type;\n\n concurrent_vector() : concurrent_vector(allocator_type()) {}\n\n explicit concurrent_vector( const allocator_type& alloc ) noexcept\n : base_type(alloc)\n {}\n\n explicit concurrent_vector( size_type count, const value_type& value,\n const allocator_type& alloc = allocator_type() )\n : concurrent_vector(alloc)\n {\n try_call( [&] {\n grow_by(count, value);\n } ).on_exception( [&] {\n base_type::clear();\n });\n }\n\n explicit concurrent_vector( size_type count, const allocator_type& alloc = allocator_type() )\n : concurrent_vector(alloc)\n {\n try_call( [&] {\n grow_by(count);\n } ).on_exception( [&] {\n base_type::clear();\n });\n }\n\n template \n concurrent_vector( InputIterator first, InputIterator last, const allocator_type& alloc = allocator_type() )\n : concurrent_vector(alloc)\n {\n try_call( [&] {\n grow_by(first, last);\n } ).on_exception( [&] {\n base_type::clear();\n });\n }\n\n concurrent_vector( const concurrent_vector& other )\n : base_type(segment_table_allocator_traits::select_on_container_copy_construction(other.get_allocator()))\n {\n try_call( [&] {\n grow_by(other.begin(), other.end());\n } ).on_exception( [&] {\n base_type::clear();\n });\n }\n\n concurrent_vector( const concurrent_vector& other, const allocator_type& alloc )\n : base_type(other, alloc) {}\n\n concurrent_vector(concurrent_vector&& other) noexcept\n : base_type(std::move(other))\n {}\n\n concurrent_vector( concurrent_vector&& other, const allocator_type& alloc )\n : base_type(std::move(other), alloc)\n {}\n\n concurrent_vector( std::initializer_list init,\n const allocator_type& alloc = allocator_type() )\n : concurrent_vector(init.begin(), init.end(), alloc)\n {}\n\n ~concurrent_vector() {}\n\n // Assignment\n concurrent_vector& operator=( const concurrent_vector& other ) {\n base_type::operator=(other);\n return *this;\n }\n\n concurrent_vector& operator=( concurrent_vector&& other ) noexcept(is_noexcept_assignment) {\n base_type::operator=(std::move(other));\n return *this;\n }\n\n concurrent_vector& operator=( std::initializer_list init ) {\n assign(init);\n return *this;\n }\n\n void assign( size_type count, const value_type& value ) {\n destroy_elements();\n grow_by(count, value);\n }\n\n template \n typename std::enable_if::value, void>::type\n assign( InputIterator first, InputIterator last ) {\n destroy_elements();\n grow_by(first, last);\n }\n\n void assign( std::initializer_list init ) {\n destroy_elements();\n assign(init.begin(), init.end());\n }\n\n // Concurrent growth\n iterator grow_by( size_type delta ) {\n return internal_grow_by_delta(delta);\n }\n\n iterator grow_by( size_type delta, const value_type& value ) {\n return internal_grow_by_delta(delta, value);\n }\n\n template \n typename std::enable_if::value, iterator>::type\n grow_by( ForwardIterator first, ForwardIterator last ) {\n auto delta = std::distance(first, last);\n return internal_grow_by_delta(delta, first, last);\n }\n\n iterator grow_by( std::initializer_list init ) {\n return grow_by(init.begin(), init.end());\n }\n\n iterator grow_to_at_least( size_type n ) {\n return internal_grow_to_at_least(n);\n }\n iterator grow_to_at_least( size_type n, const value_type& value ) {\n return internal_grow_to_at_least(n, value);\n }\n\n iterator push_back( const value_type& item ) {\n return internal_emplace_back(item);\n }\n\n iterator push_back( value_type&& item ) {\n return internal_emplace_back(std::move(item));\n }\n\n template \n iterator emplace_back( Args&&... args ) {\n return internal_emplace_back(std::forward(args)...);\n }\n\n // Items access\n reference operator[]( size_type index ) {\n return internal_subscript(index);\n }\n const_reference operator[]( size_type index ) const {\n return internal_subscript(index);\n }\n\n reference at( size_type index ) {\n return internal_subscript_with_exceptions(index);\n }\n const_reference at( size_type index ) const {\n return internal_subscript_with_exceptions(index);\n }\n\n // Get range for iterating with parallel algorithms\n range_type range( size_t grainsize = 1 ) {\n return range_type(begin(), end(), grainsize);\n }\n\n // Get const range for iterating with parallel algorithms\n const_range_type range( size_t grainsize = 1 ) const {\n return const_range_type(begin(), end(), grainsize);\n }\n\n reference front() {\n return internal_subscript(0);\n }\n\n const_reference front() const {\n return internal_subscript(0);\n }\n\n reference back() {\n return internal_subscript(size() - 1);\n }\n\n const_reference back() const {\n return internal_subscript(size() - 1);\n }\n\n // Iterators\n iterator begin() { return iterator(*this, 0); }\n const_iterator begin() const { return const_iterator(*this, 0); }\n const_iterator cbegin() const { return const_iterator(*this, 0); }\n\n iterator end() { return iterator(*this, size()); }\n const_iterator end() const { return const_iterator(*this, size()); }\n const_iterator cend() const { return const_iterator(*this, size()); }\n\n reverse_iterator rbegin() { return reverse_iterator(end()); }\n const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); }\n const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); }\n\n reverse_iterator rend() { return reverse_iterator(begin()); }\n const_reverse_iterator rend() const { return const_reverse_iterator(begin()); }\n const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); }\n\n allocator_type get_allocator() const {\n return base_type::get_allocator();\n }\n\n // Storage\n bool empty() const noexcept {\n return 0 == size();\n }\n\n size_type size() const noexcept {\n return std::min(this->my_size.load(std::memory_order_acquire), capacity());\n }\n\n size_type max_size() const noexcept {\n return allocator_traits_type::max_size(base_type::get_allocator());\n }\n\n size_type capacity() const noexcept {\n return base_type::capacity();\n }\n\n void reserve( size_type n ) {\n if (n == 0) return;\n\n if (n > max_size()) {\n tbb::detail::throw_exception(exception_id::reservation_length_error);\n }\n\n this->assign_first_block_if_necessary(this->segment_index_of(n - 1) + 1);\n base_type::reserve(n);\n }\n\n void resize( size_type n ) {\n internal_resize(n);\n }\n\n void resize( size_type n, const value_type& val ) {\n internal_resize(n, val);\n }\n\n void shrink_to_fit() {\n internal_compact();\n }\n\n void swap(concurrent_vector& other) noexcept(is_noexcept_swap) {\n base_type::swap(other);\n }\n\n void clear() {\n destroy_elements();\n }\n\nprivate:\n using segment_type = typename base_type::segment_type;\n using segment_table_type = typename base_type::segment_table_type;\n using segment_table_allocator_traits = typename base_type::segment_table_allocator_traits;\n using segment_index_type = typename base_type::segment_index_type;\n\n using segment_element_type = typename base_type::value_type;\n using segment_element_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using segment_element_allocator_traits = tbb::detail::allocator_traits;\n\n segment_table_type allocate_long_table( const typename base_type::atomic_segment* embedded_table, size_type start_index ) {\n __TBB_ASSERT(start_index <= this->embedded_table_size, \"Start index out of embedded table\");\n\n // If other threads are trying to set pointers in the short segment, wait for them to finish their\n // assignments before we copy the short segment to the long segment. Note: grow_to_at_least depends on it\n for (segment_index_type i = 0; this->segment_base(i) < start_index; ++i) {\n spin_wait_while_eq(embedded_table[i], segment_type(nullptr));\n }\n\n // It is possible that the table was extend by a thread allocating first_block, need to check this.\n if (this->get_table() != embedded_table) {\n return nullptr;\n }\n\n // Allocate long segment table and fill with null pointers\n segment_table_type new_segment_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), this->pointers_per_long_table);\n // Copy segment pointers from the embedded table\n for (size_type segment_index = 0; segment_index < this->pointers_per_embedded_table; ++segment_index) {\n segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index],\n embedded_table[segment_index].load(std::memory_order_relaxed));\n }\n for (size_type segment_index = this->pointers_per_embedded_table; segment_index < this->pointers_per_long_table; ++segment_index) {\n segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index], nullptr);\n }\n\n return new_segment_table;\n }\n\n // create_segment function is required by the segment_table base class\n segment_type create_segment( segment_table_type table, segment_index_type seg_index, size_type index ) {\n size_type first_block = this->my_first_block.load(std::memory_order_relaxed);\n // First block allocation\n if (seg_index < first_block) {\n // If 0 segment is already allocated, then it remains to wait until the segments are filled to requested\n if (table[0].load(std::memory_order_acquire) != nullptr) {\n spin_wait_while_eq(table[seg_index], segment_type(nullptr));\n return nullptr;\n }\n\n segment_element_allocator_type segment_allocator(base_type::get_allocator());\n segment_type new_segment = nullptr;\n size_type first_block_size = this->segment_size(first_block);\n try_call( [&] {\n new_segment = segment_element_allocator_traits::allocate(segment_allocator, first_block_size);\n } ).on_exception( [&] {\n segment_type disabled_segment = nullptr;\n if (table[0].compare_exchange_strong(disabled_segment, this->segment_allocation_failure_tag)) {\n size_type end_segment = table == this->my_embedded_table ? this->pointers_per_embedded_table : first_block;\n for (size_type i = 1; i < end_segment; ++i) {\n table[i].store(this->segment_allocation_failure_tag, std::memory_order_release);\n }\n }\n });\n\n segment_type disabled_segment = nullptr;\n if (table[0].compare_exchange_strong(disabled_segment, new_segment)) {\n this->extend_table_if_necessary(table, 0, first_block_size);\n for (size_type i = 1; i < first_block; ++i) {\n table[i].store(new_segment, std::memory_order_release);\n }\n\n // Other threads can wait on a snapshot of an embedded table, need to fill it.\n for (size_type i = 1; i < first_block && i < this->pointers_per_embedded_table; ++i) {\n this->my_embedded_table[i].store(new_segment, std::memory_order_release);\n }\n } else if (new_segment != this->segment_allocation_failure_tag) {\n // Deallocate the memory\n segment_element_allocator_traits::deallocate(segment_allocator, new_segment, first_block_size);\n // 0 segment is already allocated, then it remains to wait until the segments are filled to requested\n spin_wait_while_eq(table[seg_index], segment_type(nullptr));\n }\n } else {\n size_type offset = this->segment_base(seg_index);\n if (index == offset) {\n __TBB_ASSERT(table[seg_index].load(std::memory_order_relaxed) == nullptr, \"Only this thread can enable this segment\");\n segment_element_allocator_type segment_allocator(base_type::get_allocator());\n segment_type new_segment = this->segment_allocation_failure_tag;\n try_call( [&] {\n new_segment = segment_element_allocator_traits::allocate(segment_allocator,this->segment_size(seg_index));\n // Shift base address to simplify access by index\n new_segment -= this->segment_base(seg_index);\n } ).on_completion( [&] {\n table[seg_index].store(new_segment, std::memory_order_release);\n });\n } else {\n spin_wait_while_eq(table[seg_index], segment_type(nullptr));\n }\n }\n return nullptr;\n }\n\n // Returns the number of elements in the segment to be destroy\n size_type number_of_elements_in_segment( segment_index_type seg_index ) {\n size_type curr_vector_size = this->my_size.load(std::memory_order_relaxed);\n size_type curr_segment_base = this->segment_base(seg_index);\n\n if (seg_index == 0) {\n return std::min(curr_vector_size, this->segment_size(seg_index));\n } else {\n // Perhaps the segment is allocated, but there are no elements in it.\n if (curr_vector_size < curr_segment_base) {\n return 0;\n }\n return curr_segment_base * 2 > curr_vector_size ? curr_vector_size - curr_segment_base : curr_segment_base;\n }\n }\n\n segment_type nullify_segment( segment_table_type table, size_type segment_index ) {\n segment_type target_segment = table[segment_index].load(std::memory_order_relaxed);\n if (segment_index >= this->my_first_block) {\n table[segment_index].store(nullptr, std::memory_order_relaxed);\n } else {\n if (segment_index == 0) {\n for (size_type i = 0; i < this->my_first_block; ++i) {\n table[i].store(nullptr, std::memory_order_relaxed);\n }\n }\n }\n\n return target_segment;\n }\n\n void deallocate_segment( segment_type address, segment_index_type seg_index ) {\n segment_element_allocator_type segment_allocator(base_type::get_allocator());\n size_type first_block = this->my_first_block.load(std::memory_order_relaxed);\n if (seg_index >= first_block) {\n segment_element_allocator_traits::deallocate(segment_allocator, address, this->segment_size(seg_index));\n }\n else if (seg_index == 0) {\n size_type elements_to_deallocate = first_block > 0 ? this->segment_size(first_block) : this->segment_size(0);\n segment_element_allocator_traits::deallocate(segment_allocator, address, elements_to_deallocate);\n }\n }\n\n // destroy_segment function is required by the segment_table base class\n void destroy_segment( segment_type address, segment_index_type seg_index ) {\n size_type elements_to_destroy = number_of_elements_in_segment(seg_index);\n segment_element_allocator_type segment_allocator(base_type::get_allocator());\n\n for (size_type i = 0; i < elements_to_destroy; ++i) {\n segment_element_allocator_traits::destroy(segment_allocator, address + i);\n }\n\n deallocate_segment(address, seg_index);\n }\n\n // copy_segment function is required by the segment_table base class\n void copy_segment( segment_index_type seg_index, segment_type from, segment_type to ) {\n size_type i = 0;\n try_call( [&] {\n for (; i != number_of_elements_in_segment(seg_index); ++i) {\n segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, from[i]);\n }\n } ).on_exception( [&] {\n // Zero-initialize items left not constructed after the exception\n zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i);\n\n segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed));\n auto table = this->get_table();\n for (segment_index_type j = seg_index + 1; j != last_segment; ++j) {\n auto curr_segment = table[j].load(std::memory_order_relaxed);\n if (curr_segment) {\n zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j));\n }\n }\n this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed);\n });\n }\n\n // move_segment function is required by the segment_table base class\n void move_segment( segment_index_type seg_index, segment_type from, segment_type to ) {\n size_type i = 0;\n try_call( [&] {\n for (; i != number_of_elements_in_segment(seg_index); ++i) {\n segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, std::move(from[i]));\n }\n } ).on_exception( [&] {\n // Zero-initialize items left not constructed after the exception\n zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i);\n\n segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed));\n auto table = this->get_table();\n for (segment_index_type j = seg_index + 1; j != last_segment; ++j) {\n auto curr_segment = table[j].load(std::memory_order_relaxed);\n if (curr_segment) {\n zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j));\n }\n }\n this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed);\n });\n }\n\n static constexpr bool is_first_element_in_segment( size_type index ) {\n // An element is the first in a segment if its index is equal to a power of two\n return is_power_of_two_at_least(index, 2);\n }\n\n const_reference internal_subscript( size_type index ) const {\n return const_cast(this)->internal_subscript(index);\n }\n\n reference internal_subscript( size_type index ) {\n __TBB_ASSERT(index < this->my_size.load(std::memory_order_relaxed), \"Invalid subscript index\");\n return base_type::template internal_subscript(index);\n }\n\n const_reference internal_subscript_with_exceptions( size_type index ) const {\n return const_cast(this)->internal_subscript_with_exceptions(index);\n }\n\n reference internal_subscript_with_exceptions( size_type index ) {\n if (index >= this->my_size.load(std::memory_order_acquire)) {\n tbb::detail::throw_exception(exception_id::out_of_range);\n }\n\n segment_table_type table = this->my_segment_table.load(std::memory_order_acquire);\n\n size_type seg_index = this->segment_index_of(index);\n if (base_type::number_of_segments(table) < seg_index) {\n tbb::detail::throw_exception(exception_id::out_of_range);\n }\n\n if (table[seg_index] <= this->segment_allocation_failure_tag) {\n tbb::detail::throw_exception(exception_id::out_of_range);\n }\n\n return base_type::template internal_subscript(index);\n }\n\n static void zero_unconstructed_elements( pointer start, size_type count ) {\n std::memset(static_cast(start), 0, count * sizeof(value_type));\n }\n\n template \n iterator internal_emplace_back( Args&&... args ) {\n size_type old_size = this->my_size++;\n this->assign_first_block_if_necessary(default_first_block_size);\n auto element_address = &base_type::template internal_subscript(old_size);\n\n // try_call API is not convenient here due to broken\n // variadic capture on GCC 4.8.5\n auto value_guard = make_raii_guard([&] {\n zero_unconstructed_elements(element_address, /*count =*/1);\n });\n\n segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, std::forward(args)...);\n value_guard.dismiss();\n return iterator(*this, old_size, element_address);\n }\n\n template \n void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, const Args&... args ) {\n static_assert(sizeof...(Args) < 2, \"Too many parameters\");\n for (size_type idx = start_idx; idx < end_idx; ++idx) {\n auto element_address = &base_type::template internal_subscript(idx);\n // try_call API is not convenient here due to broken\n // variadic capture on GCC 4.8.5\n auto value_guard = make_raii_guard( [&] {\n segment_index_type last_allocated_segment = this->find_last_allocated_segment(table);\n size_type segment_size = this->segment_size(last_allocated_segment);\n end_idx = end_idx < segment_size ? end_idx : segment_size;\n for (size_type i = idx; i < end_idx; ++i) {\n zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1);\n }\n });\n segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, args...);\n value_guard.dismiss();\n }\n }\n\n template \n void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, ForwardIterator first, ForwardIterator ) {\n for (size_type idx = start_idx; idx < end_idx; ++idx) {\n auto element_address = &base_type::template internal_subscript(idx);\n try_call( [&] {\n segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, *first++);\n } ).on_exception( [&] {\n segment_index_type last_allocated_segment = this->find_last_allocated_segment(table);\n size_type segment_size = this->segment_size(last_allocated_segment);\n end_idx = end_idx < segment_size ? end_idx : segment_size;\n for (size_type i = idx; i < end_idx; ++i) {\n zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1);\n }\n });\n }\n }\n\n template \n iterator internal_grow( size_type start_idx, size_type end_idx, const Args&... args ) {\n this->assign_first_block_if_necessary(this->segment_index_of(end_idx - 1) + 1);\n size_type seg_index = this->segment_index_of(end_idx - 1);\n segment_table_type table = this->get_table();\n this->extend_table_if_necessary(table, start_idx, end_idx);\n\n if (seg_index > this->my_first_block.load(std::memory_order_relaxed)) {\n // So that other threads be able to work with the last segment of grow_by, allocate it immediately.\n // If the last segment is not less than the first block\n if (table[seg_index].load(std::memory_order_relaxed) == nullptr) {\n size_type first_element = this->segment_base(seg_index);\n if (first_element >= start_idx && first_element < end_idx) {\n segment_type segment = table[seg_index].load(std::memory_order_relaxed);\n base_type::enable_segment(segment, table, seg_index, first_element);\n }\n }\n }\n\n internal_loop_construct(table, start_idx, end_idx, args...);\n\n return iterator(*this, start_idx, &base_type::template internal_subscript(start_idx));\n }\n\n\n template \n iterator internal_grow_by_delta( size_type delta, const Args&... args ) {\n if (delta == size_type(0)) {\n return end();\n }\n size_type start_idx = this->my_size.fetch_add(delta);\n size_type end_idx = start_idx + delta;\n\n return internal_grow(start_idx, end_idx, args...);\n }\n\n template \n iterator internal_grow_to_at_least( size_type new_size, const Args&... args ) {\n size_type old_size = this->my_size.load(std::memory_order_relaxed);\n if (new_size == size_type(0)) return iterator(*this, 0);\n while (old_size < new_size && !this->my_size.compare_exchange_weak(old_size, new_size))\n {}\n\n int delta = static_cast(new_size) - static_cast(old_size);\n if (delta > 0) {\n return internal_grow(old_size, new_size, args...);\n }\n\n size_type end_segment = this->segment_index_of(new_size - 1);\n\n // Check/wait for segments allocation completes\n if (end_segment >= this->pointers_per_embedded_table &&\n this->get_table() == this->my_embedded_table)\n {\n spin_wait_while_eq(this->my_segment_table, this->my_embedded_table);\n }\n\n for (segment_index_type seg_idx = 0; seg_idx <= end_segment; ++seg_idx) {\n if (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) {\n atomic_backoff backoff(true);\n while (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) {\n backoff.pause();\n }\n }\n }\n\n #if TBB_USE_DEBUG\n size_type cap = capacity();\n __TBB_ASSERT( cap >= new_size, nullptr);\n #endif\n return iterator(*this, size());\n }\n\n template \n void internal_resize( size_type n, const Args&... args ) {\n if (n == 0) {\n clear();\n return;\n }\n\n size_type old_size = this->my_size.load(std::memory_order_acquire);\n if (n > old_size) {\n reserve(n);\n grow_to_at_least(n, args...);\n } else {\n if (old_size == n) {\n return;\n }\n size_type last_segment = this->segment_index_of(old_size - 1);\n // Delete segments\n for (size_type seg_idx = this->segment_index_of(n - 1) + 1; seg_idx <= last_segment; ++seg_idx) {\n this->delete_segment(seg_idx);\n }\n\n // If n > segment_size(n) => we need to destroy all of the items in the first segment\n // Otherwise, we need to destroy only items with the index < n\n size_type n_segment = this->segment_index_of(n - 1);\n size_type last_index_to_destroy = std::min(this->segment_base(n_segment) + this->segment_size(n_segment), old_size);\n // Destroy elements in curr segment\n for (size_type idx = n; idx < last_index_to_destroy; ++idx) {\n segment_table_allocator_traits::destroy(base_type::get_allocator(), &base_type::template internal_subscript(idx));\n }\n this->my_size.store(n, std::memory_order_release);\n }\n }\n\n void destroy_elements() {\n allocator_type alloc(base_type::get_allocator());\n for (size_type i = 0; i < this->my_size.load(std::memory_order_relaxed); ++i) {\n allocator_traits_type::destroy(alloc, &base_type::template internal_subscript(i));\n }\n this->my_size.store(0, std::memory_order_relaxed);\n }\n\n static bool incompact_predicate( size_type size ) {\n // memory page size\n const size_type page_size = 4096;\n return size < page_size || ((size - 1) % page_size < page_size / 2 && size < page_size * 128);\n }\n\n void internal_compact() {\n const size_type curr_size = this->my_size.load(std::memory_order_relaxed);\n segment_table_type table = this->get_table();\n const segment_index_type k_end = this->find_last_allocated_segment(table); // allocated segments\n const segment_index_type k_stop = curr_size ? this->segment_index_of(curr_size - 1) + 1 : 0; // number of segments to store existing items: 0=>0; 1,2=>1; 3,4=>2; [5-8]=>3;..\n const segment_index_type first_block = this->my_first_block; // number of merged segments, getting values from atomics\n\n segment_index_type k = first_block;\n if (k_stop < first_block) {\n k = k_stop;\n }\n else {\n while (k < k_stop && incompact_predicate(this->segment_size(k) * sizeof(value_type))) k++;\n }\n\n if (k_stop == k_end && k == first_block) {\n return;\n }\n\n // First segment optimization\n if (k != first_block && k) {\n size_type max_block = std::max(first_block, k);\n\n auto buffer_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), max_block);\n\n for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) {\n segment_table_allocator_traits::construct(base_type::get_allocator(), &buffer_table[seg_idx],\n table[seg_idx].load(std::memory_order_relaxed));\n table[seg_idx].store(nullptr, std::memory_order_relaxed);\n }\n\n this->my_first_block.store(k, std::memory_order_relaxed);\n size_type index = 0;\n try_call( [&] {\n for (; index < std::min(this->segment_size(max_block), curr_size); ++index) {\n auto element_address = &static_cast(this)->operator[](index);\n segment_index_type seg_idx = this->segment_index_of(index);\n segment_table_allocator_traits::construct(base_type::get_allocator(), element_address,\n std::move_if_noexcept(buffer_table[seg_idx].load(std::memory_order_relaxed)[index]));\n }\n } ).on_exception( [&] {\n segment_element_allocator_type allocator(base_type::get_allocator());\n for (size_type i = 0; i < index; ++i) {\n auto element_adress = &this->operator[](i);\n segment_element_allocator_traits::destroy(allocator, element_adress);\n }\n segment_element_allocator_traits::deallocate(allocator,\n table[0].load(std::memory_order_relaxed), this->segment_size(max_block));\n\n for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) {\n table[seg_idx].store(buffer_table[seg_idx].load(std::memory_order_relaxed),\n std::memory_order_relaxed);\n buffer_table[seg_idx].store(nullptr, std::memory_order_relaxed);\n }\n segment_table_allocator_traits::deallocate(base_type::get_allocator(),\n buffer_table, max_block);\n this->my_first_block.store(first_block, std::memory_order_relaxed);\n });\n\n // Need to correct deallocate old segments\n // Method destroy_segment respect active first_block, therefore,\n // in order for the segment deletion to work correctly, set the first_block size that was earlier,\n // destroy the unnecessary segments.\n this->my_first_block.store(first_block, std::memory_order_relaxed);\n for (size_type seg_idx = max_block; seg_idx > 0 ; --seg_idx) {\n auto curr_segment = buffer_table[seg_idx - 1].load(std::memory_order_relaxed);\n if (curr_segment != nullptr) {\n destroy_segment(buffer_table[seg_idx - 1].load(std::memory_order_relaxed) + this->segment_base(seg_idx - 1),\n seg_idx - 1);\n }\n }\n\n this->my_first_block.store(k, std::memory_order_relaxed);\n\n for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) {\n segment_table_allocator_traits::destroy(base_type::get_allocator(), &buffer_table[seg_idx]);\n }\n\n segment_table_allocator_traits::deallocate(base_type::get_allocator(), buffer_table, max_block);\n }\n // free unnecessary segments allocated by reserve() call\n if (k_stop < k_end) {\n for (size_type seg_idx = k_end; seg_idx != k_stop; --seg_idx) {\n if (table[seg_idx - 1].load(std::memory_order_relaxed) != nullptr) {\n this->delete_segment(seg_idx - 1);\n }\n }\n if (!k) this->my_first_block.store(0, std::memory_order_relaxed);\n }\n }\n\n // Lever for adjusting the size of first_block at the very first insertion.\n // TODO: consider >1 value, check performance\n static constexpr size_type default_first_block_size = 1;\n\n template \n friend class vector_iterator;\n}; // class concurrent_vector\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n// Deduction guide for the constructor from two iterators\ntemplate >,\n typename = std::enable_if_t>,\n typename = std::enable_if_t>>\nconcurrent_vector( It, It, Alloc = Alloc() )\n-> concurrent_vector, Alloc>;\n#endif\n\ntemplate \nvoid swap(concurrent_vector &lhs,\n concurrent_vector &rhs)\n{\n lhs.swap(rhs);\n}\n\ntemplate \nbool operator==(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin());\n}\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate \nbool operator!=(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return !(lhs == rhs);\n}\n#endif // !__TBB_CPP20_COMPARISONS_PRESENT\n\n#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \ntbb::detail::synthesized_three_way_result::value_type>\noperator<=>(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(),\n rhs.begin(), rhs.end(),\n tbb::detail::synthesized_three_way_comparator{});\n}\n\n#else\n\ntemplate \nbool operator<(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());\n}\n\ntemplate \nbool operator<=(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return !(rhs < lhs);\n}\n\ntemplate \nbool operator>(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return rhs < lhs;\n}\n\ntemplate \nbool operator>=(const concurrent_vector &lhs,\n const concurrent_vector &rhs)\n{\n return !(lhs < rhs);\n}\n#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n using detail::d1::concurrent_vector;\n} // namespace v1\n\n} // namespace tbb\n\n#endif // __TBB_concurrent_vector_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_aggregator.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n\n#ifndef __TBB_detail__aggregator_H\n#define __TBB_detail__aggregator_H\n\n#include \"_assert.h\"\n#include \"_utils.h\"\n#include \n#if !__TBBMALLOC_BUILD // TODO: check this macro with TBB Malloc\n#include \"../profiling.h\"\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n// Base class for aggregated operation\ntemplate \nclass aggregated_operation {\npublic:\n // Zero value means \"wait\" status, all other values are \"user\" specified values and\n // are defined into the scope of a class which uses \"status\"\n std::atomic status;\n\n std::atomic next;\n aggregated_operation() : status{}, next(nullptr) {}\n}; // class aggregated_operation\n\n// Aggregator base class\n/* An aggregator for collecting operations coming from multiple sources and executing\n them serially on a single thread. OperationType must be derived from\n aggregated_operation. The parameter HandlerType is a functor that will be passed the\n list of operations and is expected to handle each operation appropriately, setting the\n status of each operation to non-zero. */\ntemplate \nclass aggregator_generic {\npublic:\n aggregator_generic() : pending_operations(nullptr), handler_busy(false) {}\n\n // Execute an operation\n /* Places an operation into the waitlist (pending_operations), and either handles the list,\n or waits for the operation to complete, or returns.\n The long_life_time parameter specifies the life time of the given operation object.\n Operations with long_life_time == true may be accessed after execution.\n A \"short\" life time operation (long_life_time == false) can be destroyed\n during execution, and so any access to it after it was put into the waitlist,\n including status check, is invalid. As a consequence, waiting for completion\n of such operation causes undefined behavior. */\n template \n void execute( OperationType* op, HandlerType& handle_operations, bool long_life_time = true ) {\n // op->status should be read before inserting the operation into the\n // aggregator waitlist since it can become invalid after executing a\n // handler (if the operation has 'short' life time.)\n const uintptr_t status = op->status.load(std::memory_order_relaxed);\n\n // ITT note: &(op->status) tag is used to cover accesses to this op node. This\n // thread has created the operation, and now releases it so that the handler\n // thread may handle the associated operation w/o triggering a race condition;\n // thus this tag will be acquired just before the operation is handled in the\n // handle_operations functor.\n call_itt_notify(releasing, &(op->status));\n // insert the operation in the queue.\n OperationType* res = pending_operations.load(std::memory_order_relaxed);\n do {\n op->next.store(res, std::memory_order_relaxed);\n } while (!pending_operations.compare_exchange_strong(res, op));\n if (!res) { // first in the list; handle the operations\n // ITT note: &pending_operations tag covers access to the handler_busy flag,\n // which this waiting handler thread will try to set before entering\n // handle_operations.\n call_itt_notify(acquired, &pending_operations);\n start_handle_operations(handle_operations);\n // The operation with 'short' life time can already be destroyed\n if (long_life_time)\n __TBB_ASSERT(op->status.load(std::memory_order_relaxed), nullptr);\n }\n // Not first; wait for op to be ready\n else if (!status) { // operation is blocking here.\n __TBB_ASSERT(long_life_time, \"Waiting for an operation object that might be destroyed during processing\");\n call_itt_notify(prepare, &(op->status));\n spin_wait_while_eq(op->status, uintptr_t(0));\n }\n }\n\nprivate:\n // Trigger the handling of operations when the handler is free\n template \n void start_handle_operations( HandlerType& handle_operations ) {\n OperationType* op_list;\n\n // ITT note: &handler_busy tag covers access to pending_operations as it is passed\n // between active and waiting handlers. Below, the waiting handler waits until\n // the active handler releases, and the waiting handler acquires &handler_busy as\n // it becomes the active_handler. The release point is at the end of this\n // function, when all operations in pending_operations have been handled by the\n // owner of this aggregator.\n call_itt_notify(prepare, &handler_busy);\n // get the handler_busy:\n // only one thread can possibly spin here at a time\n spin_wait_until_eq(handler_busy, uintptr_t(0));\n call_itt_notify(acquired, &handler_busy);\n // acquire fence not necessary here due to causality rule and surrounding atomics\n handler_busy.store(1, std::memory_order_relaxed);\n\n // ITT note: &pending_operations tag covers access to the handler_busy flag\n // itself. Capturing the state of the pending_operations signifies that\n // handler_busy has been set and a new active handler will now process that list's\n // operations.\n call_itt_notify(releasing, &pending_operations);\n // grab pending_operations\n op_list = pending_operations.exchange(nullptr);\n\n // handle all the operations\n handle_operations(op_list);\n\n // release the handler\n handler_busy.store(0, std::memory_order_release);\n }\n\n // An atomically updated list (aka mailbox) of pending operations\n std::atomic pending_operations;\n // Controls threads access to handle_operations\n std::atomic handler_busy;\n}; // class aggregator_generic\n\ntemplate \nclass aggregator : public aggregator_generic {\n HandlerType handle_operations;\npublic:\n aggregator() = default;\n\n void initialize_handler( HandlerType h ) { handle_operations = h; }\n\n void execute(OperationType* op) {\n aggregator_generic::execute(op, handle_operations);\n }\n}; // class aggregator\n\n// the most-compatible friend declaration (vs, gcc, icc) is\n// template friend class aggregating_functor;\ntemplate \nclass aggregating_functor {\n AggregatingClass* my_object{nullptr};\npublic:\n aggregating_functor() = default;\n aggregating_functor( AggregatingClass* object ) : my_object(object) {\n __TBB_ASSERT(my_object, nullptr);\n }\n\n void operator()( OperationList* op_list ) {\n __TBB_ASSERT(my_object, nullptr);\n my_object->handle_operations(op_list);\n }\n}; // class aggregating_functor\n\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__aggregator_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_aligned_space.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n#ifndef __TBB_aligned_space_H\n#define __TBB_aligned_space_H\n\n#include \n\n#include \"_template_helpers.h\"\n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n//! Block of space aligned sufficiently to construct an array T with N elements.\n/** The elements are not constructed or destroyed by this class.\n @ingroup memory_allocation */\ntemplate\nclass aligned_space {\n alignas(alignof(T)) std::uint8_t aligned_array[N * sizeof(T)];\n\npublic:\n //! Pointer to beginning of array\n T* begin() const { return punned_cast(&aligned_array); }\n\n //! Pointer to one past last element in array.\n T* end() const { return begin() + N; }\n};\n\n} // namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_aligned_space_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_allocator_traits.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__allocator_traits_H\n#define __TBB_detail__allocator_traits_H\n\n#include \"_config.h\"\n#include \"_template_helpers.h\"\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT\n// Struct is_always_equal_detector provides the member type \"type\" which is\n// Allocator::is_always_equal if it is present, std::false_type otherwise\ntemplate \nstruct is_always_equal_detector {\n using type = std::false_type;\n};\n\ntemplate \nstruct is_always_equal_detector>\n{\n using type = typename Allocator::is_always_equal;\n};\n#endif // !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT\n\ntemplate \nclass allocator_traits : public std::allocator_traits\n{\n using base_type = std::allocator_traits;\npublic:\n#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT\n using is_always_equal = typename is_always_equal_detector::type;\n#endif\n\n template \n using rebind_traits = typename tbb::detail::allocator_traits>;\n}; // struct allocator_traits\n\ntemplate \nvoid copy_assign_allocators_impl( Allocator& lhs, const Allocator& rhs, /*pocca = */std::true_type ) {\n lhs = rhs;\n}\n\ntemplate \nvoid copy_assign_allocators_impl( Allocator&, const Allocator&, /*pocca = */ std::false_type ) {}\n\n// Copy assigns allocators only if propagate_on_container_copy_assignment is true\ntemplate \nvoid copy_assign_allocators( Allocator& lhs, const Allocator& rhs ) {\n using pocca_type = typename allocator_traits::propagate_on_container_copy_assignment;\n copy_assign_allocators_impl(lhs, rhs, pocca_type());\n}\n\ntemplate \nvoid move_assign_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocma = */ std::true_type ) {\n lhs = std::move(rhs);\n}\n\ntemplate \nvoid move_assign_allocators_impl( Allocator&, Allocator&, /*pocma = */ std::false_type ) {}\n\n// Move assigns allocators only if propagate_on_container_move_assignment is true\ntemplate \nvoid move_assign_allocators( Allocator& lhs, Allocator& rhs ) {\n using pocma_type = typename allocator_traits::propagate_on_container_move_assignment;\n move_assign_allocators_impl(lhs, rhs, pocma_type());\n}\n\ntemplate \nvoid swap_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocs = */ std::true_type ) {\n using std::swap;\n swap(lhs, rhs);\n}\n\ntemplate \nvoid swap_allocators_impl( Allocator&, Allocator&, /*pocs = */ std::false_type ) {}\n\n// Swaps allocators only if propagate_on_container_swap is true\ntemplate \nvoid swap_allocators( Allocator& lhs, Allocator& rhs ) {\n using pocs_type = typename allocator_traits::propagate_on_container_swap;\n swap_allocators_impl(lhs, rhs, pocs_type());\n}\n\n} // inline namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__allocator_traits_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_assert.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__assert_H\n#define __TBB_detail__assert_H\n\n#include \"_config.h\"\n\n#if __TBBMALLOC_BUILD\nnamespace rml { namespace internal {\n#else\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n#endif\n//! Process an assertion failure.\n/** Normally called from __TBB_ASSERT macro.\n If assertion handler is null, print message for assertion failure and abort.\n Otherwise call the assertion handler. */\nTBB_EXPORT void __TBB_EXPORTED_FUNC assertion_failure(const char* location, int line, const char* expression, const char* comment);\n#if __TBBMALLOC_BUILD\n}} // namespaces rml::internal\n#else\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n#endif\n\n#if __TBBMALLOC_BUILD\n//! Release version of assertions\n#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__func__,__LINE__,#predicate,message))\n#else\n#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::detail::r1::assertion_failure(__func__,__LINE__,#predicate,message))\n#endif\n\n#if TBB_USE_ASSERT\n //! Assert that predicate is true.\n /** If predicate is false, print assertion failure message.\n If the comment argument is not nullptr, it is printed as part of the failure message.\n The comment argument has no other effect. */\n #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message)\n //! \"Extended\" version\n #define __TBB_ASSERT_EX __TBB_ASSERT\n#else\n //! No-op version of __TBB_ASSERT.\n #define __TBB_ASSERT(predicate,comment) ((void)0)\n //! \"Extended\" version is useful to suppress warnings if a variable is only used with an assert\n #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate)))\n#endif // TBB_USE_ASSERT\n\n#endif // __TBB_detail__assert_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_attach.h", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__attach_H\n#define __TBB_detail__attach_H\n\n#include \"_config.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n struct attach {};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__attach_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_concurrent_queue_base.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__concurrent_queue_base_H\n#define __TBB_detail__concurrent_queue_base_H\n\n#include \"_utils.h\"\n#include \"_exception.h\"\n#include \"_machine.h\"\n#include \"_allocator_traits.h\"\n\n#include \"../profiling.h\"\n#include \"../spin_mutex.h\"\n#include \"../cache_aligned_allocator.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\nusing ticket_type = std::size_t;\n\ntemplate \ninline bool is_valid_page(const Page p) {\n return reinterpret_cast(p) > 1;\n}\n\ntemplate \nstruct concurrent_queue_rep;\n\ntemplate \nclass micro_queue_pop_finalizer;\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// unary minus operator applied to unsigned type, result still unsigned\n// #pragma warning( push )\n// #pragma warning( disable: 4146 )\n#endif\n\n// A queue using simple locking.\n// For efficiency, this class has no constructor.\n// The caller is expected to zero-initialize it.\ntemplate \nclass micro_queue {\nprivate:\n using queue_rep_type = concurrent_queue_rep;\n using self_type = micro_queue;\npublic:\n using size_type = std::size_t;\n using value_type = T;\n using reference = value_type&;\n using const_reference = const value_type&;\n\n using allocator_type = Allocator;\n using allocator_traits_type = tbb::detail::allocator_traits;\n using queue_allocator_type = typename allocator_traits_type::template rebind_alloc;\n\n static constexpr size_type item_size = sizeof(T);\n static constexpr size_type items_per_page = item_size <= 8 ? 32 :\n item_size <= 16 ? 16 :\n item_size <= 32 ? 8 :\n item_size <= 64 ? 4 :\n item_size <= 128 ? 2 : 1;\n\n struct padded_page {\n padded_page() {}\n ~padded_page() {}\n\n reference operator[] (std::size_t index) {\n __TBB_ASSERT(index < items_per_page, \"Index out of range\");\n return items[index];\n }\n\n const_reference operator[] (std::size_t index) const {\n __TBB_ASSERT(index < items_per_page, \"Index out of range\");\n return items[index];\n }\n\n padded_page* next{ nullptr };\n std::atomic mask{};\n\n union {\n value_type items[items_per_page];\n };\n }; // struct padded_page\n\n using page_allocator_type = typename allocator_traits_type::template rebind_alloc;\nprotected:\n using page_allocator_traits = tbb::detail::allocator_traits;\n\npublic:\n using item_constructor_type = void (*)(value_type* location, const void* src);\n micro_queue() = default;\n micro_queue( const micro_queue& ) = delete;\n micro_queue& operator=( const micro_queue& ) = delete;\n\n size_type prepare_page( ticket_type k, queue_rep_type& base, page_allocator_type page_allocator,\n padded_page*& p ) {\n __TBB_ASSERT(p == nullptr, \"Invalid page argument for prepare_page\");\n k &= -queue_rep_type::n_queue;\n size_type index = modulo_power_of_two(k / queue_rep_type::n_queue, items_per_page);\n if (!index) {\n try_call( [&] {\n p = page_allocator_traits::allocate(page_allocator, 1);\n }).on_exception( [&] {\n ++base.n_invalid_entries;\n invalidate_page( k );\n });\n page_allocator_traits::construct(page_allocator, p);\n }\n\n spin_wait_until_my_turn(tail_counter, k, base);\n d1::call_itt_notify(d1::acquired, &tail_counter);\n\n if (p) {\n spin_mutex::scoped_lock lock( page_mutex );\n padded_page* q = tail_page.load(std::memory_order_relaxed);\n if (is_valid_page(q)) {\n q->next = p;\n } else {\n head_page.store(p, std::memory_order_relaxed);\n }\n tail_page.store(p, std::memory_order_relaxed);\n } else {\n p = tail_page.load(std::memory_order_relaxed);\n }\n return index;\n }\n\n template\n void push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator, Args&&... args )\n {\n padded_page* p = nullptr;\n page_allocator_type page_allocator(allocator);\n size_type index = prepare_page(k, base, page_allocator, p);\n __TBB_ASSERT(p != nullptr, \"Page was not prepared\");\n\n // try_call API is not convenient here due to broken\n // variadic capture on GCC 4.8.5\n auto value_guard = make_raii_guard([&] {\n ++base.n_invalid_entries;\n d1::call_itt_notify(d1::releasing, &tail_counter);\n tail_counter.fetch_add(queue_rep_type::n_queue);\n });\n\n page_allocator_traits::construct(page_allocator, &(*p)[index], std::forward(args)...);\n // If no exception was thrown, mark item as present.\n p->mask.store(p->mask.load(std::memory_order_relaxed) | uintptr_t(1) << index, std::memory_order_relaxed);\n d1::call_itt_notify(d1::releasing, &tail_counter);\n\n value_guard.dismiss();\n tail_counter.fetch_add(queue_rep_type::n_queue);\n }\n\n void abort_push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) {\n padded_page* p = nullptr;\n prepare_page(k, base, allocator, p);\n ++base.n_invalid_entries;\n tail_counter.fetch_add(queue_rep_type::n_queue);\n }\n\n bool pop( void* dst, ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) {\n k &= -queue_rep_type::n_queue;\n spin_wait_until_eq(head_counter, k);\n d1::call_itt_notify(d1::acquired, &head_counter);\n spin_wait_while_eq(tail_counter, k);\n d1::call_itt_notify(d1::acquired, &tail_counter);\n padded_page *p = head_page.load(std::memory_order_relaxed);\n __TBB_ASSERT( p, nullptr );\n size_type index = modulo_power_of_two( k/queue_rep_type::n_queue, items_per_page );\n bool success = false;\n {\n page_allocator_type page_allocator(allocator);\n micro_queue_pop_finalizer finalizer(*this, page_allocator,\n k + queue_rep_type::n_queue, index == items_per_page - 1 ? p : nullptr );\n if (p->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) {\n success = true;\n assign_and_destroy_item(dst, *p, index);\n } else {\n --base.n_invalid_entries;\n }\n }\n return success;\n }\n\n micro_queue& assign( const micro_queue& src, queue_allocator_type& allocator,\n item_constructor_type construct_item )\n {\n head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed);\n tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed);\n\n const padded_page* srcp = src.head_page.load(std::memory_order_relaxed);\n if( is_valid_page(srcp) ) {\n ticket_type g_index = head_counter.load(std::memory_order_relaxed);\n size_type n_items = (tail_counter.load(std::memory_order_relaxed) - head_counter.load(std::memory_order_relaxed))\n / queue_rep_type::n_queue;\n size_type index = modulo_power_of_two(head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page);\n size_type end_in_first_page = (index+n_items < items_per_page) ? (index + n_items) : items_per_page;\n\n try_call( [&] {\n head_page.store(make_copy(allocator, srcp, index, end_in_first_page, g_index, construct_item), std::memory_order_relaxed);\n }).on_exception( [&] {\n head_counter.store(0, std::memory_order_relaxed);\n tail_counter.store(0, std::memory_order_relaxed);\n });\n padded_page* cur_page = head_page.load(std::memory_order_relaxed);\n\n try_call( [&] {\n if (srcp != src.tail_page.load(std::memory_order_relaxed)) {\n for (srcp = srcp->next; srcp != src.tail_page.load(std::memory_order_relaxed); srcp=srcp->next ) {\n cur_page->next = make_copy( allocator, srcp, 0, items_per_page, g_index, construct_item );\n cur_page = cur_page->next;\n }\n\n __TBB_ASSERT(srcp == src.tail_page.load(std::memory_order_relaxed), nullptr );\n size_type last_index = modulo_power_of_two(tail_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page);\n if( last_index==0 ) last_index = items_per_page;\n\n cur_page->next = make_copy( allocator, srcp, 0, last_index, g_index, construct_item );\n cur_page = cur_page->next;\n }\n tail_page.store(cur_page, std::memory_order_relaxed);\n }).on_exception( [&] {\n padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1));\n tail_page.store(invalid_page, std::memory_order_relaxed);\n });\n } else {\n head_page.store(nullptr, std::memory_order_relaxed);\n tail_page.store(nullptr, std::memory_order_relaxed);\n }\n return *this;\n }\n\n padded_page* make_copy( queue_allocator_type& allocator, const padded_page* src_page, size_type begin_in_page,\n size_type end_in_page, ticket_type& g_index, item_constructor_type construct_item )\n {\n page_allocator_type page_allocator(allocator);\n padded_page* new_page = page_allocator_traits::allocate(page_allocator, 1);\n new_page->next = nullptr;\n new_page->mask.store(src_page->mask.load(std::memory_order_relaxed), std::memory_order_relaxed);\n for (; begin_in_page!=end_in_page; ++begin_in_page, ++g_index) {\n if (new_page->mask.load(std::memory_order_relaxed) & uintptr_t(1) << begin_in_page) {\n copy_item(*new_page, begin_in_page, *src_page, begin_in_page, construct_item);\n }\n }\n return new_page;\n }\n\n void invalidate_page( ticket_type k ) {\n // Append an invalid page at address 1 so that no more pushes are allowed.\n padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1));\n {\n spin_mutex::scoped_lock lock( page_mutex );\n tail_counter.store(k + queue_rep_type::n_queue + 1, std::memory_order_relaxed);\n padded_page* q = tail_page.load(std::memory_order_relaxed);\n if (is_valid_page(q)) {\n q->next = invalid_page;\n } else {\n head_page.store(invalid_page, std::memory_order_relaxed);\n }\n tail_page.store(invalid_page, std::memory_order_relaxed);\n }\n }\n\n padded_page* get_head_page() {\n return head_page.load(std::memory_order_relaxed);\n }\n\n void clear(queue_allocator_type& allocator, padded_page* new_head = nullptr, padded_page* new_tail = nullptr) {\n padded_page* curr_page = get_head_page();\n size_type index = (head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue) % items_per_page;\n page_allocator_type page_allocator(allocator);\n\n while (curr_page && is_valid_page(curr_page)) {\n while (index != items_per_page) {\n if (curr_page->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) {\n page_allocator_traits::destroy(page_allocator, &curr_page->operator[](index));\n }\n ++index;\n }\n\n index = 0;\n padded_page* next_page = curr_page->next;\n page_allocator_traits::destroy(page_allocator, curr_page);\n page_allocator_traits::deallocate(page_allocator, curr_page, 1);\n curr_page = next_page;\n }\n head_counter.store(0, std::memory_order_relaxed);\n tail_counter.store(0, std::memory_order_relaxed);\n head_page.store(new_head, std::memory_order_relaxed);\n tail_page.store(new_tail, std::memory_order_relaxed);\n }\n\n void clear_and_invalidate(queue_allocator_type& allocator) {\n padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1));\n clear(allocator, invalid_page, invalid_page);\n }\n\nprivate:\n // template \n friend class micro_queue_pop_finalizer;\n\n // Class used to ensure exception-safety of method \"pop\"\n class destroyer {\n value_type& my_value;\n public:\n destroyer( reference value ) : my_value(value) {}\n destroyer( const destroyer& ) = delete;\n destroyer& operator=( const destroyer& ) = delete;\n ~destroyer() {my_value.~T();}\n }; // class destroyer\n\n void copy_item( padded_page& dst, size_type dindex, const padded_page& src, size_type sindex,\n item_constructor_type construct_item )\n {\n auto& src_item = src[sindex];\n construct_item( &dst[dindex], static_cast(&src_item) );\n }\n\n void assign_and_destroy_item( void* dst, padded_page& src, size_type index ) {\n auto& from = src[index];\n destroyer d(from);\n *static_cast(dst) = std::move(from);\n }\n\n void spin_wait_until_my_turn( std::atomic& counter, ticket_type k, queue_rep_type& rb ) const {\n for (atomic_backoff b{};; b.pause()) {\n ticket_type c = counter.load(std::memory_order_acquire);\n if (c == k) return;\n else if (c & 1) {\n ++rb.n_invalid_entries;\n throw_exception( exception_id::bad_last_alloc);\n }\n }\n }\n\n std::atomic head_page{};\n std::atomic head_counter{};\n\n std::atomic tail_page{};\n std::atomic tail_counter{};\n\n spin_mutex page_mutex{};\n}; // class micro_queue\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// #pragma warning( pop )\n#endif // warning 4146 is back\n\ntemplate \nclass micro_queue_pop_finalizer {\npublic:\n using padded_page = typename Container::padded_page;\n using allocator_type = Allocator;\n using allocator_traits_type = tbb::detail::allocator_traits;\n\n micro_queue_pop_finalizer( Container& queue, Allocator& alloc, ticket_type k, padded_page* p ) :\n my_ticket_type(k), my_queue(queue), my_page(p), allocator(alloc)\n {}\n\n micro_queue_pop_finalizer( const micro_queue_pop_finalizer& ) = delete;\n micro_queue_pop_finalizer& operator=( const micro_queue_pop_finalizer& ) = delete;\n\n ~micro_queue_pop_finalizer() {\n padded_page* p = my_page;\n if( is_valid_page(p) ) {\n spin_mutex::scoped_lock lock( my_queue.page_mutex );\n padded_page* q = p->next;\n my_queue.head_page.store(q, std::memory_order_relaxed);\n if( !is_valid_page(q) ) {\n my_queue.tail_page.store(nullptr, std::memory_order_relaxed);\n }\n }\n my_queue.head_counter.store(my_ticket_type, std::memory_order_release);\n if ( is_valid_page(p) ) {\n allocator_traits_type::destroy(allocator, static_cast(p));\n allocator_traits_type::deallocate(allocator, static_cast(p), 1);\n }\n }\nprivate:\n ticket_type my_ticket_type;\n Container& my_queue;\n padded_page* my_page;\n Allocator& allocator;\n}; // class micro_queue_pop_finalizer\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// structure was padded due to alignment specifier\n// #pragma warning( push )\n// #pragma warning( disable: 4324 )\n#endif\n\ntemplate \nstruct concurrent_queue_rep {\n using self_type = concurrent_queue_rep;\n using size_type = std::size_t;\n using micro_queue_type = micro_queue;\n using allocator_type = Allocator;\n using allocator_traits_type = tbb::detail::allocator_traits;\n using padded_page = typename micro_queue_type::padded_page;\n using page_allocator_type = typename micro_queue_type::page_allocator_type;\n using item_constructor_type = typename micro_queue_type::item_constructor_type;\nprivate:\n using page_allocator_traits = tbb::detail::allocator_traits;\n using queue_allocator_type = typename allocator_traits_type::template rebind_alloc;\n\npublic:\n // must be power of 2\n static constexpr size_type n_queue = 8;\n // Approximately n_queue/golden ratio\n static constexpr size_type phi = 3;\n static constexpr size_type item_size = micro_queue_type::item_size;\n static constexpr size_type items_per_page = micro_queue_type::items_per_page;\n\n concurrent_queue_rep() {}\n\n concurrent_queue_rep( const concurrent_queue_rep& ) = delete;\n concurrent_queue_rep& operator=( const concurrent_queue_rep& ) = delete;\n\n void clear( queue_allocator_type& alloc ) {\n for (size_type index = 0; index < n_queue; ++index) {\n array[index].clear(alloc);\n }\n head_counter.store(0, std::memory_order_relaxed);\n tail_counter.store(0, std::memory_order_relaxed);\n n_invalid_entries.store(0, std::memory_order_relaxed);\n }\n\n void assign( const concurrent_queue_rep& src, queue_allocator_type& alloc, item_constructor_type construct_item ) {\n head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed);\n tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed);\n n_invalid_entries.store(src.n_invalid_entries.load(std::memory_order_relaxed), std::memory_order_relaxed);\n\n // copy or move micro_queues\n size_type queue_idx = 0;\n try_call( [&] {\n for (; queue_idx < n_queue; ++queue_idx) {\n array[queue_idx].assign(src.array[queue_idx], alloc, construct_item);\n }\n }).on_exception( [&] {\n for (size_type i = 0; i < queue_idx + 1; ++i) {\n array[i].clear_and_invalidate(alloc);\n }\n head_counter.store(0, std::memory_order_relaxed);\n tail_counter.store(0, std::memory_order_relaxed);\n n_invalid_entries.store(0, std::memory_order_relaxed);\n });\n\n __TBB_ASSERT(head_counter.load(std::memory_order_relaxed) == src.head_counter.load(std::memory_order_relaxed) &&\n tail_counter.load(std::memory_order_relaxed) == src.tail_counter.load(std::memory_order_relaxed),\n \"the source concurrent queue should not be concurrently modified.\" );\n }\n\n bool empty() const {\n ticket_type tc = tail_counter.load(std::memory_order_acquire);\n ticket_type hc = head_counter.load(std::memory_order_relaxed);\n // if tc!=r.tail_counter, the queue was not empty at some point between the two reads.\n return tc == tail_counter.load(std::memory_order_relaxed) &&\n std::ptrdiff_t(tc - hc - n_invalid_entries.load(std::memory_order_relaxed)) <= 0;\n }\n\n std::ptrdiff_t size() const {\n __TBB_ASSERT(sizeof(std::ptrdiff_t) <= sizeof(size_type), nullptr);\n std::ptrdiff_t hc = head_counter.load(std::memory_order_acquire);\n std::ptrdiff_t tc = tail_counter.load(std::memory_order_relaxed);\n std::ptrdiff_t nie = n_invalid_entries.load(std::memory_order_relaxed);\n\n return tc - hc - nie;\n }\n\n friend class micro_queue;\n\n // Map ticket_type to an array index\n static size_type index( ticket_type k ) {\n return k * phi % n_queue;\n }\n\n micro_queue_type& choose( ticket_type k ) {\n // The formula here approximates LRU in a cache-oblivious way.\n return array[index(k)];\n }\n\n alignas(max_nfs_size) micro_queue_type array[n_queue];\n\n alignas(max_nfs_size) std::atomic head_counter{};\n alignas(max_nfs_size) std::atomic tail_counter{};\n alignas(max_nfs_size) std::atomic n_invalid_entries{};\n}; // class concurrent_queue_rep\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// #pragma warning( pop )\n#endif\n\ntemplate \nclass concurrent_queue_iterator_base {\n using queue_rep_type = concurrent_queue_rep;\n using padded_page = typename queue_rep_type::padded_page;\nprotected:\n concurrent_queue_iterator_base() = default;\n\n concurrent_queue_iterator_base( const concurrent_queue_iterator_base& other ) {\n assign(other);\n }\n\n concurrent_queue_iterator_base( queue_rep_type* queue_rep )\n : my_queue_rep(queue_rep),\n my_head_counter(my_queue_rep->head_counter.load(std::memory_order_relaxed))\n {\n for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) {\n my_array[i] = my_queue_rep->array[i].get_head_page();\n }\n\n if (!get_item(my_item, my_head_counter)) advance();\n }\n\n void assign( const concurrent_queue_iterator_base& other ) {\n my_item = other.my_item;\n my_queue_rep = other.my_queue_rep;\n\n if (my_queue_rep != nullptr) {\n my_head_counter = other.my_head_counter;\n\n for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) {\n my_array[i] = other.my_array[i];\n }\n }\n }\n\n void advance() {\n __TBB_ASSERT(my_item, \"Attempt to increment iterator past end of the queue\");\n std::size_t k = my_head_counter;\n#if TBB_USE_ASSERT\n Value* tmp;\n get_item(tmp, k);\n __TBB_ASSERT(my_item == tmp, nullptr);\n#endif\n std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page);\n if (i == my_queue_rep->items_per_page - 1) {\n padded_page*& root = my_array[queue_rep_type::index(k)];\n root = root->next;\n }\n // Advance k\n my_head_counter = ++k;\n if (!get_item(my_item, k)) advance();\n }\n\n concurrent_queue_iterator_base& operator=( const concurrent_queue_iterator_base& other ) {\n this->assign(other);\n return *this;\n }\n\n bool get_item( Value*& item, std::size_t k ) {\n if (k == my_queue_rep->tail_counter.load(std::memory_order_relaxed)) {\n item = nullptr;\n return true;\n } else {\n padded_page* p = my_array[queue_rep_type::index(k)];\n __TBB_ASSERT(p, nullptr);\n std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page);\n item = &(*p)[i];\n return (p->mask & uintptr_t(1) << i) != 0;\n }\n }\n\n Value* my_item{ nullptr };\n queue_rep_type* my_queue_rep{ nullptr };\n ticket_type my_head_counter{};\n padded_page* my_array[queue_rep_type::n_queue]{};\n}; // class concurrent_queue_iterator_base\n\nstruct concurrent_queue_iterator_provider {\n template \n static Iterator get( const Container& container ) {\n return Iterator(container);\n }\n}; // struct concurrent_queue_iterator_provider\n\ntemplate \nclass concurrent_queue_iterator : public concurrent_queue_iterator_base::type, Allocator> {\n using base_type = concurrent_queue_iterator_base::type, Allocator>;\npublic:\n using value_type = Value;\n using pointer = value_type*;\n using reference = value_type&;\n using difference_type = std::ptrdiff_t;\n using iterator_category = std::forward_iterator_tag;\n\n concurrent_queue_iterator() = default;\n\n /** If Value==Container::value_type, then this routine is the copy constructor.\n If Value==const Container::value_type, then this routine is a conversion constructor. */\n concurrent_queue_iterator( const concurrent_queue_iterator& other )\n : base_type(other) {}\n\nprivate:\n concurrent_queue_iterator( const Container& container )\n : base_type(container.my_queue_representation) {}\npublic:\n concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) {\n this->assign(other);\n return *this;\n }\n\n reference operator*() const {\n return *static_cast(this->my_item);\n }\n\n pointer operator->() const { return &operator*(); }\n\n concurrent_queue_iterator& operator++() {\n this->advance();\n return *this;\n }\n\n concurrent_queue_iterator operator++(int) {\n concurrent_queue_iterator tmp = *this;\n ++*this;\n return tmp;\n }\n\n friend bool operator==( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) {\n return lhs.my_item == rhs.my_item;\n }\n\n friend bool operator!=( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) {\n return lhs.my_item != rhs.my_item;\n }\nprivate:\n friend struct concurrent_queue_iterator_provider;\n}; // class concurrent_queue_iterator\n\n} // namespace d2\n} // namespace detail\n} // tbb\n\n#endif // __TBB_detail__concurrent_queue_base_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_concurrent_skip_list.h", "content": "/*\n Copyright (c) 2019-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__concurrent_skip_list_H\n#define __TBB_detail__concurrent_skip_list_H\n\n#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H)\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n#include \"_config.h\"\n#include \"_range_common.h\"\n#include \"_allocator_traits.h\"\n#include \"_template_helpers.h\"\n#include \"_node_handle.h\"\n#include \"_containers_helpers.h\"\n#include \"_assert.h\"\n#include \"_exception.h\"\n#include \"../enumerable_thread_specific.h\"\n#include \n#include \n#include \n#include \n#include \n#include // Need std::geometric_distribution\n#include // Need std::equal and std::lexicographical_compare\n#include \n#if __TBB_CPP20_COMPARISONS_PRESENT\n#include \n#endif\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(push)\n// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nclass skip_list_node {\n using node_ptr = skip_list_node*;\npublic:\n using value_type = Value;\n using atomic_node_ptr = std::atomic;\n using size_type = std::size_t;\n using container_allocator_type = Allocator;\n\n using reference = value_type&;\n using const_reference = const value_type&;\nprivate:\n using allocator_traits = tbb::detail::allocator_traits;\n\n // Allocator is the same as the container allocator=> allocates unitptr_t\n // It is required to rebind it to value_type to get the correct pointer and const_pointer\n using value_allocator_traits = typename allocator_traits::template rebind_traits;\npublic:\n using pointer = typename value_allocator_traits::pointer;\n using const_pointer = typename value_allocator_traits::const_pointer;\n\n //In perfect world these constructor and destructor would have been private,\n //however this seems technically impractical due to use of allocator_traits.\n\n //Should not be called directly, instead use create method\n skip_list_node( size_type levels )\n : my_height(levels), my_index_number(0)\n {}\n\n //Should not be called directly, instead use destroy method\n ~skip_list_node() {}\n\n skip_list_node( const skip_list_node& ) = delete;\n skip_list_node( skip_list_node&& ) = delete;\n skip_list_node& operator=( const skip_list_node& ) = delete;\n skip_list_node& operator=( skip_list_node&& ) = delete;\n\n static skip_list_node* create( container_allocator_type& alloc, size_type height ) {\n size_type sz = calc_node_size(height);\n static_assert(std::is_same::value, \"skip_list_node assumes that passed in allocator operates on bytes\");\n auto* node = reinterpret_cast(allocator_traits::allocate(alloc, sz));\n\n //Construct the node itself\n allocator_traits::construct(alloc, node, height);\n\n //Construct the level pointers\n for (size_type l = 0; l < height; ++l) {\n allocator_traits::construct(alloc, &node->get_atomic_next(l), nullptr);\n }\n\n return node;\n }\n\n static void destroy( container_allocator_type& alloc, skip_list_node* node ) {\n //Destroy the level pointers\n for (size_type l = 0; l < node->height(); ++l) {\n allocator_traits::destroy(alloc, &node->atomic_next(l));\n }\n size_type sz = calc_node_size(node->height());\n // Destroy the node itself\n allocator_traits::destroy(alloc, node);\n\n // Deallocate the node\n allocator_traits::deallocate(alloc, reinterpret_cast(node), sz);\n }\n\n\n pointer storage() {\n return &my_value;\n }\n\n reference value() {\n return *storage();\n }\n\n node_ptr next( size_type level ) const {\n node_ptr res = get_atomic_next(level).load(std::memory_order_acquire);\n __TBB_ASSERT(res == nullptr || res->height() > level, \"Broken internal structure\");\n return res;\n }\n\n atomic_node_ptr& atomic_next( size_type level ) {\n atomic_node_ptr& res = get_atomic_next(level);\n#if TBB_USE_DEBUG\n node_ptr node = res.load(std::memory_order_acquire);\n __TBB_ASSERT(node == nullptr || node->height() > level, \"Broken internal structure\");\n#endif\n return res;\n }\n\n void set_next( size_type level, node_ptr n ) {\n __TBB_ASSERT(n == nullptr || n->height() > level, \"Broken internal structure\");\n get_atomic_next(level).store(n, std::memory_order_relaxed);\n }\n\n size_type height() const {\n return my_height;\n }\n\n void set_index_number( size_type index_num ) {\n my_index_number = index_num;\n }\n\n size_type index_number() const {\n return my_index_number;\n }\n\nprivate:\n static size_type calc_node_size( size_type height ) {\n static_assert(alignof(skip_list_node) >= alignof(atomic_node_ptr), \"Incorrect alignment\");\n return sizeof(skip_list_node) + height * sizeof(atomic_node_ptr);\n }\n\n atomic_node_ptr& get_atomic_next( size_type level ) {\n atomic_node_ptr* arr = reinterpret_cast(this + 1);\n return arr[level];\n }\n\n const atomic_node_ptr& get_atomic_next( size_type level ) const {\n const atomic_node_ptr* arr = reinterpret_cast(this + 1);\n return arr[level];\n }\n\n union {\n value_type my_value;\n };\n size_type my_height;\n size_type my_index_number;\n}; // class skip_list_node\n\ntemplate \nclass skip_list_iterator {\n using node_type = NodeType;\n using node_ptr = node_type*;\npublic:\n using iterator_category = std::forward_iterator_tag;\n using value_type = ValueType;\n\n using difference_type = std::ptrdiff_t;\n using pointer = value_type*;\n using reference = value_type&;\n\n skip_list_iterator() : skip_list_iterator(nullptr) {}\n\n skip_list_iterator( const skip_list_iterator& other )\n : my_node_ptr(other.my_node_ptr) {}\n\n skip_list_iterator& operator=( const skip_list_iterator& other ) {\n my_node_ptr = other.my_node_ptr;\n return *this;\n }\n\n reference operator*() const { return my_node_ptr->value(); }\n pointer operator->() const { return my_node_ptr->storage(); }\n\n skip_list_iterator& operator++() {\n __TBB_ASSERT(my_node_ptr != nullptr, nullptr);\n my_node_ptr = my_node_ptr->next(0);\n return *this;\n }\n\n skip_list_iterator operator++(int) {\n skip_list_iterator tmp = *this;\n ++*this;\n return tmp;\n }\n\nprivate:\n skip_list_iterator(node_type* n) : my_node_ptr(n) {}\n\n node_ptr my_node_ptr;\n\n template \n friend class concurrent_skip_list;\n\n template \n friend class skip_list_iterator;\n\n friend class const_range;\n friend class range;\n\n friend bool operator==( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) {\n return lhs.my_node_ptr == rhs.my_node_ptr;\n }\n\n friend bool operator!=( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) {\n return lhs.my_node_ptr != rhs.my_node_ptr;\n }\n}; // class skip_list_iterator\n\ntemplate \nclass concurrent_skip_list {\nprotected:\n using container_traits = Traits;\n using self_type = concurrent_skip_list;\n using allocator_type = typename container_traits::allocator_type;\n using allocator_traits_type = tbb::detail::allocator_traits;\n using key_compare = typename container_traits::compare_type;\n using value_compare = typename container_traits::value_compare;\n using key_type = typename container_traits::key_type;\n using value_type = typename container_traits::value_type;\n static_assert(std::is_same::value,\n \"value_type of the container should be the same as its allocator\");\n\n using size_type = std::size_t;\n using difference_type = std::ptrdiff_t;\n\n static constexpr size_type max_level = container_traits::max_level;\n\n using node_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using node_allocator_traits = tbb::detail::allocator_traits;\n\n using list_node_type = skip_list_node;\n using node_type = d1::node_handle;\n\n using iterator = skip_list_iterator;\n using const_iterator = skip_list_iterator;\n\n using reference = value_type&;\n using const_reference = const value_type&;\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using random_level_generator_type = typename container_traits::random_level_generator_type;\n\n using node_ptr = list_node_type*;\n\n using array_type = std::array;\nprivate:\n template \n using is_transparent = dependent_bool, T>;\npublic:\n static constexpr bool allow_multimapping = container_traits::allow_multimapping;\n\n concurrent_skip_list() : my_head_ptr(nullptr), my_size(0), my_max_height(0) {}\n\n explicit concurrent_skip_list( const key_compare& comp, const allocator_type& alloc = allocator_type() )\n : my_node_allocator(alloc), my_compare(comp), my_head_ptr(nullptr), my_size(0), my_max_height(0) {}\n\n explicit concurrent_skip_list( const allocator_type& alloc )\n : concurrent_skip_list(key_compare(), alloc) {}\n\n template\n concurrent_skip_list( InputIterator first, InputIterator last, const key_compare& comp = key_compare(),\n const allocator_type& alloc = allocator_type() )\n : concurrent_skip_list(comp, alloc)\n {\n internal_copy(first, last);\n }\n\n template \n concurrent_skip_list( InputIterator first, InputIterator last, const allocator_type& alloc )\n : concurrent_skip_list(first, last, key_compare(), alloc) {}\n\n concurrent_skip_list( std::initializer_list init, const key_compare& comp = key_compare(),\n const allocator_type& alloc = allocator_type() )\n : concurrent_skip_list(init.begin(), init.end(), comp, alloc) {}\n\n concurrent_skip_list( std::initializer_list init, const allocator_type& alloc )\n : concurrent_skip_list(init, key_compare(), alloc) {}\n\n concurrent_skip_list( const concurrent_skip_list& other )\n : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())),\n my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr),\n my_size(0), my_max_height(0)\n {\n internal_copy(other);\n __TBB_ASSERT(my_size == other.my_size, \"Wrong size of copy-constructed container\");\n }\n\n concurrent_skip_list( const concurrent_skip_list& other, const allocator_type& alloc )\n : my_node_allocator(alloc), my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr),\n my_size(0), my_max_height(0)\n {\n internal_copy(other);\n __TBB_ASSERT(my_size == other.my_size, \"Wrong size of copy-constructed container\");\n }\n\n concurrent_skip_list( concurrent_skip_list&& other )\n : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare),\n my_rng(std::move(other.my_rng)), my_head_ptr(nullptr) // my_head_ptr would be stored in internal_move\n {\n internal_move(std::move(other));\n }\n\n concurrent_skip_list( concurrent_skip_list&& other, const allocator_type& alloc )\n : my_node_allocator(alloc), my_compare(other.my_compare),\n my_rng(std::move(other.my_rng)), my_head_ptr(nullptr)\n {\n using is_always_equal = typename allocator_traits_type::is_always_equal;\n internal_move_construct_with_allocator(std::move(other), is_always_equal());\n }\n\n ~concurrent_skip_list() {\n clear();\n delete_head();\n }\n\n concurrent_skip_list& operator=( const concurrent_skip_list& other ) {\n if (this != &other) {\n clear();\n copy_assign_allocators(my_node_allocator, other.my_node_allocator);\n my_compare = other.my_compare;\n my_rng = other.my_rng;\n internal_copy(other);\n }\n return *this;\n }\n\n concurrent_skip_list& operator=( concurrent_skip_list&& other ) {\n if (this != &other) {\n clear();\n delete_head();\n\n my_compare = std::move(other.my_compare);\n my_rng = std::move(other.my_rng);\n\n move_assign_allocators(my_node_allocator, other.my_node_allocator);\n using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment;\n using is_always_equal = typename node_allocator_traits::is_always_equal;\n internal_move_assign(std::move(other), tbb::detail::disjunction());\n }\n return *this;\n }\n\n concurrent_skip_list& operator=( std::initializer_list il )\n {\n clear();\n insert(il.begin(),il.end());\n return *this;\n }\n\n std::pair insert( const value_type& value ) {\n return internal_insert(value);\n }\n\n std::pair insert( value_type&& value ) {\n return internal_insert(std::move(value));\n }\n\n iterator insert( const_iterator, const_reference value ) {\n // Ignore hint\n return insert(value).first;\n }\n\n iterator insert( const_iterator, value_type&& value ) {\n // Ignore hint\n return insert(std::move(value)).first;\n }\n\n template\n void insert( InputIterator first, InputIterator last ) {\n while (first != last) {\n insert(*first);\n ++first;\n }\n }\n\n void insert( std::initializer_list init ) {\n insert(init.begin(), init.end());\n }\n\n std::pair insert( node_type&& nh ) {\n if (!nh.empty()) {\n auto insert_node = d1::node_handle_accessor::get_node_ptr(nh);\n std::pair insert_result = internal_insert_node(insert_node);\n if (insert_result.second) {\n d1::node_handle_accessor::deactivate(nh);\n }\n return insert_result;\n }\n return std::pair(end(), false);\n }\n\n iterator insert( const_iterator, node_type&& nh ) {\n // Ignore hint\n return insert(std::move(nh)).first;\n }\n\n template\n std::pair emplace( Args&&... args ) {\n return internal_insert(std::forward(args)...);\n }\n\n template\n iterator emplace_hint( const_iterator, Args&&... args ) {\n // Ignore hint\n return emplace(std::forward(args)...).first;\n }\n\n iterator unsafe_erase( iterator pos ) {\n std::pair extract_result = internal_extract(pos);\n if (extract_result.first) { // node was extracted\n delete_value_node(extract_result.first);\n return extract_result.second;\n }\n return end();\n }\n\n iterator unsafe_erase( const_iterator pos ) {\n return unsafe_erase(get_iterator(pos));\n }\n\n iterator unsafe_erase( const_iterator first, const_iterator last ) {\n while (first != last) {\n // Unsafe erase returns the iterator which follows the erased one\n first = unsafe_erase(first);\n }\n return get_iterator(first);\n }\n\n size_type unsafe_erase( const key_type& key ) {\n return internal_erase(key);\n }\n\n template \n typename std::enable_if::value\n && !std::is_convertible::value\n && !std::is_convertible::value,\n size_type>::type unsafe_erase( const K& key )\n {\n return internal_erase(key);\n }\n\n node_type unsafe_extract( const_iterator pos ) {\n std::pair extract_result = internal_extract(pos);\n return extract_result.first ? d1::node_handle_accessor::construct(extract_result.first) : node_type();\n }\n\n node_type unsafe_extract( iterator pos ) {\n return unsafe_extract(const_iterator(pos));\n }\n\n node_type unsafe_extract( const key_type& key ) {\n return unsafe_extract(find(key));\n }\n\n template \n typename std::enable_if::value\n && !std::is_convertible::value\n && !std::is_convertible::value,\n node_type>::type unsafe_extract( const K& key )\n {\n return unsafe_extract(find(key));\n }\n\n iterator lower_bound( const key_type& key ) {\n return iterator(internal_get_bound(key, my_compare));\n }\n\n const_iterator lower_bound( const key_type& key ) const {\n return const_iterator(internal_get_bound(key, my_compare));\n }\n\n template \n typename std::enable_if::value, iterator>::type lower_bound( const K& key ) {\n return iterator(internal_get_bound(key, my_compare));\n }\n\n template \n typename std::enable_if::value, const_iterator>::type lower_bound( const K& key ) const {\n return const_iterator(internal_get_bound(key, my_compare));\n }\n\n iterator upper_bound( const key_type& key ) {\n return iterator(internal_get_bound(key, not_greater_compare(my_compare)));\n }\n\n const_iterator upper_bound( const key_type& key ) const {\n return const_iterator(internal_get_bound(key, not_greater_compare(my_compare)));\n }\n\n template \n typename std::enable_if::value, iterator>::type upper_bound( const K& key ) {\n return iterator(internal_get_bound(key, not_greater_compare(my_compare)));\n }\n\n template \n typename std::enable_if::value, const_iterator>::type upper_bound( const K& key ) const {\n return const_iterator(internal_get_bound(key, not_greater_compare(my_compare)));\n }\n\n iterator find( const key_type& key ) {\n return iterator(internal_find(key));\n }\n\n const_iterator find( const key_type& key ) const {\n return const_iterator(internal_find(key));\n }\n\n template \n typename std::enable_if::value, iterator>::type find( const K& key ) {\n return iterator(internal_find(key));\n }\n\n template \n typename std::enable_if::value, const_iterator>::type find( const K& key ) const {\n return const_iterator(internal_find(key));\n }\n\n size_type count( const key_type& key ) const {\n return internal_count(key);\n }\n\n template \n typename std::enable_if::value, size_type>::type count( const K& key ) const {\n return internal_count(key);\n }\n\n bool contains( const key_type& key ) const {\n return find(key) != end();\n }\n\n template \n typename std::enable_if::value, bool>::type contains( const K& key ) const {\n return find(key) != end();\n }\n\n void clear() noexcept {\n // clear is not thread safe - load can be relaxed\n node_ptr head = my_head_ptr.load(std::memory_order_relaxed);\n\n if (head == nullptr) return; // Head is not allocated => container is empty\n\n node_ptr current = head->next(0);\n\n // Delete all value nodes in the container\n while (current) {\n node_ptr next = current->next(0);\n delete_value_node(current);\n current = next;\n }\n\n for (size_type level = 0; level < head->height(); ++level) {\n head->set_next(level, nullptr);\n }\n\n my_size.store(0, std::memory_order_relaxed);\n my_max_height.store(0, std::memory_order_relaxed);\n }\n\n iterator begin() {\n return iterator(internal_begin());\n }\n\n const_iterator begin() const {\n return const_iterator(internal_begin());\n }\n\n const_iterator cbegin() const {\n return const_iterator(internal_begin());\n }\n\n iterator end() {\n return iterator(nullptr);\n }\n\n const_iterator end() const {\n return const_iterator(nullptr);\n }\n\n const_iterator cend() const {\n return const_iterator(nullptr);\n }\n\n size_type size() const {\n return my_size.load(std::memory_order_relaxed);\n }\n\n size_type max_size() const {\n return node_allocator_traits::max_size(my_node_allocator);\n }\n\n __TBB_nodiscard bool empty() const {\n return 0 == size();\n }\n\n allocator_type get_allocator() const {\n return my_node_allocator;\n }\n\n void swap(concurrent_skip_list& other) {\n if (this != &other) {\n using pocs_type = typename node_allocator_traits::propagate_on_container_swap;\n using is_always_equal = typename node_allocator_traits::is_always_equal;\n internal_swap(other, tbb::detail::disjunction());\n }\n }\n\n std::pair equal_range(const key_type& key) {\n return internal_equal_range(key);\n }\n\n std::pair equal_range(const key_type& key) const {\n return internal_equal_range(key);\n }\n\n template \n typename std::enable_if::value, std::pair>::type equal_range( const K& key ) {\n return internal_equal_range(key);\n }\n\n template \n typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const {\n return internal_equal_range(key);\n }\n\n key_compare key_comp() const { return my_compare; }\n\n value_compare value_comp() const { return container_traits::value_comp(my_compare); }\n\n class const_range_type {\n public:\n using size_type = typename concurrent_skip_list::size_type;\n using difference_type = typename concurrent_skip_list::difference_type;\n using iterator = typename concurrent_skip_list::const_iterator;\n using value_type = typename iterator::value_type;\n using reference = typename iterator::reference;\n\n bool empty() const {\n return my_begin.my_node_ptr ? (my_begin.my_node_ptr->next(0) == my_end.my_node_ptr)\n : true;\n }\n\n bool is_divisible() const {\n return my_begin.my_node_ptr && my_level != 0\n ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr\n : false;\n }\n\n size_type size() const { return std::distance(my_begin, my_end); }\n\n const_range_type( const_range_type& r, split)\n : my_end(r.my_end) {\n if (r.empty()) {\n __TBB_ASSERT(my_end.my_node_ptr == nullptr, nullptr);\n my_begin = my_end;\n my_level = 0;\n } else {\n my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1));\n my_level = my_begin.my_node_ptr->height();\n }\n r.my_end = my_begin;\n }\n\n const_range_type( const concurrent_skip_list& l)\n : my_end(l.end()), my_begin(l.begin()),\n my_level(my_begin.my_node_ptr ? my_begin.my_node_ptr->height() : 0) {}\n\n iterator begin() const { return my_begin; }\n iterator end() const { return my_end; }\n size_type grainsize() const { return 1; }\n\n private:\n const_iterator my_end;\n const_iterator my_begin;\n size_type my_level;\n }; // class const_range_type\n\n class range_type : public const_range_type {\n public:\n using iterator = typename concurrent_skip_list::iterator;\n using value_type = typename iterator::value_type;\n using reference = typename iterator::reference;\n\n range_type(range_type& r, split) : const_range_type(r, split()) {}\n range_type(const concurrent_skip_list& l) : const_range_type(l) {}\n\n iterator begin() const {\n node_ptr node = const_range_type::begin().my_node_ptr;\n return iterator(node);\n }\n\n iterator end() const {\n node_ptr node = const_range_type::end().my_node_ptr;\n return iterator(node);\n }\n }; // class range_type\n\n range_type range() { return range_type(*this); }\n const_range_type range() const { return const_range_type(*this); }\n\nprivate:\n node_ptr internal_begin() const {\n node_ptr head = get_head();\n return head == nullptr ? head : head->next(0);\n }\n\n void internal_move(concurrent_skip_list&& other) {\n my_head_ptr.store(other.my_head_ptr.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_head_ptr.store(nullptr, std::memory_order_relaxed);\n\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_size.store(0, std::memory_order_relaxed);\n\n my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_max_height.store(0, std::memory_order_relaxed);\n }\n\n void internal_move_construct_with_allocator(concurrent_skip_list&& other,\n /*is_always_equal = */std::true_type) {\n internal_move(std::move(other));\n }\n\n void internal_move_construct_with_allocator(concurrent_skip_list&& other,\n /*is_always_equal = */std::false_type) {\n if (my_node_allocator == other.get_allocator()) {\n internal_move(std::move(other));\n } else {\n my_size.store(0, std::memory_order_relaxed);\n my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed);\n internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()));\n }\n }\n\n static const key_type& get_key( node_ptr n ) {\n __TBB_ASSERT(n, nullptr);\n return container_traits::get_key(static_cast(n)->value());\n }\n\n template \n bool found( node_ptr node, const K& key ) const {\n return node != nullptr && !my_compare(key, get_key(node));\n }\n\n template \n node_ptr internal_find(const K& key) const {\n return allow_multimapping ? internal_find_multi(key) : internal_find_unique(key);\n }\n\n template \n node_ptr internal_find_multi( const K& key ) const {\n node_ptr prev = get_head();\n if (prev == nullptr) return nullptr; // If the head node is not allocated - exit\n\n node_ptr curr = nullptr;\n node_ptr old_curr = curr;\n\n for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) {\n curr = internal_find_position(h - 1, prev, key, my_compare);\n\n if (curr != old_curr && found(curr, key)) {\n return curr;\n }\n old_curr = curr;\n }\n return nullptr;\n }\n\n template \n node_ptr internal_find_unique( const K& key ) const {\n const_iterator it = lower_bound(key);\n return (it == end() || my_compare(key, container_traits::get_key(*it))) ? nullptr : it.my_node_ptr;\n }\n\n template \n size_type internal_count( const K& key ) const {\n if (allow_multimapping) {\n // TODO: reimplement without double traversal\n std::pair r = equal_range(key);\n return std::distance(r.first, r.second);\n }\n return size_type(contains(key) ? 1 : 0);\n }\n\n template \n std::pair internal_equal_range(const K& key) const {\n iterator lb = get_iterator(lower_bound(key));\n auto result = std::make_pair(lb, lb);\n\n // If the lower bound points to the node with the requested key\n if (found(lb.my_node_ptr, key)) {\n\n if (!allow_multimapping) {\n // For unique containers - move the second iterator forward and exit\n ++result.second;\n } else {\n // For multi containers - find the upper bound starting from the lower bound\n node_ptr prev = lb.my_node_ptr;\n node_ptr curr = nullptr;\n not_greater_compare cmp(my_compare);\n\n // Start from the lower bound of the range\n for (size_type h = prev->height(); h > 0; --h) {\n curr = prev->next(h - 1);\n while (curr && cmp(get_key(curr), key)) {\n prev = curr;\n // If the height of the next node is greater than the current one - jump to its height\n if (h < curr->height()) {\n h = curr->height();\n }\n curr = prev->next(h - 1);\n }\n }\n result.second = iterator(curr);\n }\n }\n\n return result;\n }\n\n // Finds position on the level using comparator cmp starting from the node prev\n template \n node_ptr internal_find_position( size_type level, node_ptr& prev, const K& key,\n const Comparator& cmp ) const {\n __TBB_ASSERT(level < prev->height(), \"Wrong level to find position\");\n node_ptr curr = prev->next(level);\n\n while (curr && cmp(get_key(curr), key)) {\n prev = curr;\n __TBB_ASSERT(level < prev->height(), nullptr);\n curr = prev->next(level);\n }\n\n return curr;\n }\n\n // The same as previous overload, but allows index_number comparison\n template \n node_ptr internal_find_position( size_type level, node_ptr& prev, node_ptr node,\n const Comparator& cmp ) const {\n __TBB_ASSERT(level < prev->height(), \"Wrong level to find position\");\n node_ptr curr = prev->next(level);\n\n while (curr && cmp(get_key(curr), get_key(node))) {\n if (allow_multimapping && cmp(get_key(node), get_key(curr)) && curr->index_number() > node->index_number()) {\n break;\n }\n\n prev = curr;\n __TBB_ASSERT(level < prev->height(), nullptr);\n curr = prev->next(level);\n }\n return curr;\n }\n\n template \n void fill_prev_curr_arrays(array_type& prev_nodes, array_type& curr_nodes, node_ptr node, const key_type& key,\n const Comparator& cmp, node_ptr head ) {\n\n size_type curr_max_height = my_max_height.load(std::memory_order_acquire);\n size_type node_height = node->height();\n if (curr_max_height < node_height) {\n std::fill(prev_nodes.begin() + curr_max_height, prev_nodes.begin() + node_height, head);\n std::fill(curr_nodes.begin() + curr_max_height, curr_nodes.begin() + node_height, nullptr);\n }\n\n node_ptr prev = head;\n for (size_type level = curr_max_height; level > 0; --level) {\n node_ptr curr = internal_find_position(level - 1, prev, key, cmp);\n prev_nodes[level - 1] = prev;\n curr_nodes[level - 1] = curr;\n }\n }\n\n void fill_prev_array_for_existing_node( array_type& prev_nodes, node_ptr node ) {\n node_ptr head = create_head_if_necessary();\n prev_nodes.fill(head);\n\n node_ptr prev = head;\n for (size_type level = node->height(); level > 0; --level) {\n while (prev->next(level - 1) != node) {\n prev = prev->next(level - 1);\n }\n prev_nodes[level - 1] = prev;\n }\n }\n\n struct not_greater_compare {\n const key_compare& my_less_compare;\n\n not_greater_compare( const key_compare& less_compare ) : my_less_compare(less_compare) {}\n\n template \n bool operator()( const K1& first, const K2& second ) const {\n return !my_less_compare(second, first);\n }\n };\n\n not_greater_compare select_comparator( /*allow_multimapping = */ std::true_type ) {\n return not_greater_compare(my_compare);\n }\n\n key_compare select_comparator( /*allow_multimapping = */ std::false_type ) {\n return my_compare;\n }\n\n template\n std::pair internal_insert( Args&&... args ) {\n node_ptr new_node = create_value_node(std::forward(args)...);\n std::pair insert_result = internal_insert_node(new_node);\n if (!insert_result.second) {\n delete_value_node(new_node);\n }\n return insert_result;\n }\n\n std::pair internal_insert_node( node_ptr new_node ) {\n array_type prev_nodes;\n array_type curr_nodes;\n size_type new_height = new_node->height();\n auto compare = select_comparator(std::integral_constant{});\n\n node_ptr head_node = create_head_if_necessary();\n\n for (;;) {\n fill_prev_curr_arrays(prev_nodes, curr_nodes, new_node, get_key(new_node), compare, head_node);\n\n node_ptr prev = prev_nodes[0];\n node_ptr next = curr_nodes[0];\n\n if (allow_multimapping) {\n new_node->set_index_number(prev->index_number() + 1);\n } else {\n if (found(next, get_key(new_node))) {\n return std::pair(iterator(next), false);\n }\n }\n\n new_node->set_next(0, next);\n if (!prev->atomic_next(0).compare_exchange_strong(next, new_node)) {\n continue;\n }\n\n // If the node was successfully linked on the first level - it will be linked on other levels\n // Insertion cannot fail starting from this point\n\n // If the height of inserted node is greater than maximum - increase maximum\n size_type max_height = my_max_height.load(std::memory_order_acquire);\n for (;;) {\n if (new_height <= max_height || my_max_height.compare_exchange_strong(max_height, new_height)) {\n // If the maximum was successfully updated by current thread\n // or by an other thread for the value, greater or equal to new_height\n break;\n }\n }\n\n for (std::size_t level = 1; level < new_height; ++level) {\n // Link the node on upper levels\n for (;;) {\n prev = prev_nodes[level];\n next = static_cast(curr_nodes[level]);\n\n new_node->set_next(level, next);\n __TBB_ASSERT(new_node->height() > level, \"Internal structure break\");\n if (prev->atomic_next(level).compare_exchange_strong(next, new_node)) {\n break;\n }\n\n for (size_type lev = level; lev != new_height; ++lev ) {\n curr_nodes[lev] = internal_find_position(lev, prev_nodes[lev], new_node, compare);\n }\n }\n }\n ++my_size;\n return std::pair(iterator(new_node), true);\n }\n }\n\n template \n node_ptr internal_get_bound( const K& key, const Comparator& cmp ) const {\n node_ptr prev = get_head();\n if (prev == nullptr) return nullptr; // If the head node is not allocated - exit\n\n node_ptr curr = nullptr;\n\n for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) {\n curr = internal_find_position(h - 1, prev, key, cmp);\n }\n\n return curr;\n }\n\n template \n size_type internal_erase( const K& key ) {\n auto eq = equal_range(key);\n size_type old_size = size();\n unsafe_erase(eq.first, eq.second);\n return old_size - size();\n }\n\n // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted\n std::pair internal_extract( const_iterator it ) {\n std::pair result(nullptr, nullptr);\n if ( it != end() ) {\n array_type prev_nodes;\n\n node_ptr erase_node = it.my_node_ptr;\n node_ptr next_node = erase_node->next(0);\n fill_prev_array_for_existing_node(prev_nodes, erase_node);\n\n for (size_type level = 0; level < erase_node->height(); ++level) {\n prev_nodes[level]->set_next(level, erase_node->next(level));\n erase_node->set_next(level, nullptr);\n }\n my_size.fetch_sub(1, std::memory_order_relaxed);\n\n result.first = erase_node;\n result.second = next_node;\n }\n return result;\n }\n\nprotected:\n template\n void internal_merge( SourceType&& source ) {\n using source_type = typename std::decay::type;\n using source_iterator = typename source_type::iterator;\n static_assert((std::is_same::value), \"Incompatible containers cannot be merged\");\n\n for (source_iterator it = source.begin(); it != source.end();) {\n source_iterator where = it++;\n if (allow_multimapping || !contains(container_traits::get_key(*where))) {\n node_type handle = source.unsafe_extract(where);\n __TBB_ASSERT(!handle.empty(), \"Extracted handle in merge is empty\");\n\n if (!insert(std::move(handle)).second) {\n __TBB_ASSERT(!handle.empty(), \"Handle should not be empty if insert fails\");\n //If the insertion fails - return the node into source\n source.insert(std::move(handle));\n }\n __TBB_ASSERT(handle.empty(), \"Node handle should be empty after the insertion\");\n }\n }\n }\n\nprivate:\n void internal_copy( const concurrent_skip_list& other ) {\n internal_copy(other.begin(), other.end());\n }\n\n template\n void internal_copy( Iterator first, Iterator last ) {\n try_call([&] {\n for (auto it = first; it != last; ++it) {\n insert(*it);\n }\n }).on_exception([&] {\n clear();\n delete_head();\n });\n }\n\n node_ptr create_node( size_type height ) {\n return list_node_type::create(my_node_allocator, height);\n }\n\n template \n node_ptr create_value_node( Args&&... args ) {\n node_ptr node = create_node(my_rng());\n\n // try_call API is not convenient here due to broken\n // variadic capture on GCC 4.8.5\n auto value_guard = make_raii_guard([&] {\n delete_node(node);\n });\n\n // Construct the value inside the node\n node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...);\n value_guard.dismiss();\n return node;\n }\n\n node_ptr create_head_node() {\n return create_node(max_level);\n }\n\n void delete_head() {\n node_ptr head = my_head_ptr.load(std::memory_order_relaxed);\n if (head != nullptr) {\n delete_node(head);\n my_head_ptr.store(nullptr, std::memory_order_relaxed);\n }\n }\n\n void delete_node( node_ptr node ) {\n list_node_type::destroy(my_node_allocator, node);\n }\n\n void delete_value_node( node_ptr node ) {\n // Destroy the value inside the node\n node_allocator_traits::destroy(my_node_allocator, node->storage());\n delete_node(node);\n }\n\n node_ptr get_head() const {\n return my_head_ptr.load(std::memory_order_acquire);\n }\n\n node_ptr create_head_if_necessary() {\n node_ptr current_head = get_head();\n if (current_head == nullptr) {\n // Head node was not created - create it\n node_ptr new_head = create_head_node();\n if (my_head_ptr.compare_exchange_strong(current_head, new_head)) {\n current_head = new_head;\n } else {\n // If an other thread has already created the head node - destroy new_head\n // current_head now points to the actual head node\n delete_node(new_head);\n }\n }\n __TBB_ASSERT(my_head_ptr.load(std::memory_order_relaxed) != nullptr, nullptr);\n __TBB_ASSERT(current_head != nullptr, nullptr);\n return current_head;\n }\n\n static iterator get_iterator( const_iterator it ) {\n return iterator(it.my_node_ptr);\n }\n\n void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::true_type ) {\n internal_move(std::move(other));\n }\n\n void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::false_type ) {\n if (my_node_allocator == other.my_node_allocator) {\n internal_move(std::move(other));\n } else {\n internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()));\n }\n }\n\n void internal_swap_fields( concurrent_skip_list& other ) {\n using std::swap;\n swap_allocators(my_node_allocator, other.my_node_allocator);\n swap(my_compare, other.my_compare);\n swap(my_rng, other.my_rng);\n\n swap_atomics_relaxed(my_head_ptr, other.my_head_ptr);\n swap_atomics_relaxed(my_size, other.my_size);\n swap_atomics_relaxed(my_max_height, other.my_max_height);\n }\n\n void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::true_type ) {\n internal_swap_fields(other);\n }\n\n void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::false_type ) {\n __TBB_ASSERT(my_node_allocator == other.my_node_allocator, \"Swapping with unequal allocators is not allowed\");\n internal_swap_fields(other);\n }\n\n node_allocator_type my_node_allocator;\n key_compare my_compare;\n random_level_generator_type my_rng;\n std::atomic my_head_ptr;\n std::atomic my_size;\n std::atomic my_max_height;\n\n template\n friend class concurrent_skip_list;\n}; // class concurrent_skip_list\n\ntemplate \nbool operator==( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n if (lhs.size() != rhs.size()) return false;\n#if _MSC_VER\n // Passing \"unchecked\" iterators to std::equal with 3 parameters\n // causes compiler warnings.\n // The workaround is to use overload with 4 parameters, which is\n // available since C++14 - minimally supported version on MSVC\n return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());\n#else\n return std::equal(lhs.begin(), lhs.end(), rhs.begin());\n#endif\n}\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate \nbool operator!=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return !(lhs == rhs);\n}\n#endif\n\n#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \ntbb::detail::synthesized_three_way_result\noperator<=>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(),\n rhs.begin(), rhs.end(),\n tbb::detail::synthesized_three_way_comparator{});\n}\n#else\ntemplate \nbool operator<( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());\n}\n\ntemplate \nbool operator>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return rhs < lhs;\n}\n\ntemplate \nbool operator<=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return !(rhs < lhs);\n}\n\ntemplate \nbool operator>=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) {\n return !(lhs < rhs);\n}\n#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT\n\n// Generates a number from the interval [0, MaxLevel).\ntemplate \nclass concurrent_geometric_level_generator {\npublic:\n static constexpr std::size_t max_level = MaxLevel;\n // TODO: modify the algorithm to accept other values of max_level\n static_assert(max_level == 32, \"Incompatible max_level for rng\");\n\n concurrent_geometric_level_generator() : engines(std::minstd_rand::result_type(time(nullptr))) {}\n\n std::size_t operator()() {\n // +1 is required to pass at least 1 into log2 (log2(0) is undefined)\n // -1 is required to have an ability to return 0 from the generator (max_level - log2(2^31) - 1)\n std::size_t result = max_level - std::size_t(tbb::detail::log2(engines.local()() + 1)) - 1;\n __TBB_ASSERT(result <= max_level, nullptr);\n return result;\n }\n\nprivate:\n tbb::enumerable_thread_specific engines;\n};\n\n} // namespace d2\n\n} // namespace detail\n} // namespace tbb\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(pop) // warning 4127 is back\n#endif\n\n#endif // __TBB_detail__concurrent_skip_list_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_concurrent_unordered_base.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__concurrent_unordered_base_H\n#define __TBB_detail__concurrent_unordered_base_H\n\n#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H)\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n#include \"_range_common.h\"\n#include \"_containers_helpers.h\"\n#include \"_segment_table.h\"\n#include \"_hash_compare.h\"\n#include \"_allocator_traits.h\"\n#include \"_node_handle.h\"\n#include \"_assert.h\"\n#include \"_utils.h\"\n#include \"_exception.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(push)\n// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nclass concurrent_unordered_base;\n\ntemplate\nclass solist_iterator {\nprivate:\n using node_ptr = typename Container::value_node_ptr;\n template \n friend class split_ordered_list;\n template\n friend class solist_iterator;\n template \n friend class concurrent_unordered_base;\n template\n friend bool operator==( const solist_iterator& i, const solist_iterator& j );\n template\n friend bool operator!=( const solist_iterator& i, const solist_iterator& j );\npublic:\n using value_type = Value;\n using difference_type = typename Container::difference_type;\n using pointer = value_type*;\n using reference = value_type&;\n using iterator_category = std::forward_iterator_tag;\n\n solist_iterator() : my_node_ptr(nullptr) {}\n solist_iterator( const solist_iterator& other )\n : my_node_ptr(other.my_node_ptr) {}\n\n solist_iterator& operator=( const solist_iterator& other ) {\n my_node_ptr = other.my_node_ptr;\n return *this;\n }\n\n reference operator*() const {\n return my_node_ptr->value();\n }\n\n pointer operator->() const {\n return my_node_ptr->storage();\n }\n\n solist_iterator& operator++() {\n auto next_node = my_node_ptr->next();\n while(next_node && next_node->is_dummy()) {\n next_node = next_node->next();\n }\n my_node_ptr = static_cast(next_node);\n return *this;\n }\n\n solist_iterator operator++(int) {\n solist_iterator tmp = *this;\n ++*this;\n return tmp;\n }\n\nprivate:\n solist_iterator( node_ptr pnode ) : my_node_ptr(pnode) {}\n\n node_ptr get_node_ptr() const { return my_node_ptr; }\n\n node_ptr my_node_ptr;\n};\n\ntemplate\nbool operator==( const solist_iterator& i, const solist_iterator& j ) {\n return i.my_node_ptr == j.my_node_ptr;\n}\n\ntemplate\nbool operator!=( const solist_iterator& i, const solist_iterator& j ) {\n return i.my_node_ptr != j.my_node_ptr;\n}\n\ntemplate \nclass list_node {\npublic:\n using node_ptr = list_node*;\n using sokey_type = SokeyType;\n\n list_node(sokey_type key) : my_next(nullptr), my_order_key(key) {}\n\n void init( sokey_type key ) {\n my_order_key = key;\n }\n\n sokey_type order_key() const {\n return my_order_key;\n }\n\n bool is_dummy() {\n // The last bit of order key is unset for dummy nodes\n return (my_order_key & 0x1) == 0;\n }\n\n node_ptr next() const {\n return my_next.load(std::memory_order_acquire);\n }\n\n void set_next( node_ptr next_node ) {\n my_next.store(next_node, std::memory_order_release);\n }\n\n bool try_set_next( node_ptr expected_next, node_ptr new_next ) {\n return my_next.compare_exchange_strong(expected_next, new_next);\n }\n\nprivate:\n std::atomic my_next;\n sokey_type my_order_key;\n}; // class list_node\n\ntemplate \nclass value_node : public list_node\n{\npublic:\n using base_type = list_node;\n using sokey_type = typename base_type::sokey_type;\n using value_type = ValueType;\n\n value_node( sokey_type ord_key ) : base_type(ord_key) {}\n ~value_node() {}\n value_type* storage() {\n return &my_value;\n }\n\n value_type& value() {\n return *storage();\n }\n\nprivate:\n union {\n value_type my_value;\n };\n}; // class value_node\n\ntemplate \nclass concurrent_unordered_base {\n using self_type = concurrent_unordered_base;\n using traits_type = Traits;\n using hash_compare_type = typename traits_type::hash_compare_type;\n class unordered_segment_table;\npublic:\n using value_type = typename traits_type::value_type;\n using key_type = typename traits_type::key_type;\n using allocator_type = typename traits_type::allocator_type;\n\nprivate:\n using allocator_traits_type = tbb::detail::allocator_traits;\n // TODO: check assert conditions for different C++ standards\n static_assert(std::is_same::value,\n \"value_type of the container must be the same as its allocator\");\n using sokey_type = std::size_t;\n\npublic:\n using size_type = std::size_t;\n using difference_type = std::ptrdiff_t;\n\n using iterator = solist_iterator;\n using const_iterator = solist_iterator;\n using local_iterator = iterator;\n using const_local_iterator = const_iterator;\n\n using reference = value_type&;\n using const_reference = const value_type&;\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n using hasher = typename hash_compare_type::hasher;\n using key_equal = typename hash_compare_type::key_equal;\n\nprivate:\n using list_node_type = list_node;\n using value_node_type = value_node;\n using node_ptr = list_node_type*;\n using value_node_ptr = value_node_type*;\n\n using value_node_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using node_allocator_type = typename allocator_traits_type::template rebind_alloc;\n\n using node_allocator_traits = tbb::detail::allocator_traits;\n using value_node_allocator_traits = tbb::detail::allocator_traits;\n\n static constexpr size_type round_up_to_power_of_two( size_type bucket_count ) {\n return size_type(1) << size_type(tbb::detail::log2(uintptr_t(bucket_count == 0 ? 1 : bucket_count) * 2 - 1));\n }\n\n template \n using is_transparent = dependent_bool, T>;\npublic:\n using node_type = d1::node_handle;\n\n explicit concurrent_unordered_base( size_type bucket_count, const hasher& hash = hasher(),\n const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() )\n : my_size(0),\n my_bucket_count(round_up_to_power_of_two(bucket_count)),\n my_max_load_factor(float(initial_max_load_factor)),\n my_hash_compare(hash, equal),\n my_head(sokey_type(0)),\n my_segments(alloc) {}\n\n concurrent_unordered_base() : concurrent_unordered_base(initial_bucket_count) {}\n\n concurrent_unordered_base( size_type bucket_count, const allocator_type& alloc )\n : concurrent_unordered_base(bucket_count, hasher(), key_equal(), alloc) {}\n\n concurrent_unordered_base( size_type bucket_count, const hasher& hash, const allocator_type& alloc )\n : concurrent_unordered_base(bucket_count, hash, key_equal(), alloc) {}\n\n explicit concurrent_unordered_base( const allocator_type& alloc )\n : concurrent_unordered_base(initial_bucket_count, hasher(), key_equal(), alloc) {}\n\n template \n concurrent_unordered_base( InputIterator first, InputIterator last,\n size_type bucket_count = initial_bucket_count, const hasher& hash = hasher(),\n const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() )\n : concurrent_unordered_base(bucket_count, hash, equal, alloc)\n {\n insert(first, last);\n }\n\n template \n concurrent_unordered_base( InputIterator first, InputIterator last,\n size_type bucket_count, const allocator_type& alloc )\n : concurrent_unordered_base(first, last, bucket_count, hasher(), key_equal(), alloc) {}\n\n template \n concurrent_unordered_base( InputIterator first, InputIterator last,\n size_type bucket_count, const hasher& hash, const allocator_type& alloc )\n : concurrent_unordered_base(first, last, bucket_count, hash, key_equal(), alloc) {}\n\n concurrent_unordered_base( const concurrent_unordered_base& other )\n : my_size(other.my_size.load(std::memory_order_relaxed)),\n my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)),\n my_max_load_factor(other.my_max_load_factor),\n my_hash_compare(other.my_hash_compare),\n my_head(other.my_head.order_key()),\n my_segments(other.my_segments)\n {\n try_call( [&] {\n internal_copy(other);\n } ).on_exception( [&] {\n clear();\n });\n }\n\n concurrent_unordered_base( const concurrent_unordered_base& other, const allocator_type& alloc )\n : my_size(other.my_size.load(std::memory_order_relaxed)),\n my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)),\n my_max_load_factor(other.my_max_load_factor),\n my_hash_compare(other.my_hash_compare),\n my_head(other.my_head.order_key()),\n my_segments(other.my_segments, alloc)\n {\n try_call( [&] {\n internal_copy(other);\n } ).on_exception( [&] {\n clear();\n });\n }\n\n concurrent_unordered_base( concurrent_unordered_base&& other )\n : my_size(other.my_size.load(std::memory_order_relaxed)),\n my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)),\n my_max_load_factor(std::move(other.my_max_load_factor)),\n my_hash_compare(std::move(other.my_hash_compare)),\n my_head(other.my_head.order_key()),\n my_segments(std::move(other.my_segments))\n {\n move_content(std::move(other));\n }\n\n concurrent_unordered_base( concurrent_unordered_base&& other, const allocator_type& alloc )\n : my_size(other.my_size.load(std::memory_order_relaxed)),\n my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)),\n my_max_load_factor(std::move(other.my_max_load_factor)),\n my_hash_compare(std::move(other.my_hash_compare)),\n my_head(other.my_head.order_key()),\n my_segments(std::move(other.my_segments), alloc)\n {\n using is_always_equal = typename allocator_traits_type::is_always_equal;\n internal_move_construct_with_allocator(std::move(other), alloc, is_always_equal());\n }\n\n concurrent_unordered_base( std::initializer_list init,\n size_type bucket_count = initial_bucket_count,\n const hasher& hash = hasher(), const key_equal& equal = key_equal(),\n const allocator_type& alloc = allocator_type() )\n : concurrent_unordered_base(init.begin(), init.end(), bucket_count, hash, equal, alloc) {}\n\n concurrent_unordered_base( std::initializer_list init,\n size_type bucket_count, const allocator_type& alloc )\n : concurrent_unordered_base(init, bucket_count, hasher(), key_equal(), alloc) {}\n\n concurrent_unordered_base( std::initializer_list init,\n size_type bucket_count, const hasher& hash, const allocator_type& alloc )\n : concurrent_unordered_base(init, bucket_count, hash, key_equal(), alloc) {}\n\n ~concurrent_unordered_base() {\n internal_clear();\n }\n\n concurrent_unordered_base& operator=( const concurrent_unordered_base& other ) {\n if (this != &other) {\n clear();\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_max_load_factor = other.my_max_load_factor;\n my_hash_compare = other.my_hash_compare;\n my_segments = other.my_segments;\n internal_copy(other); // TODO: guards for exceptions?\n }\n return *this;\n }\n\n concurrent_unordered_base& operator=( concurrent_unordered_base&& other ) noexcept(unordered_segment_table::is_noexcept_assignment) {\n if (this != &other) {\n clear();\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_max_load_factor = std::move(other.my_max_load_factor);\n my_hash_compare = std::move(other.my_hash_compare);\n my_segments = std::move(other.my_segments);\n\n using pocma_type = typename allocator_traits_type::propagate_on_container_move_assignment;\n using is_always_equal = typename allocator_traits_type::is_always_equal;\n internal_move_assign(std::move(other), tbb::detail::disjunction());\n }\n return *this;\n }\n\n concurrent_unordered_base& operator=( std::initializer_list init ) {\n clear();\n insert(init);\n return *this;\n }\n\n void swap( concurrent_unordered_base& other ) noexcept(unordered_segment_table::is_noexcept_swap) {\n if (this != &other) {\n using pocs_type = typename allocator_traits_type::propagate_on_container_swap;\n using is_always_equal = typename allocator_traits_type::is_always_equal;\n internal_swap(other, tbb::detail::disjunction());\n }\n }\n\n allocator_type get_allocator() const noexcept { return my_segments.get_allocator(); }\n\n iterator begin() noexcept { return iterator(first_value_node(&my_head)); }\n const_iterator begin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); }\n const_iterator cbegin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); }\n\n iterator end() noexcept { return iterator(nullptr); }\n const_iterator end() const noexcept { return const_iterator(nullptr); }\n const_iterator cend() const noexcept { return const_iterator(nullptr); }\n\n __TBB_nodiscard bool empty() const noexcept { return size() == 0; }\n size_type size() const noexcept { return my_size.load(std::memory_order_relaxed); }\n size_type max_size() const noexcept { return allocator_traits_type::max_size(get_allocator()); }\n\n void clear() noexcept {\n internal_clear();\n }\n\n std::pair insert( const value_type& value ) {\n return internal_insert_value(value);\n }\n\n std::pair insert( value_type&& value ) {\n return internal_insert_value(std::move(value));\n }\n\n iterator insert( const_iterator, const value_type& value ) {\n // Ignore hint\n return insert(value).first;\n }\n\n iterator insert( const_iterator, value_type&& value ) {\n // Ignore hint\n return insert(std::move(value)).first;\n }\n\n template \n void insert( InputIterator first, InputIterator last ) {\n for (; first != last; ++first) {\n insert(*first);\n }\n }\n\n void insert( std::initializer_list init ) {\n insert(init.begin(), init.end());\n }\n\n std::pair insert( node_type&& nh ) {\n if (!nh.empty()) {\n value_node_ptr insert_node = d1::node_handle_accessor::get_node_ptr(nh);\n auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr {\n insert_node->init(order_key);\n return insert_node;\n };\n auto insert_result = internal_insert(insert_node->value(), init_node);\n if (insert_result.inserted) {\n // If the insertion succeeded - set node handle to the empty state\n __TBB_ASSERT(insert_result.remaining_node == nullptr,\n \"internal_insert_node should not return the remaining node if the insertion succeeded\");\n d1::node_handle_accessor::deactivate(nh);\n }\n return { iterator(insert_result.node_with_equal_key), insert_result.inserted };\n }\n return {end(), false};\n }\n\n iterator insert( const_iterator, node_type&& nh ) {\n // Ignore hint\n return insert(std::move(nh)).first;\n }\n\n template \n std::pair emplace( Args&&... args ) {\n // Create a node with temporary order_key 0, which will be reinitialize\n // in internal_insert after the hash calculation\n value_node_ptr insert_node = create_node(0, std::forward(args)...);\n\n auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr {\n insert_node->init(order_key);\n return insert_node;\n };\n\n auto insert_result = internal_insert(insert_node->value(), init_node);\n\n if (!insert_result.inserted) {\n // If the insertion failed - destroy the node which was created\n insert_node->init(split_order_key_regular(1));\n destroy_node(insert_node);\n }\n\n return { iterator(insert_result.node_with_equal_key), insert_result.inserted };\n }\n\n template \n iterator emplace_hint( const_iterator, Args&&... args ) {\n // Ignore hint\n return emplace(std::forward(args)...).first;\n }\n\n iterator unsafe_erase( const_iterator pos ) {\n return iterator(first_value_node(internal_erase(pos.get_node_ptr())));\n }\n\n iterator unsafe_erase( iterator pos ) {\n return iterator(first_value_node(internal_erase(pos.get_node_ptr())));\n }\n\n iterator unsafe_erase( const_iterator first, const_iterator last ) {\n while(first != last) {\n first = unsafe_erase(first);\n }\n return iterator(first.get_node_ptr());\n }\n\n size_type unsafe_erase( const key_type& key ) {\n return internal_erase_by_key(key);\n }\n\n template \n typename std::enable_if::value\n && !std::is_convertible::value\n && !std::is_convertible::value,\n size_type>::type unsafe_erase( const K& key )\n {\n return internal_erase_by_key(key);\n }\n\n node_type unsafe_extract( const_iterator pos ) {\n internal_extract(pos.get_node_ptr());\n return d1::node_handle_accessor::construct(pos.get_node_ptr());\n }\n\n node_type unsafe_extract( iterator pos ) {\n internal_extract(pos.get_node_ptr());\n return d1::node_handle_accessor::construct(pos.get_node_ptr());\n }\n\n node_type unsafe_extract( const key_type& key ) {\n iterator item = find(key);\n return item == end() ? node_type() : unsafe_extract(item);\n }\n\n template \n typename std::enable_if::value\n && !std::is_convertible::value\n && !std::is_convertible::value,\n node_type>::type unsafe_extract( const K& key )\n {\n iterator item = find(key);\n return item == end() ? node_type() : unsafe_extract(item);\n }\n\n // Lookup functions\n iterator find( const key_type& key ) {\n value_node_ptr result = internal_find(key);\n return result == nullptr ? end() : iterator(result);\n }\n\n const_iterator find( const key_type& key ) const {\n value_node_ptr result = const_cast(this)->internal_find(key);\n return result == nullptr ? end() : const_iterator(result);\n }\n\n template \n typename std::enable_if::value, iterator>::type find( const K& key ) {\n value_node_ptr result = internal_find(key);\n return result == nullptr ? end() : iterator(result);\n }\n\n template \n typename std::enable_if::value, const_iterator>::type find( const K& key ) const {\n value_node_ptr result = const_cast(this)->internal_find(key);\n return result == nullptr ? end() : const_iterator(result);\n }\n\n std::pair equal_range( const key_type& key ) {\n auto result = internal_equal_range(key);\n return std::make_pair(iterator(result.first), iterator(result.second));\n }\n\n std::pair equal_range( const key_type& key ) const {\n auto result = const_cast(this)->internal_equal_range(key);\n return std::make_pair(const_iterator(result.first), const_iterator(result.second));\n }\n\n template \n typename std::enable_if::value, std::pair>::type equal_range( const K& key ) {\n auto result = internal_equal_range(key);\n return std::make_pair(iterator(result.first), iterator(result.second));\n }\n\n template \n typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const {\n auto result = const_cast(this)->internal_equal_range(key);\n return std::make_pair(iterator(result.first), iterator(result.second));\n }\n\n size_type count( const key_type& key ) const {\n return internal_count(key);\n }\n\n template \n typename std::enable_if::value, size_type>::type count( const K& key ) const {\n return internal_count(key);\n }\n\n bool contains( const key_type& key ) const {\n return find(key) != end();\n }\n\n template \n typename std::enable_if::value, bool>::type contains( const K& key ) const {\n return find(key) != end();\n }\n\n // Bucket interface\n local_iterator unsafe_begin( size_type n ) {\n return local_iterator(first_value_node(get_bucket(n)));\n }\n\n const_local_iterator unsafe_begin( size_type n ) const {\n auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n));\n return const_local_iterator(bucket_begin);\n }\n\n const_local_iterator unsafe_cbegin( size_type n ) const {\n auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n));\n return const_local_iterator(bucket_begin);\n }\n\n local_iterator unsafe_end( size_type n ) {\n size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed);\n return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : local_iterator(nullptr);\n }\n\n const_local_iterator unsafe_end( size_type n ) const {\n size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed);\n return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr);\n }\n\n const_local_iterator unsafe_cend( size_type n ) const {\n size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed);\n return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr);\n }\n\n size_type unsafe_bucket_count() const { return my_bucket_count.load(std::memory_order_relaxed); }\n\n size_type unsafe_max_bucket_count() const {\n return max_size();\n }\n\n size_type unsafe_bucket_size( size_type n ) const {\n return size_type(std::distance(unsafe_begin(n), unsafe_end(n)));\n }\n\n size_type unsafe_bucket( const key_type& key ) const {\n return my_hash_compare(key) % my_bucket_count.load(std::memory_order_relaxed);\n }\n\n // Hash policy\n float load_factor() const {\n return float(size() / float(my_bucket_count.load(std::memory_order_acquire)));\n }\n\n float max_load_factor() const { return my_max_load_factor; }\n\n void max_load_factor( float mlf ) {\n if (mlf != mlf || mlf < 0) {\n tbb::detail::throw_exception(exception_id::invalid_load_factor);\n }\n my_max_load_factor = mlf;\n } // TODO: unsafe?\n\n void rehash( size_type bucket_count ) {\n size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire);\n if (current_bucket_count < bucket_count) {\n // TODO: do we need do-while here?\n my_bucket_count.compare_exchange_strong(current_bucket_count, round_up_to_power_of_two(bucket_count));\n }\n }\n\n void reserve( size_type elements_count ) {\n size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire);\n size_type necessary_bucket_count = current_bucket_count;\n\n // max_load_factor() is currently unsafe, so we can assume that my_max_load_factor\n // would not be changed during the calculation\n // TODO: Log2 seems useful here\n while (necessary_bucket_count * max_load_factor() < elements_count) {\n necessary_bucket_count <<= 1;\n }\n\n while (!my_bucket_count.compare_exchange_strong(current_bucket_count, necessary_bucket_count)) {\n if (current_bucket_count >= necessary_bucket_count)\n break;\n }\n }\n\n // Observers\n hasher hash_function() const { return my_hash_compare.hash_function(); }\n key_equal key_eq() const { return my_hash_compare.key_eq(); }\n\n class const_range_type {\n private:\n const concurrent_unordered_base& my_instance;\n node_ptr my_begin_node; // may be node* const\n node_ptr my_end_node;\n mutable node_ptr my_midpoint_node;\n public:\n using size_type = typename concurrent_unordered_base::size_type;\n using value_type = typename concurrent_unordered_base::value_type;\n using reference = typename concurrent_unordered_base::reference;\n using difference_type = typename concurrent_unordered_base::difference_type;\n using iterator = typename concurrent_unordered_base::const_iterator;\n\n bool empty() const { return my_begin_node == my_end_node; }\n\n bool is_divisible() const {\n return my_midpoint_node != my_end_node;\n }\n\n size_type grainsize() const { return 1; }\n\n const_range_type( const_range_type& range, split )\n : my_instance(range.my_instance),\n my_begin_node(range.my_midpoint_node),\n my_end_node(range.my_end_node)\n {\n range.my_end_node = my_begin_node;\n __TBB_ASSERT(!empty(), \"Splitting despite the range is not divisible\");\n __TBB_ASSERT(!range.empty(), \"Splitting despite the range is not divisible\");\n set_midpoint();\n range.set_midpoint();\n }\n\n iterator begin() const { return iterator(my_instance.first_value_node(my_begin_node)); }\n iterator end() const { return iterator(my_instance.first_value_node(my_end_node)); }\n\n const_range_type( const concurrent_unordered_base& table )\n : my_instance(table), my_begin_node(my_instance.first_value_node(const_cast(&table.my_head))), my_end_node(nullptr)\n {\n set_midpoint();\n }\n private:\n void set_midpoint() const {\n if (empty()) {\n my_midpoint_node = my_end_node;\n } else {\n sokey_type invalid_key = ~sokey_type(0);\n sokey_type begin_key = my_begin_node != nullptr ? my_begin_node->order_key() : invalid_key;\n sokey_type end_key = my_end_node != nullptr ? my_end_node->order_key() : invalid_key;\n\n size_type mid_bucket = reverse_bits(begin_key + (end_key - begin_key) / 2) %\n my_instance.my_bucket_count.load(std::memory_order_relaxed);\n while( my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed) == nullptr) {\n mid_bucket = my_instance.get_parent(mid_bucket);\n }\n if (reverse_bits(mid_bucket) > begin_key) {\n // Found a dummy node between begin and end\n my_midpoint_node = my_instance.first_value_node(\n my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed));\n } else {\n // Didn't find a dummy node between begin and end\n my_midpoint_node = my_end_node;\n }\n }\n }\n }; // class const_range_type\n\n class range_type : public const_range_type {\n public:\n using iterator = typename concurrent_unordered_base::iterator;\n using const_range_type::const_range_type;\n\n iterator begin() const { return iterator(const_range_type::begin().get_node_ptr()); }\n iterator end() const { return iterator(const_range_type::end().get_node_ptr()); }\n }; // class range_type\n\n // Parallel iteration\n range_type range() {\n return range_type(*this);\n }\n\n const_range_type range() const {\n return const_range_type(*this);\n }\nprotected:\n static constexpr bool allow_multimapping = traits_type::allow_multimapping;\n\nprivate:\n static constexpr size_type initial_bucket_count = 8;\n static constexpr float initial_max_load_factor = 4; // TODO: consider 1?\n static constexpr size_type pointers_per_embedded_table = sizeof(size_type) * 8 - 1;\n\n class unordered_segment_table\n : public d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table>\n {\n using self_type = unordered_segment_table;\n using atomic_node_ptr = std::atomic;\n using base_type = d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table>;\n using segment_type = typename base_type::segment_type;\n using base_allocator_type = typename base_type::allocator_type;\n\n using segment_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using segment_allocator_traits = tbb::detail::allocator_traits;\n public:\n // Segment table for unordered containers should not be extended in the wait- free implementation\n static constexpr bool allow_table_extending = false;\n static constexpr bool is_noexcept_assignment = std::is_nothrow_move_assignable::value &&\n std::is_nothrow_move_assignable::value &&\n segment_allocator_traits::is_always_equal::value;\n static constexpr bool is_noexcept_swap = tbb::detail::is_nothrow_swappable::value &&\n tbb::detail::is_nothrow_swappable::value &&\n segment_allocator_traits::is_always_equal::value;\n\n // TODO: using base_type::base_type is not compiling on Windows and Intel Compiler - investigate\n unordered_segment_table( const base_allocator_type& alloc = base_allocator_type() )\n : base_type(alloc) {}\n\n unordered_segment_table( const unordered_segment_table& ) = default;\n\n unordered_segment_table( const unordered_segment_table& other, const base_allocator_type& alloc )\n : base_type(other, alloc) {}\n\n unordered_segment_table( unordered_segment_table&& ) = default;\n\n unordered_segment_table( unordered_segment_table&& other, const base_allocator_type& alloc )\n : base_type(std::move(other), alloc) {}\n\n unordered_segment_table& operator=( const unordered_segment_table& ) = default;\n\n unordered_segment_table& operator=( unordered_segment_table&& ) = default;\n\n segment_type create_segment( typename base_type::segment_table_type, typename base_type::segment_index_type segment_index, size_type ) {\n segment_allocator_type alloc(this->get_allocator());\n size_type seg_size = this->segment_size(segment_index);\n segment_type new_segment = segment_allocator_traits::allocate(alloc, seg_size);\n for (size_type i = 0; i != seg_size; ++i) {\n segment_allocator_traits::construct(alloc, new_segment + i, nullptr);\n }\n return new_segment;\n }\n\n segment_type nullify_segment( typename base_type::segment_table_type table, size_type segment_index ) {\n segment_type target_segment = table[segment_index].load(std::memory_order_relaxed);\n table[segment_index].store(nullptr, std::memory_order_relaxed);\n return target_segment;\n }\n\n // deallocate_segment is required by the segment_table base class, but\n // in unordered, it is also necessary to call the destructor during deallocation\n void deallocate_segment( segment_type address, size_type index ) {\n destroy_segment(address, index);\n }\n\n void destroy_segment( segment_type address, size_type index ) {\n segment_allocator_type alloc(this->get_allocator());\n for (size_type i = 0; i != this->segment_size(index); ++i) {\n segment_allocator_traits::destroy(alloc, address + i);\n }\n segment_allocator_traits::deallocate(alloc, address, this->segment_size(index));\n }\n\n\n void copy_segment( size_type index, segment_type, segment_type to ) {\n if (index == 0) {\n // The first element in the first segment is embedded into the table (my_head)\n // so the first pointer should not be stored here\n // It would be stored during move ctor/assignment operation\n to[1].store(nullptr, std::memory_order_relaxed);\n } else {\n for (size_type i = 0; i != this->segment_size(index); ++i) {\n to[i].store(nullptr, std::memory_order_relaxed);\n }\n }\n }\n\n void move_segment( size_type index, segment_type from, segment_type to ) {\n if (index == 0) {\n // The first element in the first segment is embedded into the table (my_head)\n // so the first pointer should not be stored here\n // It would be stored during move ctor/assignment operation\n to[1].store(from[1].load(std::memory_order_relaxed), std::memory_order_relaxed);\n } else {\n for (size_type i = 0; i != this->segment_size(index); ++i) {\n to[i].store(from[i].load(std::memory_order_relaxed), std::memory_order_relaxed);\n from[i].store(nullptr, std::memory_order_relaxed);\n }\n }\n }\n\n // allocate_long_table is required by the segment_table base class, but unused for unordered containers\n typename base_type::segment_table_type allocate_long_table( const typename base_type::atomic_segment*, size_type ) {\n __TBB_ASSERT(false, \"This method should never been called\");\n // TableType is a pointer\n return nullptr;\n }\n\n // destroy_elements is required by the segment_table base class, but unused for unordered containers\n // this function call but do nothing\n void destroy_elements() {}\n }; // struct unordered_segment_table\n\n void internal_clear() {\n // TODO: consider usefulness of two versions of clear() - with dummy nodes deallocation and without it\n node_ptr next = my_head.next();\n node_ptr curr = next;\n\n my_head.set_next(nullptr);\n\n while (curr != nullptr) {\n next = curr->next();\n destroy_node(curr);\n curr = next;\n }\n\n my_size.store(0, std::memory_order_relaxed);\n my_segments.clear();\n }\n\n void destroy_node( node_ptr node ) {\n if (node->is_dummy()) {\n node_allocator_type dummy_node_allocator(my_segments.get_allocator());\n // Destroy the node\n node_allocator_traits::destroy(dummy_node_allocator, node);\n // Deallocate the memory\n node_allocator_traits::deallocate(dummy_node_allocator, node, 1);\n } else {\n // GCC 11.1 issues a warning here that incorrect destructor might be called for dummy_nodes\n #if (__TBB_GCC_VERSION >= 110100 && __TBB_GCC_VERSION < 150000 ) && !__clang__ && !__INTEL_COMPILER\n volatile\n #endif\n value_node_ptr val_node = static_cast(node);\n value_node_allocator_type value_node_allocator(my_segments.get_allocator());\n // Destroy the value\n value_node_allocator_traits::destroy(value_node_allocator, val_node->storage());\n // Destroy the node\n value_node_allocator_traits::destroy(value_node_allocator, val_node);\n // Deallocate the memory\n value_node_allocator_traits::deallocate(value_node_allocator, val_node, 1);\n }\n }\n\n struct internal_insert_return_type {\n // If the insertion failed - the remaining_node points to the node, which was failed to insert\n // This node can be allocated in process of insertion\n value_node_ptr remaining_node;\n // If the insertion failed - node_with_equal_key points to the node in the list with the\n // key, equivalent to the inserted, otherwise it points to the node, which was inserted.\n value_node_ptr node_with_equal_key;\n // Insertion status\n // NOTE: if it is true - remaining_node should be nullptr\n bool inserted;\n }; // struct internal_insert_return_type\n\n // Inserts the value into the split ordered list\n template \n std::pair internal_insert_value( ValueType&& value ) {\n\n auto create_value_node = [&value, this]( sokey_type order_key )->value_node_ptr {\n return create_node(order_key, std::forward(value));\n };\n\n auto insert_result = internal_insert(value, create_value_node);\n\n if (insert_result.remaining_node != nullptr) {\n // If the insertion fails - destroy the node which was failed to insert if it exist\n __TBB_ASSERT(!insert_result.inserted,\n \"remaining_node should be nullptr if the node was successfully inserted\");\n destroy_node(insert_result.remaining_node);\n }\n\n return { iterator(insert_result.node_with_equal_key), insert_result.inserted };\n }\n\n // Inserts the node into the split ordered list\n // Creates a node using the specified callback after the place for insertion was found\n // Returns internal_insert_return_type object, where:\n // - If the insertion succeeded:\n // - remaining_node is nullptr\n // - node_with_equal_key point to the inserted node\n // - inserted is true\n // - If the insertion failed:\n // - remaining_node points to the node, that was failed to insert if it was created.\n // nullptr if the node was not created, because the requested key was already\n // presented in the list\n // - node_with_equal_key point to the element in the list with the key, equivalent to\n // to the requested key\n // - inserted is false\n template \n internal_insert_return_type internal_insert( ValueType&& value, CreateInsertNode create_insert_node ) {\n static_assert(std::is_same::type, value_type>::value,\n \"Incorrect type in internal_insert\");\n const key_type& key = traits_type::get_key(value);\n sokey_type hash_key = sokey_type(my_hash_compare(key));\n\n sokey_type order_key = split_order_key_regular(hash_key);\n node_ptr prev = prepare_bucket(hash_key);\n __TBB_ASSERT(prev != nullptr, \"Invalid head node\");\n\n auto search_result = search_after(prev, order_key, key);\n\n if (search_result.second) {\n return internal_insert_return_type{ nullptr, search_result.first, false };\n }\n\n value_node_ptr new_node = create_insert_node(order_key);\n node_ptr curr = search_result.first;\n\n while (!try_insert(prev, new_node, curr)) {\n search_result = search_after(prev, order_key, key);\n if (search_result.second) {\n return internal_insert_return_type{ new_node, search_result.first, false };\n }\n curr = search_result.first;\n }\n\n auto sz = my_size.fetch_add(1);\n adjust_table_size(sz + 1, my_bucket_count.load(std::memory_order_acquire));\n return internal_insert_return_type{ nullptr, static_cast(new_node), true };\n }\n\n // Searches the node with the key, equivalent to key with requested order key after the node prev\n // Returns the existing node and true if the node is already in the list\n // Returns the first node with the order key, greater than requested and false if the node is not presented in the list\n std::pair search_after( node_ptr& prev, sokey_type order_key, const key_type& key ) {\n // NOTE: static_cast(curr) should be done only after we would ensure\n // that the node is not a dummy node\n\n node_ptr curr = prev->next();\n\n while (curr != nullptr && (curr->order_key() < order_key ||\n (curr->order_key() == order_key && !my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key))))\n {\n prev = curr;\n curr = curr->next();\n }\n\n if (curr != nullptr && curr->order_key() == order_key && !allow_multimapping) {\n return { static_cast(curr), true };\n }\n return { static_cast(curr), false };\n }\n\n void adjust_table_size( size_type total_elements, size_type current_size ) {\n // Grow the table by a factor of 2 if possible and needed\n if ( (float(total_elements) / float(current_size)) > my_max_load_factor ) {\n // Double the size of the hash only if size hash not changed in between loads\n my_bucket_count.compare_exchange_strong(current_size, 2u * current_size);\n }\n }\n\n node_ptr insert_dummy_node( node_ptr parent_dummy_node, sokey_type order_key ) {\n node_ptr prev_node = parent_dummy_node;\n\n node_ptr dummy_node = create_dummy_node(order_key);\n node_ptr next_node;\n\n do {\n next_node = prev_node->next();\n // Move forward through the list while the order key is less than requested\n while (next_node != nullptr && next_node->order_key() < order_key) {\n prev_node = next_node;\n next_node = next_node->next();\n }\n\n if (next_node != nullptr && next_node->order_key() == order_key) {\n // Another dummy node with the same order key was inserted by another thread\n // Destroy the node and exit\n destroy_node(dummy_node);\n return next_node;\n }\n } while (!try_insert(prev_node, dummy_node, next_node));\n\n return dummy_node;\n }\n\n // Try to insert a node between prev_node and expected next\n // If the next is not equal to expected next - return false\n static bool try_insert( node_ptr prev_node, node_ptr new_node, node_ptr current_next_node ) {\n new_node->set_next(current_next_node);\n return prev_node->try_set_next(current_next_node, new_node);\n }\n\n // Returns the bucket, associated with the hash_key\n node_ptr prepare_bucket( sokey_type hash_key ) {\n size_type bucket = hash_key % my_bucket_count.load(std::memory_order_acquire);\n return get_bucket(bucket);\n }\n\n // Initialize the corresponding bucket if it is not initialized\n node_ptr get_bucket( size_type bucket_index ) {\n if (my_segments[bucket_index].load(std::memory_order_acquire) == nullptr) {\n init_bucket(bucket_index);\n }\n return my_segments[bucket_index].load(std::memory_order_acquire);\n }\n\n void init_bucket( size_type bucket ) {\n if (bucket == 0) {\n // Atomicaly store the first bucket into my_head\n node_ptr disabled = nullptr;\n my_segments[0].compare_exchange_strong(disabled, &my_head);\n return;\n }\n\n size_type parent_bucket = get_parent(bucket);\n\n while (my_segments[parent_bucket].load(std::memory_order_acquire) == nullptr) {\n // Initialize all of the parent buckets\n init_bucket(parent_bucket);\n }\n\n __TBB_ASSERT(my_segments[parent_bucket].load(std::memory_order_acquire) != nullptr, \"Parent bucket should be initialized\");\n node_ptr parent = my_segments[parent_bucket].load(std::memory_order_acquire);\n\n // Insert dummy node into the list\n node_ptr dummy_node = insert_dummy_node(parent, split_order_key_dummy(bucket));\n // TODO: consider returning pair to avoid store operation if the bucket was stored by an other thread\n // or move store to insert_dummy_node\n // Add dummy_node into the segment table\n my_segments[bucket].store(dummy_node, std::memory_order_release);\n }\n\n node_ptr create_dummy_node( sokey_type order_key ) {\n node_allocator_type dummy_node_allocator(my_segments.get_allocator());\n node_ptr dummy_node = node_allocator_traits::allocate(dummy_node_allocator, 1);\n node_allocator_traits::construct(dummy_node_allocator, dummy_node, order_key);\n return dummy_node;\n }\n\n template \n value_node_ptr create_node( sokey_type order_key, Args&&... args ) {\n value_node_allocator_type value_node_allocator(my_segments.get_allocator());\n // Allocate memory for the value_node\n value_node_ptr new_node = value_node_allocator_traits::allocate(value_node_allocator, 1);\n // Construct the node\n value_node_allocator_traits::construct(value_node_allocator, new_node, order_key);\n\n // try_call API is not convenient here due to broken\n // variadic capture on GCC 4.8.5\n auto value_guard = make_raii_guard([&] {\n value_node_allocator_traits::destroy(value_node_allocator, new_node);\n value_node_allocator_traits::deallocate(value_node_allocator, new_node, 1);\n });\n\n // Construct the value in the node\n value_node_allocator_traits::construct(value_node_allocator, new_node->storage(), std::forward(args)...);\n value_guard.dismiss();\n return new_node;\n }\n\n value_node_ptr first_value_node( node_ptr first_node ) const {\n while (first_node != nullptr && first_node->is_dummy()) {\n first_node = first_node->next();\n }\n return static_cast(first_node);\n }\n\n // Unsafe method, which removes the node from the list and returns the next node\n node_ptr internal_erase( value_node_ptr node_to_erase ) {\n __TBB_ASSERT(node_to_erase != nullptr, \"Invalid iterator for erase\");\n node_ptr next_node = node_to_erase->next();\n internal_extract(node_to_erase);\n destroy_node(node_to_erase);\n return next_node;\n }\n\n template \n size_type internal_erase_by_key( const K& key ) {\n // TODO: consider reimplementation without equal_range - it is not effective to perform lookup over a bucket\n // for each unsafe_erase call\n auto eq_range = equal_range(key);\n size_type erased_count = 0;\n\n for (auto it = eq_range.first; it != eq_range.second;) {\n it = unsafe_erase(it);\n ++erased_count;\n }\n return erased_count;\n }\n\n // Unsafe method, which extracts the node from the list\n void internal_extract( value_node_ptr node_to_extract ) {\n const key_type& key = traits_type::get_key(node_to_extract->value());\n sokey_type hash_key = sokey_type(my_hash_compare(key));\n\n node_ptr prev_node = prepare_bucket(hash_key);\n\n for (node_ptr node = prev_node->next(); node != nullptr; prev_node = node, node = node->next()) {\n if (node == node_to_extract) {\n unlink_node(prev_node, node, node_to_extract->next());\n my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed);\n return;\n }\n __TBB_ASSERT(node->order_key() <= node_to_extract->order_key(),\n \"node, which is going to be extracted should be presented in the list\");\n }\n }\n\nprotected:\n template \n void internal_merge( SourceType&& source ) {\n static_assert(std::is_same::type::node_type>::value,\n \"Incompatible containers cannot be merged\");\n\n for (node_ptr source_prev = &source.my_head; source_prev->next() != nullptr;) {\n if (!source_prev->next()->is_dummy()) {\n value_node_ptr curr = static_cast(source_prev->next());\n // If the multimapping is allowed, or the key is not presented\n // in the *this container - extract the node from the list\n if (allow_multimapping || !contains(traits_type::get_key(curr->value()))) {\n node_ptr next_node = curr->next();\n source.unlink_node(source_prev, curr, next_node);\n\n // Remember the old order key\n sokey_type old_order_key = curr->order_key();\n\n // Node handle with curr cannot be used directly in insert call, because\n // the destructor of node_type will destroy curr\n node_type curr_node = d1::node_handle_accessor::construct(curr);\n\n // If the insertion fails - return ownership of the node to the source\n if (!insert(std::move(curr_node)).second) {\n __TBB_ASSERT(!allow_multimapping, \"Insertion should succeed for multicontainer\");\n __TBB_ASSERT(source_prev->next() == next_node,\n \"Concurrent operations with the source container in merge are prohibited\");\n\n // Initialize the node with the old order key, because the order key\n // can change during the insertion\n curr->init(old_order_key);\n __TBB_ASSERT(old_order_key >= source_prev->order_key() &&\n (next_node == nullptr || old_order_key <= next_node->order_key()),\n \"Wrong nodes order in the source container\");\n // Merge is unsafe for source container, so the insertion back can be done without compare_exchange\n curr->set_next(next_node);\n source_prev->set_next(curr);\n source_prev = curr;\n d1::node_handle_accessor::deactivate(curr_node);\n } else {\n source.my_size.fetch_sub(1, std::memory_order_relaxed);\n }\n } else {\n source_prev = curr;\n }\n } else {\n source_prev = source_prev->next();\n }\n }\n }\n\nprivate:\n // Unsafe method, which unlinks the node between prev and next\n void unlink_node( node_ptr prev_node, node_ptr node_to_unlink, node_ptr next_node ) {\n __TBB_ASSERT(prev_node->next() == node_to_unlink &&\n node_to_unlink->next() == next_node,\n \"erasing and extracting nodes from the containers are unsafe in concurrent mode\");\n prev_node->set_next(next_node);\n node_to_unlink->set_next(nullptr);\n }\n\n template \n value_node_ptr internal_find( const K& key ) {\n sokey_type hash_key = sokey_type(my_hash_compare(key));\n sokey_type order_key = split_order_key_regular(hash_key);\n\n node_ptr curr = prepare_bucket(hash_key);\n\n while (curr != nullptr) {\n if (curr->order_key() > order_key) {\n // If the order key is greater than the requested order key,\n // the element is not in the hash table\n return nullptr;\n } else if (curr->order_key() == order_key &&\n my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) {\n // The fact that order keys match does not mean that the element is found.\n // Key function comparison has to be performed to check whether this is the\n // right element. If not, keep searching while order key is the same.\n return static_cast(curr);\n }\n curr = curr->next();\n }\n\n return nullptr;\n }\n\n template \n std::pair internal_equal_range( const K& key ) {\n sokey_type hash_key = sokey_type(my_hash_compare(key));\n sokey_type order_key = split_order_key_regular(hash_key);\n\n node_ptr curr = prepare_bucket(hash_key);\n\n while (curr != nullptr) {\n if (curr->order_key() > order_key) {\n // If the order key is greater than the requested order key,\n // the element is not in the hash table\n return std::make_pair(nullptr, nullptr);\n } else if (curr->order_key() == order_key &&\n my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) {\n value_node_ptr first = static_cast(curr);\n node_ptr last = first;\n do {\n last = last->next();\n } while (allow_multimapping && last != nullptr && !last->is_dummy() &&\n my_hash_compare(traits_type::get_key(static_cast(last)->value()), key));\n return std::make_pair(first, first_value_node(last));\n }\n curr = curr->next();\n }\n return {nullptr, nullptr};\n }\n\n template \n size_type internal_count( const K& key ) const {\n if (allow_multimapping) {\n // TODO: consider reimplementing the internal_equal_range with elements counting to avoid std::distance\n auto eq_range = equal_range(key);\n return std::distance(eq_range.first, eq_range.second);\n } else {\n return contains(key) ? 1 : 0;\n }\n }\n\n void internal_copy( const concurrent_unordered_base& other ) {\n node_ptr last_node = &my_head;\n my_segments[0].store(&my_head, std::memory_order_relaxed);\n\n for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) {\n node_ptr new_node;\n if (!node->is_dummy()) {\n // The node in the right table contains a value\n new_node = create_node(node->order_key(), static_cast(node)->value());\n } else {\n // The node in the right table is a dummy node\n new_node = create_dummy_node(node->order_key());\n my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed);\n }\n\n last_node->set_next(new_node);\n last_node = new_node;\n }\n }\n\n void internal_move( concurrent_unordered_base&& other ) {\n node_ptr last_node = &my_head;\n my_segments[0].store(&my_head, std::memory_order_relaxed);\n\n for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) {\n node_ptr new_node;\n if (!node->is_dummy()) {\n // The node in the right table contains a value\n new_node = create_node(node->order_key(), std::move(static_cast(node)->value()));\n } else {\n // TODO: do we need to destroy a dummy node in the right container?\n // The node in the right table is a dummy_node\n new_node = create_dummy_node(node->order_key());\n my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed);\n }\n\n last_node->set_next(new_node);\n last_node = new_node;\n }\n }\n\n void move_content( concurrent_unordered_base&& other ) {\n // NOTE: allocators should be equal\n my_head.set_next(other.my_head.next());\n other.my_head.set_next(nullptr);\n my_segments[0].store(&my_head, std::memory_order_relaxed);\n\n other.my_bucket_count.store(initial_bucket_count, std::memory_order_relaxed);\n other.my_max_load_factor = initial_max_load_factor;\n other.my_size.store(0, std::memory_order_relaxed);\n }\n\n void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type&,\n /*is_always_equal = */std::true_type ) {\n // Allocators are always equal - no need to compare for equality\n move_content(std::move(other));\n }\n\n void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type& alloc,\n /*is_always_equal = */std::false_type ) {\n // Allocators are not always equal\n if (alloc == other.my_segments.get_allocator()) {\n move_content(std::move(other));\n } else {\n try_call( [&] {\n internal_move(std::move(other));\n } ).on_exception( [&] {\n clear();\n });\n }\n }\n\n // Move assigns the hash table to other is any instances of allocator_type are always equal\n // or propagate_on_container_move_assignment is true\n void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::true_type ) {\n move_content(std::move(other));\n }\n\n // Move assigns the hash table to other is any instances of allocator_type are not always equal\n // and propagate_on_container_move_assignment is false\n void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::false_type ) {\n if (my_segments.get_allocator() == other.my_segments.get_allocator()) {\n move_content(std::move(other));\n } else {\n // TODO: guards for exceptions\n internal_move(std::move(other));\n }\n }\n\n void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::true_type ) {\n internal_swap_fields(other);\n }\n\n void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::false_type ) {\n __TBB_ASSERT(my_segments.get_allocator() == other.my_segments.get_allocator(),\n \"Swapping with unequal allocators is not allowed\");\n internal_swap_fields(other);\n }\n\n void internal_swap_fields( concurrent_unordered_base& other ) {\n node_ptr first_node = my_head.next();\n my_head.set_next(other.my_head.next());\n other.my_head.set_next(first_node);\n\n size_type current_size = my_size.load(std::memory_order_relaxed);\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_size.store(current_size, std::memory_order_relaxed);\n\n size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed);\n my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed);\n other.my_bucket_count.store(bucket_count, std::memory_order_relaxed);\n\n using std::swap;\n swap(my_max_load_factor, other.my_max_load_factor);\n swap(my_hash_compare, other.my_hash_compare);\n my_segments.swap(other.my_segments);\n\n // swap() method from segment table swaps all of the segments including the first segment\n // We should restore it to my_head. Without it the first segment of the container will point\n // to other.my_head.\n my_segments[0].store(&my_head, std::memory_order_relaxed);\n other.my_segments[0].store(&other.my_head, std::memory_order_relaxed);\n }\n\n // A regular order key has its original hash value reversed and the last bit set\n static constexpr sokey_type split_order_key_regular( sokey_type hash ) {\n return reverse_bits(hash) | 0x1;\n }\n\n // A dummy order key has its original hash value reversed and the last bit unset\n static constexpr sokey_type split_order_key_dummy( sokey_type hash ) {\n return reverse_bits(hash) & ~sokey_type(0x1);\n }\n\n size_type get_parent( size_type bucket ) const {\n // Unset bucket's most significant turned-on bit\n __TBB_ASSERT(bucket != 0, \"Unable to get_parent of the bucket 0\");\n size_type msb = tbb::detail::log2(bucket);\n return bucket & ~(size_type(1) << msb);\n }\n\n size_type get_next_bucket_index( size_type bucket ) const {\n size_type bits = tbb::detail::log2(my_bucket_count.load(std::memory_order_relaxed));\n size_type reversed_next = reverse_n_bits(bucket, bits) + 1;\n return reverse_n_bits(reversed_next, bits);\n }\n\n std::atomic my_size;\n std::atomic my_bucket_count;\n float my_max_load_factor;\n hash_compare_type my_hash_compare;\n\n list_node_type my_head; // Head node for split ordered list\n unordered_segment_table my_segments; // Segment table of pointers to nodes\n\n template \n friend class solist_iterator;\n\n template \n friend class concurrent_unordered_base;\n}; // class concurrent_unordered_base\n\ntemplate \nbool operator==( const concurrent_unordered_base& lhs,\n const concurrent_unordered_base& rhs ) {\n if (&lhs == &rhs) { return true; }\n if (lhs.size() != rhs.size()) { return false; }\n\n#if _MSC_VER\n // Passing \"unchecked\" iterators to std::permutation with 3 parameters\n // causes compiler warnings.\n // The workaround is to use overload with 4 parameters, which is\n // available since C++14 - minimally supported version on MSVC\n return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());\n#else\n return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin());\n#endif\n}\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate \nbool operator!=( const concurrent_unordered_base& lhs,\n const concurrent_unordered_base& rhs ) {\n return !(lhs == rhs);\n}\n#endif\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(pop) // warning 4127 is back\n#endif\n\n} // namespace d2\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__concurrent_unordered_base_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_config.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__config_H\n#define __TBB_detail__config_H\n\n/** This header is supposed to contain macro definitions only.\n The macros defined here are intended to control such aspects of TBB build as\n - presence of compiler features\n - compilation modes\n - feature sets\n - known compiler/platform issues\n**/\n\n/* Check which standard library we use. */\n#include \n\n#ifdef __has_include\n#if __has_include()\n#include \n#endif\n#endif\n\n#include \"_export.h\"\n\n#if _MSC_VER\n #define __TBB_EXPORTED_FUNC __cdecl\n #define __TBB_EXPORTED_METHOD __thiscall\n#else\n #define __TBB_EXPORTED_FUNC\n #define __TBB_EXPORTED_METHOD\n#endif\n\n#if defined(_MSVC_LANG)\n #define __TBB_LANG _MSVC_LANG\n#else\n #define __TBB_LANG __cplusplus\n#endif // _MSVC_LANG\n\n#define __TBB_CPP14_PRESENT (__TBB_LANG >= 201402L)\n#define __TBB_CPP17_PRESENT (__TBB_LANG >= 201703L)\n#define __TBB_CPP20_PRESENT (__TBB_LANG >= 202002L)\n\n#if __INTEL_COMPILER || _MSC_VER\n #define __TBB_NOINLINE(decl) __declspec(noinline) decl\n#elif __GNUC__\n #define __TBB_NOINLINE(decl) decl __attribute__ ((noinline))\n#else\n #define __TBB_NOINLINE(decl) decl\n#endif\n\n#define __TBB_STRING_AUX(x) #x\n#define __TBB_STRING(x) __TBB_STRING_AUX(x)\n\n// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match\n// the actual GCC version on the system.\n#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)\n\n/* Check which standard library we use. */\n\n// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro.\n// Therefore we use different ways to detect its version.\n#ifdef TBB_USE_GLIBCXX_VERSION\n // The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro.\n // Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1.\n #define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION\n#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__\n // Reported versions of GCC and libstdc++ do not match; trust the latter\n #define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000)\n#elif __GLIBCPP__ || __GLIBCXX__\n // The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION\n #define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION\n#endif\n\n#if __clang__\n // according to clang documentation, version can be vendor specific\n #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)\n#endif\n\n/** Macro helpers **/\n\n#define __TBB_CONCAT_AUX(A,B) A##B\n// The additional level of indirection is needed to expand macros A and B (not to get the AB macro).\n// See [cpp.subst] and [cpp.concat] for more details.\n#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B)\n// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty).\n#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A)\n#define __TBB_MACRO_EMPTY 1\n\n#if _M_X64 || _M_ARM64\n #define __TBB_W(name) name##64\n#else\n #define __TBB_W(name) name\n#endif\n\n/** User controlled TBB features & modes **/\n\n#ifndef TBB_USE_DEBUG\n /*\n There are four cases that are supported:\n 1. \"_DEBUG is undefined\" means \"no debug\";\n 2. \"_DEBUG defined to something that is evaluated to 0\" (including \"garbage\", as per [cpp.cond]) means \"no debug\";\n 3. \"_DEBUG defined to something that is evaluated to a non-zero value\" means \"debug\";\n 4. \"_DEBUG defined to nothing (empty)\" means \"debug\".\n */\n #ifdef _DEBUG\n // Check if _DEBUG is empty.\n #define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY)\n #if __TBB_IS__DEBUG_EMPTY\n #define TBB_USE_DEBUG 1\n #else\n #define TBB_USE_DEBUG _DEBUG\n #endif // __TBB_IS__DEBUG_EMPTY\n #else\n #define TBB_USE_DEBUG 0\n #endif // _DEBUG\n#endif // TBB_USE_DEBUG\n\n#ifndef TBB_USE_ASSERT\n #define TBB_USE_ASSERT TBB_USE_DEBUG\n#endif // TBB_USE_ASSERT\n\n#ifndef TBB_USE_PROFILING_TOOLS\n#if TBB_USE_DEBUG\n #define TBB_USE_PROFILING_TOOLS 2\n#else // TBB_USE_DEBUG\n #define TBB_USE_PROFILING_TOOLS 0\n#endif // TBB_USE_DEBUG\n#endif // TBB_USE_PROFILING_TOOLS\n\n// Exceptions support cases\n#if !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC)\n #if TBB_USE_EXCEPTIONS\n #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0.\n #elif !defined(TBB_USE_EXCEPTIONS)\n #define TBB_USE_EXCEPTIONS 0\n #endif\n#elif !defined(TBB_USE_EXCEPTIONS)\n #define TBB_USE_EXCEPTIONS 1\n#endif\n\n/** Preprocessor symbols to determine HW architecture **/\n\n#if _WIN32 || _WIN64\n #if defined(_M_X64) || defined(__x86_64__) // the latter for MinGW support\n #define __TBB_x86_64 1\n #elif defined(_M_IA64)\n #define __TBB_ipf 1\n #elif defined(_M_IX86) || defined(__i386__) // the latter for MinGW support\n #define __TBB_x86_32 1\n #else\n #define __TBB_generic_arch 1\n #endif\n#else /* Assume generic Unix */\n #if __x86_64__\n #define __TBB_x86_64 1\n #elif __ia64__\n #define __TBB_ipf 1\n #elif __i386__||__i386 // __i386 is for Sun OS\n #define __TBB_x86_32 1\n #else\n #define __TBB_generic_arch 1\n #endif\n#endif\n\n/** Windows API or POSIX API **/\n\n#if _WIN32 || _WIN64\n #define __TBB_USE_WINAPI 1\n#else\n #define __TBB_USE_POSIX 1\n#endif\n\n/** Internal TBB features & modes **/\n\n/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/\n#ifndef __TBB_DYNAMIC_LOAD_ENABLED\n #define __TBB_DYNAMIC_LOAD_ENABLED (!__EMSCRIPTEN__)\n#endif\n\n/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load\n shared libraries at run time only from application container **/\n#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP\n #define __TBB_WIN8UI_SUPPORT 1\n#else\n #define __TBB_WIN8UI_SUPPORT 0\n#endif\n\n/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/\n#ifndef __TBB_WEAK_SYMBOLS_PRESENT\n #define __TBB_WEAK_SYMBOLS_PRESENT ( !__EMSCRIPTEN__ && !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) )\n#endif\n\n/** Presence of compiler features **/\n\n#if __clang__ && !__INTEL_COMPILER\n #define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti)\n#elif defined(_CPPRTTI)\n #define __TBB_USE_OPTIONAL_RTTI 1\n#else\n #define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__)\n#endif\n\n/** Address sanitizer detection **/\n#ifdef __SANITIZE_ADDRESS__\n #define __TBB_USE_ADDRESS_SANITIZER 1\n#elif defined(__has_feature)\n#if __has_feature(address_sanitizer)\n #define __TBB_USE_ADDRESS_SANITIZER 1\n#endif\n#endif\n\n/** Library features presence macros **/\n\n#define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__TBB_LANG >= 201402L)\n#define __TBB_CPP17_INVOKE_PRESENT (__TBB_LANG >= 201703L)\n\n// TODO: Remove the condition(__INTEL_COMPILER > 2021) from the __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n// macro when this feature start working correctly on this compiler.\n#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MOVE__)\n #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L)\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 2021 && __TBB_LANG >= 201703L)\n #define __TBB_CPP20_CONCEPTS_PRESENT 0 // TODO: add a mechanism for future addition\n#elif __clang__\n #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates))\n #define __TBB_CPP20_CONCEPTS_PRESENT 0 // TODO: add a mechanism for future addition\n #ifdef __cpp_deduction_guides\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201611L)\n #else\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT 0\n #endif\n#elif __GNUC__\n #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L && __TBB_GCC_VERSION >= 50000)\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L)\n #define __TBB_CPP20_CONCEPTS_PRESENT (__TBB_LANG >= 201709L && __TBB_GCC_VERSION >= 100201)\n#elif _MSC_VER\n #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700))\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSC_VER >= 1914 && __TBB_LANG >= 201703L && (!__INTEL_COMPILER || __INTEL_COMPILER > 2021))\n #define __TBB_CPP20_CONCEPTS_PRESENT (_MSC_VER >= 1923 && __TBB_LANG >= 202002L) // TODO: INTEL_COMPILER?\n#else\n #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L)\n #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__TBB_LANG >= 201703L)\n #define __TBB_CPP20_CONCEPTS_PRESENT (__TBB_LANG >= 202002L)\n#endif\n\n// GCC4.8 on RHEL7 does not support std::get_new_handler\n#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION)\n// GCC4.8 on RHEL7 does not support std::is_trivially_copyable\n#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__))\n\n#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (__TBB_LANG > 201402L) || \\\n __TBB_GLIBCXX_VERSION >= 90000 && __TBB_LANG >= 201703L)\n#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911)\n#define __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT (__TBB_LANG >= 201703L)\n#define __TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT (__TBB_LANG >= 201703L)\n#define __TBB_CPP17_IS_SWAPPABLE_PRESENT (__TBB_LANG >= 201703L)\n\n#if defined(__cpp_impl_three_way_comparison) && defined(__cpp_lib_three_way_comparison)\n #define __TBB_CPP20_COMPARISONS_PRESENT ((__cpp_impl_three_way_comparison >= 201907L) && (__cpp_lib_three_way_comparison >= 201907L))\n#else\n #define __TBB_CPP20_COMPARISONS_PRESENT 0\n#endif\n\n#define __TBB_RESUMABLE_TASKS (!__TBB_WIN8UI_SUPPORT && !__ANDROID__ && !__QNXNTO__ && (!__linux__ || __GLIBC__))\n\n/* This macro marks incomplete code or comments describing ideas which are considered for the future.\n * See also for plain comment with TODO and FIXME marks for small improvement opportunities.\n */\n#define __TBB_TODO 0\n\n/* Check which standard library we use. */\n/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */\n#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY\n #include \n#endif\n\n/** Target OS is either iOS* or iOS* simulator **/\n#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__\n #define __TBB_IOS 1\n#endif\n\n#if __APPLE__\n #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \\\n && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000\n // ICC does not correctly set the macro if -mmacosx-min-version is not specified\n #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000))\n #else\n #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__\n #endif\n#endif\n\n#if defined(__GNUC__) && !defined(__INTEL_COMPILER)\n #define __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT (__TBB_GCC_VERSION >= 60100)\n#endif\n\n#if __GNUC__ && !__INTEL_COMPILER && !__clang__\n #define __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN (__TBB_GCC_VERSION <= 40805)\n#endif\n\n#define __TBB_CPP17_FALLTHROUGH_PRESENT (__TBB_LANG >= 201703L)\n#define __TBB_CPP17_NODISCARD_PRESENT (__TBB_LANG >= 201703L)\n#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER)\n\n#if __TBB_CPP17_FALLTHROUGH_PRESENT\n #define __TBB_fallthrough [[fallthrough]]\n#elif __TBB_FALLTHROUGH_PRESENT\n #define __TBB_fallthrough __attribute__ ((fallthrough))\n#else\n #define __TBB_fallthrough\n#endif\n\n#if __TBB_CPP17_NODISCARD_PRESENT\n #define __TBB_nodiscard [[nodiscard]]\n#elif __clang__ || __GNUC__\n #define __TBB_nodiscard __attribute__((warn_unused_result))\n#else\n #define __TBB_nodiscard\n#endif\n\n#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \\\n || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200))\n\n#define __TBB_TSX_INTRINSICS_PRESENT (__RTM__ || __INTEL_COMPILER || (_MSC_VER>=1700 && (__TBB_x86_64 || __TBB_x86_32)))\n\n#define __TBB_WAITPKG_INTRINSICS_PRESENT ((__INTEL_COMPILER >= 1900 || (__TBB_GCC_VERSION >= 110000 && __TBB_GNU_ASM_VERSION >= 2032) || __TBB_CLANG_VERSION >= 120000) \\\n && (_WIN32 || _WIN64 || __unix__ || __APPLE__) && (__TBB_x86_32 || __TBB_x86_64) && !__ANDROID__)\n\n/** Internal TBB features & modes **/\n\n/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when\n it's necessary to test internal functions not exported from TBB DLLs\n**/\n#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY)\n #define __TBB_NO_IMPLICIT_LINKAGE 1\n #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1\n#endif\n\n#if (__TBB_BUILD || __TBBMALLOC_BUILD || __TBBMALLOCPROXY_BUILD || __TBBBIND_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE)\n #define __TBB_NO_IMPLICIT_LINKAGE 1\n#endif\n\n#if _MSC_VER\n #if !__TBB_NO_IMPLICIT_LINKAGE\n #ifdef _DEBUG\n #pragma comment(lib, \"tbb12_debug.lib\")\n #else\n #pragma comment(lib, \"tbb12.lib\")\n #endif\n #endif\n#endif\n\n#ifndef __TBB_SCHEDULER_OBSERVER\n #define __TBB_SCHEDULER_OBSERVER 1\n#endif /* __TBB_SCHEDULER_OBSERVER */\n\n#ifndef __TBB_FP_CONTEXT\n #define __TBB_FP_CONTEXT 1\n#endif /* __TBB_FP_CONTEXT */\n\n#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official\n\n#ifndef __TBB_ARENA_OBSERVER\n #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER\n#endif /* __TBB_ARENA_OBSERVER */\n\n#ifndef __TBB_ARENA_BINDING\n #define __TBB_ARENA_BINDING 1\n#endif\n\n// Thread pinning is not available on macOS*\n#define __TBB_CPUBIND_PRESENT (__TBB_ARENA_BINDING && !__APPLE__)\n\n#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY\n #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1\n#endif\n\n#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \\\n (_WIN32 || _WIN64 || __APPLE__ || (defined(__unix__) && !__ANDROID__))\n #define __TBB_SURVIVE_THREAD_SWITCH 1\n#endif /* __TBB_SURVIVE_THREAD_SWITCH */\n\n#ifndef TBB_PREVIEW_FLOW_GRAPH_FEATURES\n #define TBB_PREVIEW_FLOW_GRAPH_FEATURES __TBB_CPF_BUILD\n#endif\n\n#ifndef __TBB_DEFAULT_PARTITIONER\n #define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner\n#endif\n\n#ifndef __TBB_FLOW_TRACE_CODEPTR\n #define __TBB_FLOW_TRACE_CODEPTR __TBB_CPF_BUILD\n#endif\n\n// Intel(R) C++ Compiler starts analyzing usages of the deprecated content at the template\n// instantiation site, which is too late for suppression of the corresponding messages for internal\n// stuff.\n#if !defined(__INTEL_COMPILER) && (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0))\n #if (__TBB_LANG >= 201402L && (!defined(_MSC_VER) || _MSC_VER >= 1920))\n #define __TBB_DEPRECATED [[deprecated]]\n #define __TBB_DEPRECATED_MSG(msg) [[deprecated(msg)]]\n #elif _MSC_VER\n #define __TBB_DEPRECATED __declspec(deprecated)\n #define __TBB_DEPRECATED_MSG(msg) __declspec(deprecated(msg))\n #elif (__GNUC__ && __TBB_GCC_VERSION >= 40805) || __clang__\n #define __TBB_DEPRECATED __attribute__((deprecated))\n #define __TBB_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))\n #endif\n#endif // !defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)\n\n#if !defined(__TBB_DEPRECATED)\n #define __TBB_DEPRECATED\n #define __TBB_DEPRECATED_MSG(msg)\n#elif !defined(__TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES)\n // Suppress deprecated messages from self\n #define __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES 1\n#endif\n\n#if defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) && (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)\n #define __TBB_DEPRECATED_VERBOSE __TBB_DEPRECATED\n #define __TBB_DEPRECATED_VERBOSE_MSG(msg) __TBB_DEPRECATED_MSG(msg)\n#else\n #define __TBB_DEPRECATED_VERBOSE\n #define __TBB_DEPRECATED_VERBOSE_MSG(msg)\n#endif // (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)\n\n#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !(__TBB_LANG >= 201103L || _MSC_VER >= 1900)\n #pragma message(\"TBB Warning: Support for C++98/03 is deprecated. Please use the compiler that supports C++11 features at least.\")\n#endif\n\n#ifdef _VARIADIC_MAX\n #define __TBB_VARIADIC_MAX _VARIADIC_MAX\n#else\n #if _MSC_VER == 1700\n #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12\n #elif _MSC_VER == 1600\n #define __TBB_VARIADIC_MAX 10 // VS10 setting\n #else\n #define __TBB_VARIADIC_MAX 15\n #endif\n#endif\n\n#if __SANITIZE_THREAD__\n #define __TBB_USE_THREAD_SANITIZER 1\n#elif defined(__has_feature)\n#if __has_feature(thread_sanitizer)\n #define __TBB_USE_THREAD_SANITIZER 1\n#endif\n#endif\n\n#ifndef __TBB_USE_SANITIZERS\n#define __TBB_USE_SANITIZERS (__TBB_USE_THREAD_SANITIZER || __TBB_USE_ADDRESS_SANITIZER)\n#endif\n\n#ifndef __TBB_RESUMABLE_TASKS_USE_THREADS\n#define __TBB_RESUMABLE_TASKS_USE_THREADS __TBB_USE_SANITIZERS\n#endif\n\n#ifndef __TBB_USE_CONSTRAINTS\n#define __TBB_USE_CONSTRAINTS 1\n#endif\n\n#ifndef __TBB_STRICT_CONSTRAINTS\n#define __TBB_STRICT_CONSTRAINTS 1\n#endif\n\n#if __TBB_CPP20_CONCEPTS_PRESENT && __TBB_USE_CONSTRAINTS\n #define __TBB_requires(...) requires __VA_ARGS__\n#else // __TBB_CPP20_CONCEPTS_PRESENT\n #define __TBB_requires(...)\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n\n/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by\n the bugs in compilers, standard or OS specific libraries. They should be\n removed as soon as the corresponding bugs are fixed or the buggy OS/compiler\n versions go out of the support list.\n**/\n\n// Some STL containers not support allocator traits in old GCC versions\n#if __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION <= 50301\n #define TBB_ALLOCATOR_TRAITS_BROKEN 1\n#endif\n\n// GCC 4.8 C++ standard library implements std::this_thread::yield as no-op.\n#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900\n #define __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 1\n#endif\n\n/** End of __TBB_XXX_BROKEN macro section **/\n\n#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER)\n // A macro to suppress erroneous or benign \"unreachable code\" MSVC warning (4702)\n #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1\n#endif\n\n// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang\n// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19)\n#if __ANDROID__\n #include \n#endif\n\n#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES)\n\n#ifndef __TBB_PREVIEW_CRITICAL_TASKS\n#define __TBB_PREVIEW_CRITICAL_TASKS 1\n#endif\n\n#ifndef __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n#define __TBB_PREVIEW_FLOW_GRAPH_NODE_SET (TBB_PREVIEW_FLOW_GRAPH_FEATURES)\n#endif\n\n#ifndef __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n#define __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT (TBB_PREVIEW_FLOW_GRAPH_FEATURES \\\n || TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT)\n#endif\n\n#if TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS\n#define __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS 1\n#endif\n\n#if TBB_PREVIEW_TASK_GROUP_EXTENSIONS || __TBB_BUILD\n#define __TBB_PREVIEW_TASK_GROUP_EXTENSIONS 1\n#endif\n\n#endif // __TBB_detail__config_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_containers_helpers.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__containers_helpers_H\n#define __TBB_detail__containers_helpers_H\n\n#include \"_template_helpers.h\"\n#include \"_allocator_traits.h\"\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\ntemplate \nstruct comp_is_transparent : std::false_type {};\n\ntemplate \nstruct comp_is_transparent> : std::true_type {};\n\ntemplate \nstruct has_transparent_key_equal : std::false_type { using type = KeyEqual; };\n\ntemplate \nstruct has_transparent_key_equal> : std::true_type {\n using type = typename Hasher::transparent_key_equal;\n static_assert(comp_is_transparent::value, \"Hash::transparent_key_equal::is_transparent is not valid or does not denote a type.\");\n static_assert((std::is_same>::value ||\n std::is_same::value), \"KeyEqual is a different type than equal_to or Hash::transparent_key_equal.\");\n };\n\nstruct is_iterator_impl {\ntemplate \nusing iter_traits_category = typename std::iterator_traits::iterator_category;\n\ntemplate \nusing input_iter_category = typename std::enable_if>::value>::type;\n}; // struct is_iterator_impl\n\ntemplate \nusing is_input_iterator = supports;\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate \ninline constexpr bool is_input_iterator_v = is_input_iterator::value;\n#endif\n\n} // inline namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__containers_helpers_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_exception.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__exception_H\n#define __TBB__exception_H\n\n#include \"_config.h\"\n\n#include // std::bad_alloc\n#include // std::exception\n#include // std::runtime_error\n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\nenum class exception_id {\n bad_alloc = 1,\n bad_last_alloc,\n user_abort,\n nonpositive_step,\n out_of_range,\n reservation_length_error,\n missing_wait,\n invalid_load_factor,\n invalid_key,\n bad_tagged_msg_cast,\n unsafe_wait,\n last_entry\n};\n} // namespace d0\n\n#if _MSC_VER\n // #pragma warning(disable: 4275)\n#endif\n\nnamespace r1 {\n//! Exception for concurrent containers\nclass TBB_EXPORT bad_last_alloc : public std::bad_alloc {\npublic:\n const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override;\n};\n\n//! Exception for user-initiated abort\nclass TBB_EXPORT user_abort : public std::exception {\npublic:\n const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override;\n};\n\n//! Exception for missing wait on structured_task_group\nclass TBB_EXPORT missing_wait : public std::exception {\npublic:\n const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override;\n};\n\n//! Exception for impossible finalization of task_sheduler_handle\nclass TBB_EXPORT unsafe_wait : public std::runtime_error {\npublic:\n unsafe_wait(const char* msg) : std::runtime_error(msg) {}\n};\n\n//! Gathers all throw operators in one place.\n/** Its purpose is to minimize code bloat that can be caused by throw operators\n scattered in multiple places, especially in templates. **/\nTBB_EXPORT void __TBB_EXPORTED_FUNC throw_exception ( exception_id );\n} // namespace r1\n\ninline namespace d0 {\nusing r1::throw_exception;\n} // namespace d0\n\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB__exception_H\n\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_export.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__export_H\n#define __TBB_detail__export_H\n\n#if defined(__MINGW32__)\n #define _EXPORT __declspec(dllexport)\n#elif defined(_WIN32) || defined(__unix__) || defined(__APPLE__) // Use .def files for these\n #define _EXPORT\n#else\n #error \"Unknown platform/compiler\"\n#endif\n\n#if __TBB_BUILD\n #define TBB_EXPORT _EXPORT\n#else\n #define TBB_EXPORT\n#endif\n\n#if __TBBMALLOC_BUILD\n #define TBBMALLOC_EXPORT _EXPORT\n#else\n #define TBBMALLOC_EXPORT\n#endif\n\n#if __TBBBIND_BUILD\n #define TBBBIND_EXPORT _EXPORT\n#else\n #define TBBBIND_EXPORT\n#endif\n\n#endif\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_body_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_body_impl_H\n#define __TBB__flow_graph_body_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included in namespace tbb::detail::d2 (in flow_graph.h)\n\ntypedef std::uint64_t tag_value;\n\n\n// TODO revamp: find out if there is already helper for has_policy.\ntemplate struct Policy {};\n\ntemplate struct has_policy;\n\ntemplate\nstruct has_policy :\n std::integral_constant::value ||\n has_policy::value> {};\n\ntemplate\nstruct has_policy :\n std::integral_constant::value> {};\n\ntemplate\nstruct has_policy > : has_policy {};\n\nnamespace graph_policy_namespace {\n\n struct rejecting { };\n struct reserving { };\n struct queueing { };\n struct lightweight { };\n\n // K == type of field used for key-matching. Each tag-matching port will be provided\n // functor that, given an object accepted by the port, will return the\n /// field of type K being used for matching.\n template::type > >\n __TBB_requires(tbb::detail::hash_compare)\n struct key_matching {\n typedef K key_type;\n typedef typename std::decay::type base_key_type;\n typedef KHash hash_compare_type;\n };\n\n // old tag_matching join's new specifier\n typedef key_matching tag_matching;\n\n // Aliases for Policy combinations\n typedef Policy queueing_lightweight;\n typedef Policy rejecting_lightweight;\n\n} // namespace graph_policy_namespace\n\n// -------------- function_body containers ----------------------\n\n//! A functor that takes no input and generates a value of type Output\ntemplate< typename Output >\nclass input_body : no_assign {\npublic:\n virtual ~input_body() {}\n virtual Output operator()(d1::flow_control& fc) = 0;\n virtual input_body* clone() = 0;\n};\n\n//! The leaf for input_body\ntemplate< typename Output, typename Body>\nclass input_body_leaf : public input_body {\npublic:\n input_body_leaf( const Body &_body ) : body(_body) { }\n Output operator()(d1::flow_control& fc) override { return body(fc); }\n input_body_leaf* clone() override {\n return new input_body_leaf< Output, Body >(body);\n }\n Body get_body() { return body; }\nprivate:\n Body body;\n};\n\n//! A functor that takes an Input and generates an Output\ntemplate< typename Input, typename Output >\nclass function_body : no_assign {\npublic:\n virtual ~function_body() {}\n virtual Output operator()(const Input &input) = 0;\n virtual function_body* clone() = 0;\n};\n\n//! the leaf for function_body\ntemplate \nclass function_body_leaf : public function_body< Input, Output > {\npublic:\n function_body_leaf( const B &_body ) : body(_body) { }\n Output operator()(const Input &i) override { return tbb::detail::invoke(body,i); }\n B get_body() { return body; }\n function_body_leaf* clone() override {\n return new function_body_leaf< Input, Output, B >(body);\n }\nprivate:\n B body;\n};\n\n//! the leaf for function_body specialized for Input and output of continue_msg\ntemplate \nclass function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > {\npublic:\n function_body_leaf( const B &_body ) : body(_body) { }\n continue_msg operator()( const continue_msg &i ) override {\n body(i);\n return i;\n }\n B get_body() { return body; }\n function_body_leaf* clone() override {\n return new function_body_leaf< continue_msg, continue_msg, B >(body);\n }\nprivate:\n B body;\n};\n\n//! the leaf for function_body specialized for Output of continue_msg\ntemplate \nclass function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > {\npublic:\n function_body_leaf( const B &_body ) : body(_body) { }\n continue_msg operator()(const Input &i) override {\n body(i);\n return continue_msg();\n }\n B get_body() { return body; }\n function_body_leaf* clone() override {\n return new function_body_leaf< Input, continue_msg, B >(body);\n }\nprivate:\n B body;\n};\n\n//! the leaf for function_body specialized for Input of continue_msg\ntemplate \nclass function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > {\npublic:\n function_body_leaf( const B &_body ) : body(_body) { }\n Output operator()(const continue_msg &i) override {\n return body(i);\n }\n B get_body() { return body; }\n function_body_leaf* clone() override {\n return new function_body_leaf< continue_msg, Output, B >(body);\n }\nprivate:\n B body;\n};\n\n//! function_body that takes an Input and a set of output ports\ntemplate\nclass multifunction_body : no_assign {\npublic:\n virtual ~multifunction_body () {}\n virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0;\n virtual multifunction_body* clone() = 0;\n virtual void* get_body_ptr() = 0;\n};\n\n//! leaf for multifunction. OutputSet can be a std::tuple or a vector.\ntemplate\nclass multifunction_body_leaf : public multifunction_body {\npublic:\n multifunction_body_leaf(const B &_body) : body(_body) { }\n void operator()(const Input &input, OutputSet &oset) override {\n tbb::detail::invoke(body, input, oset); // body may explicitly put() to one or more of oset.\n }\n void* get_body_ptr() override { return &body; }\n multifunction_body_leaf* clone() override {\n return new multifunction_body_leaf(body);\n }\n\nprivate:\n B body;\n};\n\n// ------ function bodies for hash_buffers and key-matching joins.\n\ntemplate\nclass type_to_key_function_body : no_assign {\n public:\n virtual ~type_to_key_function_body() {}\n virtual Output operator()(const Input &input) = 0; // returns an Output\n virtual type_to_key_function_body* clone() = 0;\n};\n\n// specialization for ref output\ntemplate\nclass type_to_key_function_body : no_assign {\n public:\n virtual ~type_to_key_function_body() {}\n virtual const Output & operator()(const Input &input) = 0; // returns a const Output&\n virtual type_to_key_function_body* clone() = 0;\n};\n\ntemplate \nclass type_to_key_function_body_leaf : public type_to_key_function_body {\npublic:\n type_to_key_function_body_leaf( const B &_body ) : body(_body) { }\n Output operator()(const Input &i) override { return tbb::detail::invoke(body, i); }\n type_to_key_function_body_leaf* clone() override {\n return new type_to_key_function_body_leaf< Input, Output, B>(body);\n }\nprivate:\n B body;\n};\n\ntemplate \nclass type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> {\npublic:\n type_to_key_function_body_leaf( const B &_body ) : body(_body) { }\n const Output& operator()(const Input &i) override {\n return tbb::detail::invoke(body, i);\n }\n type_to_key_function_body_leaf* clone() override {\n return new type_to_key_function_body_leaf< Input, Output&, B>(body);\n }\nprivate:\n B body;\n};\n\n// --------------------------- end of function_body containers ------------------------\n\n// --------------------------- node task bodies ---------------------------------------\n\n//! A task that calls a node's forward_task function\ntemplate< typename NodeType >\nclass forward_task_bypass : public graph_task {\n NodeType &my_node;\npublic:\n forward_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n\n , node_priority_t node_priority = no_priority\n ) : graph_task(g, allocator, node_priority),\n my_node(n) {}\n\n d1::task* execute(d1::execution_data& ed) override {\n graph_task* next_task = my_node.forward_task();\n if (SUCCESSFULLY_ENQUEUED == next_task)\n next_task = nullptr;\n else if (next_task)\n next_task = prioritize_task(my_node.graph_reference(), *next_task);\n finalize(ed);\n return next_task;\n }\n\n d1::task* cancel(d1::execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n};\n\n//! A task that calls a node's apply_body_bypass function, passing in an input of type Input\n// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return nullptr\ntemplate< typename NodeType, typename Input, typename BaseTaskType = graph_task>\nclass apply_body_task_bypass\n : public BaseTaskType\n{\n NodeType &my_node;\n Input my_input;\n\n using check_metainfo = std::is_same;\n using without_metainfo = std::true_type;\n using with_metainfo = std::false_type;\n\n graph_task* call_apply_body_bypass_impl(without_metainfo) {\n return my_node.apply_body_bypass(my_input\n __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* call_apply_body_bypass_impl(with_metainfo) {\n return my_node.apply_body_bypass(my_input, message_metainfo{this->get_msg_wait_context_vertices()});\n }\n#endif\n\n graph_task* call_apply_body_bypass() {\n return call_apply_body_bypass_impl(check_metainfo{});\n }\n\npublic:\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n, const Input &i,\n node_priority_t node_priority, Metainfo&& metainfo )\n : BaseTaskType(g, allocator, node_priority, std::forward(metainfo).waiters())\n , my_node(n), my_input(i) {}\n#endif\n\n apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType& n, const Input& i,\n node_priority_t node_priority = no_priority )\n : BaseTaskType(g, allocator, node_priority), my_node(n), my_input(i) {}\n\n d1::task* execute(d1::execution_data& ed) override {\n graph_task* next_task = call_apply_body_bypass();\n if (SUCCESSFULLY_ENQUEUED == next_task)\n next_task = nullptr;\n else if (next_task)\n next_task = prioritize_task(my_node.graph_reference(), *next_task);\n BaseTaskType::template finalize(ed);\n return next_task;\n }\n\n d1::task* cancel(d1::execution_data& ed) override {\n BaseTaskType::template finalize(ed);\n return nullptr;\n }\n};\n\n//! A task that calls a node's apply_body_bypass function with no input\ntemplate< typename NodeType >\nclass input_node_task_bypass : public graph_task {\n NodeType &my_node;\npublic:\n input_node_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n )\n : graph_task(g, allocator), my_node(n) {}\n\n d1::task* execute(d1::execution_data& ed) override {\n graph_task* next_task = my_node.apply_body_bypass( );\n if (SUCCESSFULLY_ENQUEUED == next_task)\n next_task = nullptr;\n else if (next_task)\n next_task = prioritize_task(my_node.graph_reference(), *next_task);\n finalize(ed);\n return next_task;\n }\n\n d1::task* cancel(d1::execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n};\n\n// ------------------------ end of node task bodies -----------------------------------\n\ntemplate\nclass threshold_regulator;\n\ntemplate\nclass threshold_regulator::value>::type>\n : public receiver, no_copy\n{\n T* my_node;\nprotected:\n\n graph_task* try_put_task( const DecrementType& value ) override {\n graph_task* result = my_node->decrement_counter( value );\n if( !result )\n result = SUCCESSFULLY_ENQUEUED;\n return result;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n // Intentionally ignore the metainformation\n // If there are more items associated with passed metainfo to be processed\n // They should be stored in the buffer before the limiter_node\n graph_task* try_put_task(const DecrementType& value, const message_metainfo&) override {\n return try_put_task(value);\n }\n#endif\n\n graph& graph_reference() const override {\n return my_node->my_graph;\n }\n\n template friend class limiter_node;\n void reset_receiver( reset_flags ) {}\n\npublic:\n threshold_regulator(T* owner) : my_node(owner) {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n};\n\ntemplate\nclass threshold_regulator : public continue_receiver, no_copy {\n\n T *my_node;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n // Intentionally ignore the metainformation\n // If there are more items associated with passed metainfo to be processed\n // They should be stored in the buffer before the limiter_node\n graph_task* execute(const message_metainfo&) override {\n#else\n graph_task* execute() override {\n#endif\n return my_node->decrement_counter( 1 );\n }\n\nprotected:\n\n graph& graph_reference() const override {\n return my_node->my_graph;\n }\n\npublic:\n\n typedef continue_msg input_type;\n typedef continue_msg output_type;\n threshold_regulator(T* owner)\n : continue_receiver( /*number_of_predecessors=*/0, no_priority ), my_node(owner)\n {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n};\n\n#endif // __TBB__flow_graph_body_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_cache_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_cache_impl_H\n#define __TBB__flow_graph_cache_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included in namespace tbb::detail::d2 (in flow_graph.h)\n\n//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock.\ntemplate< typename T, typename M=spin_mutex >\nclass node_cache {\n public:\n\n typedef size_t size_type;\n\n bool empty() {\n typename mutex_type::scoped_lock lock( my_mutex );\n return internal_empty();\n }\n\n void add( T &n ) {\n typename mutex_type::scoped_lock lock( my_mutex );\n internal_push(n);\n }\n\n void remove( T &n ) {\n typename mutex_type::scoped_lock lock( my_mutex );\n for ( size_t i = internal_size(); i != 0; --i ) {\n T &s = internal_pop();\n if ( &s == &n )\n break; // only remove one predecessor per request\n internal_push(s);\n }\n }\n\n void clear() {\n while( !my_q.empty()) (void)my_q.pop();\n }\n\nprotected:\n\n typedef M mutex_type;\n mutex_type my_mutex;\n std::queue< T * > my_q;\n\n // Assumes lock is held\n inline bool internal_empty( ) {\n return my_q.empty();\n }\n\n // Assumes lock is held\n inline size_type internal_size( ) {\n return my_q.size();\n }\n\n // Assumes lock is held\n inline void internal_push( T &n ) {\n my_q.push(&n);\n }\n\n // Assumes lock is held\n inline T &internal_pop() {\n T *v = my_q.front();\n my_q.pop();\n return *v;\n }\n\n};\n\n//! A cache of predecessors that only supports try_get\ntemplate< typename T, typename M=spin_mutex >\nclass predecessor_cache : public node_cache< sender, M > {\npublic:\n typedef M mutex_type;\n typedef T output_type;\n typedef sender predecessor_type;\n typedef receiver successor_type;\n\n predecessor_cache( successor_type* owner ) : my_owner( owner ) {\n __TBB_ASSERT( my_owner, \"predecessor_cache should have an owner.\" );\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\nprivate:\n bool get_item_impl( output_type& v\n __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo_ptr = nullptr) )\n {\n\n bool successful_get = false;\n\n do {\n predecessor_type *src;\n {\n typename mutex_type::scoped_lock lock(this->my_mutex);\n if ( this->internal_empty() ) {\n break;\n }\n src = &this->internal_pop();\n }\n\n // Try to get from this sender\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (metainfo_ptr) {\n successful_get = src->try_get( v, *metainfo_ptr );\n } else\n#endif\n {\n successful_get = src->try_get( v );\n }\n\n if (successful_get == false) {\n // Relinquish ownership of the edge\n register_successor(*src, *my_owner);\n } else {\n // Retain ownership of the edge\n this->add(*src);\n }\n } while ( successful_get == false );\n return successful_get;\n }\npublic:\n bool get_item( output_type& v ) {\n return get_item_impl(v);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool get_item( output_type& v, message_metainfo& metainfo ) {\n return get_item_impl(v, &metainfo);\n }\n#endif\n\n // If we are removing arcs (rf_clear_edges), call clear() rather than reset().\n void reset() {\n for(;;) {\n predecessor_type *src;\n {\n if (this->internal_empty()) break;\n src = &this->internal_pop();\n }\n register_successor(*src, *my_owner);\n }\n }\n\nprotected:\n successor_type* my_owner;\n};\n\n//! An cache of predecessors that supports requests and reservations\ntemplate< typename T, typename M=spin_mutex >\nclass reservable_predecessor_cache : public predecessor_cache< T, M > {\npublic:\n typedef M mutex_type;\n typedef T output_type;\n typedef sender predecessor_type;\n typedef receiver successor_type;\n\n reservable_predecessor_cache( successor_type* owner )\n : predecessor_cache(owner), reserved_src(nullptr)\n {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\nprivate:\n bool try_reserve_impl( output_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo) ) {\n bool successful_reserve = false;\n\n do {\n predecessor_type* pred = nullptr;\n {\n typename mutex_type::scoped_lock lock(this->my_mutex);\n if ( reserved_src.load(std::memory_order_relaxed) || this->internal_empty() )\n return false;\n\n pred = &this->internal_pop();\n reserved_src.store(pred, std::memory_order_relaxed);\n }\n\n // Try to get from this sender\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (metainfo) {\n successful_reserve = pred->try_reserve( v, *metainfo );\n } else\n#endif\n {\n successful_reserve = pred->try_reserve( v );\n }\n\n if (successful_reserve == false) {\n typename mutex_type::scoped_lock lock(this->my_mutex);\n // Relinquish ownership of the edge\n register_successor( *pred, *this->my_owner );\n reserved_src.store(nullptr, std::memory_order_relaxed);\n } else {\n // Retain ownership of the edge\n this->add( *pred);\n }\n } while ( successful_reserve == false );\n\n return successful_reserve;\n }\npublic:\n bool try_reserve( output_type& v ) {\n return try_reserve_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(nullptr));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_reserve( output_type& v, message_metainfo& metainfo ) {\n return try_reserve_impl(v, &metainfo);\n }\n#endif\n\n bool try_release() {\n reserved_src.load(std::memory_order_relaxed)->try_release();\n reserved_src.store(nullptr, std::memory_order_relaxed);\n return true;\n }\n\n bool try_consume() {\n reserved_src.load(std::memory_order_relaxed)->try_consume();\n reserved_src.store(nullptr, std::memory_order_relaxed);\n return true;\n }\n\n void reset() {\n reserved_src.store(nullptr, std::memory_order_relaxed);\n predecessor_cache::reset();\n }\n\n void clear() {\n reserved_src.store(nullptr, std::memory_order_relaxed);\n predecessor_cache::clear();\n }\n\nprivate:\n std::atomic reserved_src;\n};\n\n\n//! An abstract cache of successors\ntemplate\nclass successor_cache : no_copy {\nprotected:\n\n typedef M mutex_type;\n mutex_type my_mutex;\n\n typedef receiver successor_type;\n typedef receiver* pointer_type;\n typedef sender owner_type;\n // TODO revamp: introduce heapified collection of successors for strict priorities\n typedef std::list< pointer_type > successors_type;\n successors_type my_successors;\n\n owner_type* my_owner;\n\npublic:\n successor_cache( owner_type* owner ) : my_owner(owner) {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\n virtual ~successor_cache() {}\n\n void register_successor( successor_type& r ) {\n typename mutex_type::scoped_lock l(my_mutex, true);\n if( r.priority() != no_priority )\n my_successors.push_front( &r );\n else\n my_successors.push_back( &r );\n }\n\n void remove_successor( successor_type& r ) {\n typename mutex_type::scoped_lock l(my_mutex, true);\n for ( typename successors_type::iterator i = my_successors.begin();\n i != my_successors.end(); ++i ) {\n if ( *i == & r ) {\n my_successors.erase(i);\n break;\n }\n }\n }\n\n bool empty() {\n typename mutex_type::scoped_lock l(my_mutex, false);\n return my_successors.empty();\n }\n\n void clear() {\n my_successors.clear();\n }\n\n virtual graph_task* try_put_task( const T& t ) = 0;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) = 0;\n#endif\n}; // successor_cache\n\n//! An abstract cache of successors, specialized to continue_msg\ntemplate\nclass successor_cache< continue_msg, M > : no_copy {\nprotected:\n\n typedef M mutex_type;\n mutex_type my_mutex;\n\n typedef receiver successor_type;\n typedef receiver* pointer_type;\n typedef sender owner_type;\n typedef std::list< pointer_type > successors_type;\n successors_type my_successors;\n owner_type* my_owner;\n\npublic:\n successor_cache( sender* owner ) : my_owner(owner) {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\n virtual ~successor_cache() {}\n\n void register_successor( successor_type& r ) {\n typename mutex_type::scoped_lock l(my_mutex, true);\n if( r.priority() != no_priority )\n my_successors.push_front( &r );\n else\n my_successors.push_back( &r );\n __TBB_ASSERT( my_owner, \"Cache of successors must have an owner.\" );\n if ( r.is_continue_receiver() ) {\n r.register_predecessor( *my_owner );\n }\n }\n\n void remove_successor( successor_type& r ) {\n typename mutex_type::scoped_lock l(my_mutex, true);\n for ( successors_type::iterator i = my_successors.begin(); i != my_successors.end(); ++i ) {\n if ( *i == &r ) {\n __TBB_ASSERT(my_owner, \"Cache of successors must have an owner.\");\n // TODO: check if we need to test for continue_receiver before removing from r.\n r.remove_predecessor( *my_owner );\n my_successors.erase(i);\n break;\n }\n }\n }\n\n bool empty() {\n typename mutex_type::scoped_lock l(my_mutex, false);\n return my_successors.empty();\n }\n\n void clear() {\n my_successors.clear();\n }\n\n virtual graph_task* try_put_task( const continue_msg& t ) = 0;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual graph_task* try_put_task( const continue_msg& t, const message_metainfo& metainfo ) = 0;\n#endif\n}; // successor_cache< continue_msg >\n\n//! A cache of successors that are broadcast to\ntemplate\nclass broadcast_cache : public successor_cache {\n typedef successor_cache base_type;\n typedef M mutex_type;\n typedef typename successor_cache::successors_type successors_type;\n\n graph_task* try_put_task_impl( const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) {\n graph_task * last_task = nullptr;\n typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true);\n typename successors_type::iterator i = this->my_successors.begin();\n while ( i != this->my_successors.end() ) {\n graph_task *new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n // workaround for icc bug\n graph& graph_ref = (*i)->graph_reference();\n last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary\n if(new_task) {\n ++i;\n }\n else { // failed\n if ( (*i)->register_predecessor(*this->my_owner) ) {\n i = this->my_successors.erase(i);\n } else {\n ++i;\n }\n }\n }\n return last_task;\n }\npublic:\n\n broadcast_cache( typename base_type::owner_type* owner ): base_type(owner) {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\n graph_task* try_put_task( const T &t ) override {\n return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task( const T &t, const message_metainfo& metainfo ) override {\n return try_put_task_impl(t, metainfo);\n }\n#endif\n\n // call try_put_task and return list of received tasks\n bool gather_successful_try_puts( const T &t, graph_task_list& tasks ) {\n bool is_at_least_one_put_successful = false;\n typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true);\n typename successors_type::iterator i = this->my_successors.begin();\n while ( i != this->my_successors.end() ) {\n graph_task * new_task = (*i)->try_put_task(t);\n if(new_task) {\n ++i;\n if(new_task != SUCCESSFULLY_ENQUEUED) {\n tasks.push_back(*new_task);\n }\n is_at_least_one_put_successful = true;\n }\n else { // failed\n if ( (*i)->register_predecessor(*this->my_owner) ) {\n i = this->my_successors.erase(i);\n } else {\n ++i;\n }\n }\n }\n return is_at_least_one_put_successful;\n }\n};\n\n//! A cache of successors that are put in a round-robin fashion\ntemplate\nclass round_robin_cache : public successor_cache {\n typedef successor_cache base_type;\n typedef size_t size_type;\n typedef M mutex_type;\n typedef typename successor_cache::successors_type successors_type;\n\npublic:\n\n round_robin_cache( typename base_type::owner_type* owner ): base_type(owner) {\n // Do not work with the passed pointer here as it may not be fully initialized yet\n }\n\n size_type size() {\n typename mutex_type::scoped_lock l(this->my_mutex, false);\n return this->my_successors.size();\n }\n\nprivate:\n\n graph_task* try_put_task_impl( const T &t\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) )\n {\n typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true);\n typename successors_type::iterator i = this->my_successors.begin();\n while ( i != this->my_successors.end() ) {\n graph_task* new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n if ( new_task ) {\n return new_task;\n } else {\n if ( (*i)->register_predecessor(*this->my_owner) ) {\n i = this->my_successors.erase(i);\n }\n else {\n ++i;\n }\n }\n }\n return nullptr;\n }\n\npublic:\n graph_task* try_put_task(const T& t) override {\n return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) override {\n return try_put_task_impl(t, metainfo);\n }\n#endif\n};\n\n#endif // __TBB__flow_graph_cache_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_flow_graph_impl_H\n#define __TBB_flow_graph_impl_H\n\n// #include \"../config.h\"\n#include \"_task.h\"\n#include \"../task_group.h\"\n#include \"../task_arena.h\"\n#include \"../flow_graph_abstractions.h\"\n\n#include \"../concurrent_priority_queue.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d2 {\n\nclass graph_task;\nstatic graph_task* const SUCCESSFULLY_ENQUEUED = (graph_task*)-1;\ntypedef unsigned int node_priority_t;\nstatic const node_priority_t no_priority = node_priority_t(0);\n\nclass graph;\nclass graph_node;\n\ntemplate \nclass graph_iterator {\n friend class graph;\n friend class graph_node;\npublic:\n typedef size_t size_type;\n typedef GraphNodeType value_type;\n typedef GraphNodeType* pointer;\n typedef GraphNodeType& reference;\n typedef const GraphNodeType& const_reference;\n typedef std::forward_iterator_tag iterator_category;\n\n //! Copy constructor\n graph_iterator(const graph_iterator& other) :\n my_graph(other.my_graph), current_node(other.current_node)\n {}\n\n //! Assignment\n graph_iterator& operator=(const graph_iterator& other) {\n if (this != &other) {\n my_graph = other.my_graph;\n current_node = other.current_node;\n }\n return *this;\n }\n\n //! Dereference\n reference operator*() const;\n\n //! Dereference\n pointer operator->() const;\n\n //! Equality\n bool operator==(const graph_iterator& other) const {\n return ((my_graph == other.my_graph) && (current_node == other.current_node));\n }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\n //! Inequality\n bool operator!=(const graph_iterator& other) const { return !(operator==(other)); }\n#endif\n\n //! Pre-increment\n graph_iterator& operator++() {\n internal_forward();\n return *this;\n }\n\n //! Post-increment\n graph_iterator operator++(int) {\n graph_iterator result = *this;\n operator++();\n return result;\n }\n\nprivate:\n // the graph over which we are iterating\n GraphContainerType *my_graph;\n // pointer into my_graph's my_nodes list\n pointer current_node;\n\n //! Private initializing constructor for begin() and end() iterators\n graph_iterator(GraphContainerType *g, bool begin);\n void internal_forward();\n}; // class graph_iterator\n\n// flags to modify the behavior of the graph reset(). Can be combined.\nenum reset_flags {\n rf_reset_protocol = 0,\n rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body.\n rf_clear_edges = 1 << 1 // delete edges\n};\n\nvoid activate_graph(graph& g);\nvoid deactivate_graph(graph& g);\nbool is_graph_active(graph& g);\ngraph_task* prioritize_task(graph& g, graph_task& arena_task);\nvoid spawn_in_graph_arena(graph& g, graph_task& arena_task);\nvoid enqueue_in_graph_arena(graph &g, graph_task& arena_task);\n\nclass graph;\n\n//! Base class for tasks generated by graph nodes.\nclass graph_task : public d1::task {\npublic:\n graph_task(graph& g, d1::small_object_allocator& allocator,\n node_priority_t node_priority = no_priority);\n\n graph& my_graph; // graph instance the task belongs to\n // TODO revamp: rename to my_priority\n node_priority_t priority;\n template \n void destruct_and_deallocate(const d1::execution_data& ed);\nprotected:\n template \n void finalize(const d1::execution_data& ed);\nprivate:\n // To organize task_list\n graph_task* my_next{ nullptr };\n d1::small_object_allocator my_allocator;\n d1::wait_tree_vertex_interface* my_reference_vertex;\n // TODO revamp: elaborate internal interfaces to avoid friends declarations\n friend class graph_task_list;\n friend graph_task* prioritize_task(graph& g, graph_task& gt);\n};\n\ninline bool is_this_thread_in_graph_arena(graph& g);\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\nclass trackable_messages_graph_task : public graph_task {\npublic:\n trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator,\n node_priority_t node_priority,\n const std::forward_list& msg_waiters)\n : graph_task(g, allocator, node_priority)\n , my_msg_wait_context_vertices(msg_waiters)\n {\n auto last_iterator = my_msg_reference_vertices.cbefore_begin();\n\n for (auto& msg_waiter : my_msg_wait_context_vertices) {\n // If the task is created by the thread outside the graph arena, the lifetime of the thread reference vertex\n // may be shorter that the lifetime of the task, so thread reference vertex approach cannot be used\n // and the task should be associated with the msg wait context itself\n d1::wait_tree_vertex_interface* ref_vertex = is_this_thread_in_graph_arena(g) ?\n r1::get_thread_reference_vertex(msg_waiter) :\n msg_waiter;\n last_iterator = my_msg_reference_vertices.emplace_after(last_iterator,\n ref_vertex);\n ref_vertex->reserve(1);\n }\n }\n\n trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator,\n node_priority_t node_priority,\n std::forward_list&& msg_waiters)\n : graph_task(g, allocator, node_priority)\n , my_msg_wait_context_vertices(std::move(msg_waiters))\n {\n }\n\n const std::forward_list get_msg_wait_context_vertices() const {\n return my_msg_wait_context_vertices;\n }\n\nprotected:\n template \n void finalize(const d1::execution_data& ed) {\n auto wait_context_vertices = std::move(my_msg_wait_context_vertices);\n auto msg_reference_vertices = std::move(my_msg_reference_vertices);\n graph_task::finalize(ed);\n\n // If there is no thread reference vertices associated with the task\n // then this task was created by transferring the ownership from other metainfo\n // instance (e.g. while taking from the buffer)\n if (msg_reference_vertices.empty()) {\n for (auto& msg_waiter : wait_context_vertices) {\n msg_waiter->release(1);\n }\n } else {\n for (auto& msg_waiter : msg_reference_vertices) {\n msg_waiter->release(1);\n }\n }\n }\nprivate:\n // Each task that holds information about single message wait_contexts should hold two lists\n // The first one is wait_contexts associated with the message itself. They are needed\n // to be able to broadcast the list of wait_contexts to the node successors while executing the task.\n // The second list is a list of reference vertices for each wait_context_vertex in the first list\n // to support the distributed reference counting schema\n std::forward_list my_msg_wait_context_vertices;\n std::forward_list my_msg_reference_vertices;\n}; // class trackable_messages_graph_task\n#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n\nstruct graph_task_comparator {\n bool operator()(const graph_task* left, const graph_task* right) {\n return left->priority < right->priority;\n }\n};\n\ntypedef tbb::concurrent_priority_queue graph_task_priority_queue_t;\n\nclass priority_task_selector : public d1::task {\npublic:\n priority_task_selector(graph_task_priority_queue_t& priority_queue, d1::small_object_allocator& allocator)\n : my_priority_queue(priority_queue), my_allocator(allocator), my_task() {}\n task* execute(d1::execution_data& ed) override {\n next_task();\n __TBB_ASSERT(my_task, nullptr);\n task* t_next = my_task->execute(ed);\n my_allocator.delete_object(this, ed);\n return t_next;\n }\n task* cancel(d1::execution_data& ed) override {\n if (!my_task) {\n next_task();\n }\n __TBB_ASSERT(my_task, nullptr);\n task* t_next = my_task->cancel(ed);\n my_allocator.delete_object(this, ed);\n return t_next;\n }\nprivate:\n void next_task() {\n // TODO revamp: hold functors in priority queue instead of real tasks\n bool result = my_priority_queue.try_pop(my_task);\n __TBB_ASSERT_EX(result, \"Number of critical tasks for scheduler and tasks\"\n \" in graph's priority queue mismatched\");\n __TBB_ASSERT(my_task && my_task != SUCCESSFULLY_ENQUEUED,\n \"Incorrect task submitted to graph priority queue\");\n __TBB_ASSERT(my_task->priority != no_priority,\n \"Tasks from graph's priority queue must have priority\");\n }\n\n graph_task_priority_queue_t& my_priority_queue;\n d1::small_object_allocator my_allocator;\n graph_task* my_task;\n};\n\ntemplate class run_and_put_task;\ntemplate class run_task;\n\n//********************************************************************************\n// graph tasks helpers\n//********************************************************************************\n\n//! The list of graph tasks\nclass graph_task_list : no_copy {\nprivate:\n graph_task* my_first;\n graph_task** my_next_ptr;\npublic:\n //! Construct empty list\n graph_task_list() : my_first(nullptr), my_next_ptr(&my_first) {}\n\n //! True if list is empty; false otherwise.\n bool empty() const { return !my_first; }\n\n //! Push task onto back of list.\n void push_back(graph_task& task) {\n task.my_next = nullptr;\n *my_next_ptr = &task;\n my_next_ptr = &task.my_next;\n }\n\n //! Pop the front task from the list.\n graph_task& pop_front() {\n __TBB_ASSERT(!empty(), \"attempt to pop item from empty task_list\");\n graph_task* result = my_first;\n my_first = result->my_next;\n if (!my_first) {\n my_next_ptr = &my_first;\n }\n return *result;\n }\n};\n\n//! The graph class\n/** This class serves as a handle to the graph */\nclass graph : no_copy, public graph_proxy {\n friend class graph_node;\n\n void prepare_task_arena(bool reinit = false) {\n if (reinit) {\n __TBB_ASSERT(my_task_arena, \"task arena is nullptr\");\n my_task_arena->terminate();\n my_task_arena->initialize(task_arena::attach());\n }\n else {\n __TBB_ASSERT(my_task_arena == nullptr, \"task arena is not nullptr\");\n my_task_arena = new task_arena(task_arena::attach());\n }\n if (!my_task_arena->is_active()) // failed to attach\n my_task_arena->initialize(); // create a new, default-initialized arena\n __TBB_ASSERT(my_task_arena->is_active(), \"task arena is not active\");\n }\n\npublic:\n //! Constructs a graph with isolated task_group_context\n graph();\n\n //! Constructs a graph with use_this_context as context\n explicit graph(task_group_context& use_this_context);\n\n //! Destroys the graph.\n /** Calls wait_for_all, then destroys the root task and context. */\n ~graph();\n\n //! Used to register that an external entity may still interact with the graph.\n /** The graph will not return from wait_for_all until a matching number of release_wait calls is\n made. */\n void reserve_wait() override;\n\n //! Deregisters an external entity that may have interacted with the graph.\n /** The graph will not return from wait_for_all until all the number of reserve_wait calls\n matches the number of release_wait calls. */\n void release_wait() override;\n\n //! Wait until graph is idle and the number of release_wait calls equals to the number of\n //! reserve_wait calls.\n /** The waiting thread will go off and steal work while it is blocked in the wait_for_all. */\n void wait_for_all() {\n cancelled = false;\n caught_exception = false;\n try_call([this] {\n my_task_arena->execute([this] {\n wait(my_wait_context_vertex.get_context(), *my_context);\n });\n cancelled = my_context->is_group_execution_cancelled();\n }).on_exception([this] {\n my_context->reset();\n caught_exception = true;\n cancelled = true;\n });\n // TODO: the \"if\" condition below is just a work-around to support the concurrent wait\n // mode. The cancellation and exception mechanisms are still broken in this mode.\n // Consider using task group not to re-implement the same functionality.\n if (!(my_context->traits() & task_group_context::concurrent_wait)) {\n my_context->reset(); // consistent with behavior in catch()\n }\n }\n\n // TODO revamp: consider adding getter for task_group_context.\n\n // ITERATORS\n template\n friend class graph_iterator;\n\n // Graph iterator typedefs\n typedef graph_iterator iterator;\n typedef graph_iterator const_iterator;\n\n // Graph iterator constructors\n //! start iterator\n iterator begin();\n //! end iterator\n iterator end();\n //! start const iterator\n const_iterator begin() const;\n //! end const iterator\n const_iterator end() const;\n //! start const iterator\n const_iterator cbegin() const;\n //! end const iterator\n const_iterator cend() const;\n\n // thread-unsafe state reset.\n void reset(reset_flags f = rf_reset_protocol);\n\n //! cancels execution of the associated task_group_context\n void cancel();\n\n //! return status of graph execution\n bool is_cancelled() { return cancelled; }\n bool exception_thrown() { return caught_exception; }\n\nprivate:\n d1::wait_context_vertex my_wait_context_vertex;\n task_group_context *my_context;\n bool own_context;\n bool cancelled;\n bool caught_exception;\n bool my_is_active;\n\n graph_node *my_nodes, *my_nodes_last;\n\n tbb::spin_mutex nodelist_mutex;\n void register_node(graph_node *n);\n void remove_node(graph_node *n);\n\n task_arena* my_task_arena;\n\n graph_task_priority_queue_t my_priority_queue;\n\n d1::wait_context_vertex& get_wait_context_vertex() { return my_wait_context_vertex; }\n\n friend void activate_graph(graph& g);\n friend void deactivate_graph(graph& g);\n friend bool is_graph_active(graph& g);\n friend bool is_this_thread_in_graph_arena(graph& g);\n friend graph_task* prioritize_task(graph& g, graph_task& arena_task);\n friend void spawn_in_graph_arena(graph& g, graph_task& arena_task);\n friend void enqueue_in_graph_arena(graph &g, graph_task& arena_task);\n\n friend class d1::task_arena_base;\n friend class graph_task;\n\n template \n friend class receiver;\n}; // class graph\n\ntemplate\ninline void graph_task::destruct_and_deallocate(const d1::execution_data& ed) {\n auto allocator = my_allocator;\n // TODO: investigate if direct call of derived destructor gives any benefits.\n this->~graph_task();\n allocator.deallocate(static_cast(this), ed);\n}\n\ntemplate\ninline void graph_task::finalize(const d1::execution_data& ed) {\n d1::wait_tree_vertex_interface* reference_vertex = my_reference_vertex;\n destruct_and_deallocate(ed);\n reference_vertex->release();\n}\n\ninline graph_task::graph_task(graph& g, d1::small_object_allocator& allocator,\n node_priority_t node_priority)\n : my_graph(g)\n , priority(node_priority)\n , my_allocator(allocator)\n{\n // If the task is created by the thread outside the graph arena, the lifetime of the thread reference vertex\n // may be shorter that the lifetime of the task, so thread reference vertex approach cannot be used\n // and the task should be associated with the graph wait context itself\n // TODO: consider how reference counting can be improved for such a use case. Most common example is the async_node\n d1::wait_context_vertex* graph_wait_context_vertex = &my_graph.get_wait_context_vertex();\n my_reference_vertex = is_this_thread_in_graph_arena(g) ? r1::get_thread_reference_vertex(graph_wait_context_vertex)\n : graph_wait_context_vertex;\n __TBB_ASSERT(my_reference_vertex, nullptr);\n my_reference_vertex->reserve();\n}\n\n//********************************************************************************\n// end of graph tasks helpers\n//********************************************************************************\n\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\nclass get_graph_helper;\n#endif\n\n//! The base of all graph nodes.\nclass graph_node : no_copy {\n friend class graph;\n template\n friend class graph_iterator;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n friend class get_graph_helper;\n#endif\n\nprotected:\n graph& my_graph;\n graph& graph_reference() const {\n // TODO revamp: propagate graph_reference() method to all the reference places.\n return my_graph;\n }\n graph_node* next = nullptr;\n graph_node* prev = nullptr;\npublic:\n explicit graph_node(graph& g);\n\n virtual ~graph_node();\n\nprotected:\n // performs the reset on an individual node.\n virtual void reset_node(reset_flags f = rf_reset_protocol) = 0;\n}; // class graph_node\n\ninline void activate_graph(graph& g) {\n g.my_is_active = true;\n}\n\ninline void deactivate_graph(graph& g) {\n g.my_is_active = false;\n}\n\ninline bool is_graph_active(graph& g) {\n return g.my_is_active;\n}\n\ninline bool is_this_thread_in_graph_arena(graph& g) {\n __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), nullptr);\n return r1::execution_slot(*g.my_task_arena) != d1::slot_id(-1);\n}\n\ninline graph_task* prioritize_task(graph& g, graph_task& gt) {\n if( no_priority == gt.priority )\n return >\n\n //! Non-preemptive priority pattern. The original task is submitted as a work item to the\n //! priority queue, and a new critical task is created to take and execute a work item with\n //! the highest known priority. The reference counting responsibility is transferred to\n //! the new task.\n d1::task* critical_task = gt.my_allocator.new_object(g.my_priority_queue, gt.my_allocator);\n __TBB_ASSERT( critical_task, \"bad_alloc?\" );\n g.my_priority_queue.push(>);\n using tbb::detail::d1::submit;\n submit( *critical_task, *g.my_task_arena, *g.my_context, /*as_critical=*/true );\n return nullptr;\n}\n\n//! Spawns a task inside graph arena\ninline void spawn_in_graph_arena(graph& g, graph_task& arena_task) {\n if (is_graph_active(g)) {\n d1::task* gt = prioritize_task(g, arena_task);\n if( !gt )\n return;\n\n __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), nullptr);\n submit( *gt, *g.my_task_arena, *g.my_context\n#if __TBB_PREVIEW_CRITICAL_TASKS\n , /*as_critical=*/false\n#endif\n );\n }\n}\n\n// TODO revamp: unify *_in_graph_arena functions\n\n//! Enqueues a task inside graph arena\ninline void enqueue_in_graph_arena(graph &g, graph_task& arena_task) {\n if (is_graph_active(g)) {\n __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), \"Is graph's arena initialized and active?\" );\n\n // TODO revamp: decide on the approach that does not postpone critical task\n if( d1::task* gt = prioritize_task(g, arena_task) )\n submit( *gt, *g.my_task_arena, *g.my_context, /*as_critical=*/false);\n }\n}\n\n} // namespace d2\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_flow_graph_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_indexer_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_indexer_impl_H\n#define __TBB__flow_graph_indexer_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included in namespace tbb::detail::d2\n\n#include \"_flow_graph_types_impl.h\"\n\n // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of\n // the form tagged_msg\n // where the value of tag will indicate which result was put to the\n // successor.\n\n template\n graph_task* do_try_put(const T &v, void *p __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n typename IndexerNodeBaseType::output_type o(K, v);\n return reinterpret_cast(p)->try_put_task(&o __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n\n template\n struct indexer_helper {\n template\n static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) {\n typedef typename std::tuple_element::type T;\n auto indexer_node_put_task = do_try_put;\n std::get(my_input).set_up(p, indexer_node_put_task, g);\n indexer_helper::template set_indexer_node_pointer(my_input, p, g);\n }\n };\n\n template\n struct indexer_helper {\n template\n static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) {\n typedef typename std::tuple_element<0, TupleTypes>::type T;\n auto indexer_node_put_task = do_try_put;\n std::get<0>(my_input).set_up(p, indexer_node_put_task, g);\n }\n };\n\n template\n class indexer_input_port : public receiver {\n private:\n void* my_indexer_ptr;\n typedef graph_task* (* forward_function_ptr)(T const &, void*\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&));\n forward_function_ptr my_try_put_task;\n graph* my_graph;\n public:\n void set_up(void* p, forward_function_ptr f, graph& g) {\n my_indexer_ptr = p;\n my_try_put_task = f;\n my_graph = &g;\n }\n\n protected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n graph_task* try_put_task(const T &v) override {\n return my_try_put_task(v, my_indexer_ptr __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override {\n return my_try_put_task(v, my_indexer_ptr, metainfo);\n }\n#endif\n\n graph& graph_reference() const override {\n return *my_graph;\n }\n };\n\n template\n class indexer_node_FE {\n public:\n static const int N = std::tuple_size::value;\n typedef OutputType output_type;\n typedef InputTuple input_type;\n\n // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member.\n indexer_node_FE() : my_inputs() {}\n\n input_type &input_ports() { return my_inputs; }\n protected:\n input_type my_inputs;\n };\n\n //! indexer_node_base\n template\n class indexer_node_base : public graph_node, public indexer_node_FE,\n public sender {\n protected:\n using graph_node::my_graph;\n public:\n static const size_t N = std::tuple_size::value;\n typedef OutputType output_type;\n typedef StructTypes tuple_types;\n typedef typename sender::successor_type successor_type;\n typedef indexer_node_FE input_ports_type;\n\n private:\n // ----------- Aggregator ------------\n enum op_type { reg_succ, rem_succ, try__put_task\n };\n typedef indexer_node_base class_type;\n\n class indexer_node_base_operation : public d1::aggregated_operation {\n public:\n char type;\n union {\n output_type const *my_arg;\n successor_type *my_succ;\n graph_task* bypass_t;\n };\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo const* metainfo;\n#endif\n indexer_node_base_operation(const output_type* e, op_type t) :\n type(char(t)), my_arg(e) __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr))\n {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n indexer_node_base_operation(const output_type* e, op_type t, const message_metainfo& info)\n : type(char(t)), my_arg(e), metainfo(&info) {}\n#endif\n indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)),\n my_succ(const_cast(&s)) {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator my_aggregator;\n\n void handle_operations(indexer_node_base_operation* op_list) {\n indexer_node_base_operation *current;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n\n case reg_succ:\n my_successors.register_successor(*(current->my_succ));\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n\n case rem_succ:\n my_successors.remove_successor(*(current->my_succ));\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n case try__put_task: {\n current->bypass_t = my_successors.try_put_task(*(current->my_arg)\n __TBB_FLOW_GRAPH_METAINFO_ARG(*(current->metainfo)));\n current->status.store( SUCCEEDED, std::memory_order_release); // return of try_put_task actual return value\n }\n break;\n }\n }\n }\n // ---------- end aggregator -----------\n public:\n indexer_node_base(graph& g) : graph_node(g), input_ports_type(), my_successors(this) {\n indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g);\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n indexer_node_base(const indexer_node_base& other)\n : graph_node(other.my_graph), input_ports_type(), sender(), my_successors(this)\n {\n indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph);\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n bool register_successor(successor_type &r) override {\n indexer_node_base_operation op_data(r, reg_succ);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n bool remove_successor( successor_type &r) override {\n indexer_node_base_operation op_data(r, rem_succ);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n // not a virtual method in this class\n graph_task* try_put_task(output_type const *v\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n indexer_node_base_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n my_aggregator.execute(&op_data);\n return op_data.bypass_t;\n }\n\n protected:\n void reset_node(reset_flags f) override {\n if(f & rf_clear_edges) {\n my_successors.clear();\n }\n }\n\n private:\n broadcast_cache my_successors;\n }; //indexer_node_base\n\n\n template struct input_types;\n\n template\n struct input_types<1, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<2, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<3, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<4, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<5, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<6, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef typename std::tuple_element<5, InputTuple>::type sixth_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<7, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef typename std::tuple_element<5, InputTuple>::type sixth_type;\n typedef typename std::tuple_element<6, InputTuple>::type seventh_type;\n typedef tagged_msg type;\n };\n\n\n template\n struct input_types<8, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef typename std::tuple_element<5, InputTuple>::type sixth_type;\n typedef typename std::tuple_element<6, InputTuple>::type seventh_type;\n typedef typename std::tuple_element<7, InputTuple>::type eighth_type;\n typedef tagged_msg type;\n };\n\n\n template\n struct input_types<9, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef typename std::tuple_element<5, InputTuple>::type sixth_type;\n typedef typename std::tuple_element<6, InputTuple>::type seventh_type;\n typedef typename std::tuple_element<7, InputTuple>::type eighth_type;\n typedef typename std::tuple_element<8, InputTuple>::type nineth_type;\n typedef tagged_msg type;\n };\n\n template\n struct input_types<10, InputTuple> {\n typedef typename std::tuple_element<0, InputTuple>::type first_type;\n typedef typename std::tuple_element<1, InputTuple>::type second_type;\n typedef typename std::tuple_element<2, InputTuple>::type third_type;\n typedef typename std::tuple_element<3, InputTuple>::type fourth_type;\n typedef typename std::tuple_element<4, InputTuple>::type fifth_type;\n typedef typename std::tuple_element<5, InputTuple>::type sixth_type;\n typedef typename std::tuple_element<6, InputTuple>::type seventh_type;\n typedef typename std::tuple_element<7, InputTuple>::type eighth_type;\n typedef typename std::tuple_element<8, InputTuple>::type nineth_type;\n typedef typename std::tuple_element<9, InputTuple>::type tenth_type;\n typedef tagged_msg type;\n };\n\n // type generators\n template\n struct indexer_types : public input_types::value, OutputTuple> {\n static const int N = std::tuple_size::value;\n typedef typename input_types::type output_type;\n typedef typename wrap_tuple_elements::type input_ports_type;\n typedef indexer_node_FE indexer_FE_type;\n typedef indexer_node_base indexer_base_type;\n };\n\n template\n class unfolded_indexer_node : public indexer_types::indexer_base_type {\n public:\n typedef typename indexer_types::input_ports_type input_ports_type;\n typedef OutputTuple tuple_types;\n typedef typename indexer_types::output_type output_type;\n private:\n typedef typename indexer_types::indexer_base_type base_type;\n public:\n unfolded_indexer_node(graph& g) : base_type(g) {}\n unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {}\n };\n\n#endif /* __TBB__flow_graph_indexer_impl_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_item_buffer_impl_H\n#define __TBB__flow_graph_item_buffer_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n#include \"_aligned_space.h\"\n\n// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h)\n\n//! Expandable buffer of items. The possible operations are push, pop,\n//* tests for empty and so forth. No mutual exclusion is built in.\n//* objects are constructed into and explicitly-destroyed. get_my_item gives\n// a read-only reference to the item in the buffer. set_my_item may be called\n// with either an empty or occupied slot.\n\ntemplate >\nclass item_buffer {\npublic:\n typedef T item_type;\n enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 };\nprotected:\n struct aligned_space_item {\n item_type item;\n buffer_item_state state;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n };\n typedef size_t size_type;\n typedef aligned_space buffer_item_type;\n typedef typename allocator_traits::template rebind_alloc allocator_type;\n buffer_item_type *my_array;\n size_type my_array_size;\n static const size_type initial_buffer_size = 4;\n size_type my_head;\n size_type my_tail;\n\n bool buffer_empty() const { return my_head == my_tail; }\n\n aligned_space_item &element(size_type i) {\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr);\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr);\n#endif\n return *my_array[i & (my_array_size - 1) ].begin();\n }\n\n const aligned_space_item &element(size_type i) const {\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr);\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr);\n#endif\n return *my_array[i & (my_array_size-1)].begin();\n }\n\n bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (element(i).state != no_item); }\n#if TBB_USE_ASSERT\n bool my_item_reserved(size_type i) const { return element(i).state == reserved_item; }\n#endif\n\n // object management in buffer\n const item_type &get_my_item(size_t i) const {\n __TBB_ASSERT(my_item_valid(i),\"attempt to get invalid item\");\n return element(i).item;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo& get_my_metainfo(size_t i) {\n __TBB_ASSERT(my_item_valid(i), \"attempt to get invalid item\");\n return element(i).metainfo;\n }\n#endif\n\n // may be called with an empty slot or a slot that has already been constructed into.\n void set_my_item(size_t i, const item_type &o\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n if(element(i).state != no_item) {\n destroy_item(i);\n }\n new(&(element(i).item)) item_type(o);\n element(i).state = has_item;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n new(&element(i).metainfo) message_metainfo(metainfo);\n\n for (auto& waiter : metainfo.waiters()) {\n waiter->reserve(1);\n }\n#endif\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n void set_my_item(size_t i, const item_type& o, message_metainfo&& metainfo) {\n if(element(i).state != no_item) {\n destroy_item(i);\n }\n\n new(&(element(i).item)) item_type(o);\n new(&element(i).metainfo) message_metainfo(std::move(metainfo));\n // Skipping the reservation on metainfo.waiters since the ownership\n // is moving from metainfo to the cache\n element(i).state = has_item;\n }\n#endif\n\n // destructively-fetch an object from the buffer\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n void fetch_item(size_t i, item_type& o, message_metainfo& metainfo) {\n __TBB_ASSERT(my_item_valid(i), \"Trying to fetch an empty slot\");\n o = get_my_item(i); // could have std::move assign semantics\n metainfo = std::move(get_my_metainfo(i));\n destroy_item(i);\n }\n#else\n void fetch_item(size_t i, item_type &o) {\n __TBB_ASSERT(my_item_valid(i), \"Trying to fetch an empty slot\");\n o = get_my_item(i); // could have std::move assign semantics\n destroy_item(i);\n }\n#endif\n\n // move an existing item from one slot to another. The moved-to slot must be unoccupied,\n // the moved-from slot must exist and not be reserved. The after, from will be empty,\n // to will be occupied but not reserved\n void move_item(size_t to, size_t from) {\n __TBB_ASSERT(!my_item_valid(to), \"Trying to move to a non-empty slot\");\n __TBB_ASSERT(my_item_valid(from), \"Trying to move from an empty slot\");\n // could have std::move semantics\n set_my_item(to, get_my_item(from) __TBB_FLOW_GRAPH_METAINFO_ARG(get_my_metainfo(from)));\n destroy_item(from);\n }\n\n // put an item in an empty slot. Return true if successful, else false\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n bool place_item(size_t here, const item_type &me, Metainfo&& metainfo) {\n#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES\n if(my_item_valid(here)) return false;\n#endif\n set_my_item(here, me, std::forward(metainfo));\n return true;\n }\n#else\n bool place_item(size_t here, const item_type &me) {\n#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES\n if(my_item_valid(here)) return false;\n#endif\n set_my_item(here, me);\n return true;\n }\n#endif\n\n // could be implemented with std::move semantics\n void swap_items(size_t i, size_t j) {\n __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), \"attempt to swap invalid item(s)\");\n item_type temp = get_my_item(i);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo temp_metainfo = get_my_metainfo(i);\n set_my_item(i, get_my_item(j), get_my_metainfo(j));\n set_my_item(j, temp, temp_metainfo);\n#else\n set_my_item(i, get_my_item(j));\n set_my_item(j, temp);\n#endif\n }\n\n void destroy_item(size_type i) {\n __TBB_ASSERT(my_item_valid(i), \"destruction of invalid item\");\n\n auto& e = element(i);\n e.item.~item_type();\n e.state = no_item;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (auto& msg_waiter : e.metainfo.waiters()) {\n msg_waiter->release(1);\n }\n\n e.metainfo.~message_metainfo();\n#endif\n }\n\n // returns the front element\n const item_type& front() const\n {\n __TBB_ASSERT(my_item_valid(my_head), \"attempt to fetch head non-item\");\n return get_my_item(my_head);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n const message_metainfo& front_metainfo() const\n {\n __TBB_ASSERT(my_item_valid(my_head), \"attempt to fetch head non-item\");\n return element(my_head).metainfo;\n }\n#endif\n\n // returns the back element\n const item_type& back() const\n {\n __TBB_ASSERT(my_item_valid(my_tail - 1), \"attempt to fetch head non-item\");\n return get_my_item(my_tail - 1);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n const message_metainfo& back_metainfo() const {\n __TBB_ASSERT(my_item_valid(my_tail - 1), \"attempt to fetch head non-item\");\n return element(my_tail - 1).metainfo;\n }\n#endif\n\n // following methods are for reservation of the front of a buffer.\n void reserve_item(size_type i) {\n __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), \"item cannot be reserved\");\n element(i).state = reserved_item;\n }\n\n void release_item(size_type i) {\n __TBB_ASSERT(my_item_reserved(i), \"item is not reserved\");\n element(i).state = has_item;\n }\n\n void destroy_front() { destroy_item(my_head); ++my_head; }\n void destroy_back() { destroy_item(my_tail-1); --my_tail; }\n\n // we have to be able to test against a new tail value without changing my_tail\n // grow_array doesn't work if we change my_tail when the old array is too small\n size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; }\n size_type capacity() { return my_array_size; }\n // sequencer_node does not use this method, so we don't\n // need a version that passes in the new_tail value.\n bool buffer_full() { return size() >= capacity(); }\n\n //! Grows the internal array.\n void grow_my_array( size_t minimum_size ) {\n // test that we haven't made the structure inconsistent.\n __TBB_ASSERT(capacity() >= my_tail - my_head, \"total items exceed capacity\");\n size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size;\n while( new_sizestate = no_item; }\n\n for( size_type i=my_head; iitem);\n (void)new(new_space) item_type(get_my_item(i));\n new_array[i&(new_size-1)].begin()->state = element(i).state;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n char* meta_space = (char *)&(new_array[i&(new_size-1)].begin()->metainfo);\n ::new(meta_space) message_metainfo(std::move(element(i).metainfo));\n#endif\n }\n }\n\n clean_up_buffer(/*reset_pointers*/false);\n\n my_array = new_array;\n my_array_size = new_size;\n }\n\n bool push_back(item_type& v\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n if (buffer_full()) {\n grow_my_array(size() + 1);\n }\n set_my_item(my_tail, v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n ++my_tail;\n return true;\n }\n\n bool pop_back(item_type& v\n __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo))\n {\n if (!my_item_valid(my_tail - 1)) {\n return false;\n }\n auto& e = element(my_tail - 1);\n v = e.item;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n metainfo = std::move(e.metainfo);\n#endif\n\n destroy_back();\n return true;\n }\n\n bool pop_front(item_type& v\n __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo))\n {\n if (!my_item_valid(my_head)) {\n return false;\n }\n auto& e = element(my_head);\n v = e.item;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n metainfo = std::move(e.metainfo);\n#endif\n\n destroy_front();\n return true;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool pop_back(item_type& v) {\n message_metainfo metainfo;\n return pop_back(v, metainfo);\n }\n\n bool pop_front(item_type& v) {\n message_metainfo metainfo;\n return pop_front(v, metainfo);\n }\n#endif\n\n // This is used both for reset and for grow_my_array. In the case of grow_my_array\n // we want to retain the values of the head and tail.\n void clean_up_buffer(bool reset_pointers) {\n if (my_array) {\n for( size_type i=my_head; i >\nclass reservable_item_buffer : public item_buffer {\nprotected:\n using item_buffer::my_item_valid;\n using item_buffer::my_head;\n\npublic:\n reservable_item_buffer() : item_buffer(), my_reserved(false) {}\n void reset() {my_reserved = false; item_buffer::reset(); }\nprotected:\n\n bool reserve_front(T &v) {\n if(my_reserved || !my_item_valid(this->my_head)) return false;\n my_reserved = true;\n // reserving the head\n v = this->front();\n this->reserve_item(this->my_head);\n return true;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool reserve_front(T& v, message_metainfo& metainfo) {\n if (my_reserved || !my_item_valid(this->my_head)) return false;\n my_reserved = true;\n // reserving the head\n v = this->front();\n metainfo = this->front_metainfo();\n this->reserve_item(this->my_head);\n return true;\n }\n#endif\n\n void consume_front() {\n __TBB_ASSERT(my_reserved, \"Attempt to consume a non-reserved item\");\n this->destroy_front();\n my_reserved = false;\n }\n\n void release_front() {\n __TBB_ASSERT(my_reserved, \"Attempt to release a non-reserved item\");\n this->release_item(this->my_head);\n my_reserved = false;\n }\n\n bool my_reserved;\n};\n\n#endif // __TBB__flow_graph_item_buffer_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_join_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_join_impl_H\n#define __TBB__flow_graph_join_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included into namespace tbb::detail::d2\n\n struct forwarding_base : no_assign {\n forwarding_base(graph &g) : graph_ref(g) {}\n virtual ~forwarding_base() {}\n graph& graph_ref;\n };\n\n struct queueing_forwarding_base : forwarding_base {\n using forwarding_base::forwarding_base;\n // decrement_port_count may create a forwarding task. If we cannot handle the task\n // ourselves, ask decrement_port_count to deal with it.\n virtual graph_task* decrement_port_count(bool handle_task) = 0;\n };\n\n struct reserving_forwarding_base : forwarding_base {\n using forwarding_base::forwarding_base;\n // decrement_port_count may create a forwarding task. If we cannot handle the task\n // ourselves, ask decrement_port_count to deal with it.\n virtual graph_task* decrement_port_count() = 0;\n virtual void increment_port_count() = 0;\n };\n\n // specialization that lets us keep a copy of the current_key for building results.\n // KeyType can be a reference type.\n template\n struct matching_forwarding_base : public forwarding_base {\n typedef typename std::decay::type current_key_type;\n matching_forwarding_base(graph &g) : forwarding_base(g) { }\n virtual graph_task* increment_key_count(current_key_type const & /*t*/) = 0;\n current_key_type current_key; // so ports can refer to FE's desired items\n };\n\n template< int N >\n struct join_helper {\n\n template< typename TupleType, typename PortType >\n static inline void set_join_node_pointer(TupleType &my_input, PortType *port) {\n std::get( my_input ).set_join_node_pointer(port);\n join_helper::set_join_node_pointer( my_input, port );\n }\n template< typename TupleType >\n static inline void consume_reservations( TupleType &my_input ) {\n std::get( my_input ).consume();\n join_helper::consume_reservations( my_input );\n }\n\n template< typename TupleType >\n static inline void release_my_reservation( TupleType &my_input ) {\n std::get( my_input ).release();\n }\n\n template \n static inline void release_reservations( TupleType &my_input) {\n join_helper::release_reservations(my_input);\n release_my_reservation(my_input);\n }\n\n template< typename InputTuple, typename OutputTuple >\n static inline bool reserve( InputTuple &my_input, OutputTuple &out) {\n if ( !std::get( my_input ).reserve( std::get( out ) ) ) return false;\n if ( !join_helper::reserve( my_input, out ) ) {\n release_my_reservation( my_input );\n return false;\n }\n return true;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n message_metainfo element_metainfo;\n if (!std::get(my_input).reserve(std::get(out), element_metainfo)) return false;\n if (!join_helper::reserve(my_input, out, metainfo)) {\n release_my_reservation(my_input);\n return false;\n }\n metainfo.merge(element_metainfo);\n return true;\n\n }\n#endif\n\n template\n static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) {\n bool res = std::get(my_input).get_item(std::get(out) ); // may fail\n return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n message_metainfo element_metainfo;\n bool res = std::get(my_input).get_item(std::get(out), element_metainfo);\n metainfo.merge(element_metainfo);\n return join_helper::get_my_item(my_input, out, metainfo) && res;\n }\n#endif\n\n template\n static inline bool get_items(InputTuple &my_input, OutputTuple &out) {\n return get_my_item(my_input, out);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n return get_my_item(my_input, out, metainfo);\n }\n#endif\n\n template\n static inline void reset_my_port(InputTuple &my_input) {\n join_helper::reset_my_port(my_input);\n std::get(my_input).reset_port();\n }\n\n template\n static inline void reset_ports(InputTuple& my_input) {\n reset_my_port(my_input);\n }\n\n template\n static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) {\n std::get(my_input).set_my_key_func(std::get(my_key_funcs));\n std::get(my_key_funcs) = nullptr;\n join_helper::set_key_functors(my_input, my_key_funcs);\n }\n\n template< typename KeyFuncTuple>\n static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) {\n __TBB_ASSERT(\n std::get(other_inputs).get_my_key_func(),\n \"key matching join node should not be instantiated without functors.\"\n );\n std::get(my_inputs).set_my_key_func(std::get(other_inputs).get_my_key_func()->clone());\n join_helper::copy_key_functors(my_inputs, other_inputs);\n }\n\n template\n static inline void reset_inputs(InputTuple &my_input, reset_flags f) {\n join_helper::reset_inputs(my_input, f);\n std::get(my_input).reset_receiver(f);\n }\n }; // join_helper\n\n template< >\n struct join_helper<1> {\n\n template< typename TupleType, typename PortType >\n static inline void set_join_node_pointer(TupleType &my_input, PortType *port) {\n std::get<0>( my_input ).set_join_node_pointer(port);\n }\n\n template< typename TupleType >\n static inline void consume_reservations( TupleType &my_input ) {\n std::get<0>( my_input ).consume();\n }\n\n template< typename TupleType >\n static inline void release_my_reservation( TupleType &my_input ) {\n std::get<0>( my_input ).release();\n }\n\n template\n static inline void release_reservations( TupleType &my_input) {\n release_my_reservation(my_input);\n }\n\n template< typename InputTuple, typename OutputTuple >\n static inline bool reserve( InputTuple &my_input, OutputTuple &out) {\n return std::get<0>( my_input ).reserve( std::get<0>( out ) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n message_metainfo element_metainfo;\n bool result = std::get<0>(my_input).reserve(std::get<0>(out), element_metainfo);\n metainfo.merge(element_metainfo);\n return result;\n }\n#endif\n\n template\n static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) {\n return std::get<0>(my_input).get_item(std::get<0>(out));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n message_metainfo element_metainfo;\n bool res = std::get<0>(my_input).get_item(std::get<0>(out), element_metainfo);\n metainfo.merge(element_metainfo);\n return res;\n }\n#endif\n\n template\n static inline bool get_items(InputTuple &my_input, OutputTuple &out) {\n return get_my_item(my_input, out);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) {\n return get_my_item(my_input, out, metainfo);\n }\n#endif\n\n template\n static inline void reset_my_port(InputTuple &my_input) {\n std::get<0>(my_input).reset_port();\n }\n\n template\n static inline void reset_ports(InputTuple& my_input) {\n reset_my_port(my_input);\n }\n\n template\n static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) {\n std::get<0>(my_input).set_my_key_func(std::get<0>(my_key_funcs));\n std::get<0>(my_key_funcs) = nullptr;\n }\n\n template< typename KeyFuncTuple>\n static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) {\n __TBB_ASSERT(\n std::get<0>(other_inputs).get_my_key_func(),\n \"key matching join node should not be instantiated without functors.\"\n );\n std::get<0>(my_inputs).set_my_key_func(std::get<0>(other_inputs).get_my_key_func()->clone());\n }\n template\n static inline void reset_inputs(InputTuple &my_input, reset_flags f) {\n std::get<0>(my_input).reset_receiver(f);\n }\n }; // join_helper<1>\n\n //! The two-phase join port\n template< typename T >\n class reserving_port : public receiver {\n public:\n typedef T input_type;\n typedef typename receiver::predecessor_type predecessor_type;\n\n private:\n // ----------- Aggregator ------------\n enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res\n };\n typedef reserving_port class_type;\n\n class reserving_port_operation : public d1::aggregated_operation {\n public:\n char type;\n union {\n T *my_arg;\n predecessor_type *my_pred;\n };\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo;\n#endif\n reserving_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) :\n type(char(t)), my_arg(const_cast(&e))\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n reserving_port_operation(const T& e, op_type t)\n : type(char(t)), my_arg(const_cast(&e)), metainfo(nullptr) {}\n#endif\n reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)),\n my_pred(const_cast(&s)) {}\n reserving_port_operation(op_type t) : type(char(t)) {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator my_aggregator;\n\n void handle_operations(reserving_port_operation* op_list) {\n reserving_port_operation *current;\n bool was_missing_predecessors = false;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n case reg_pred:\n was_missing_predecessors = my_predecessors.empty();\n my_predecessors.add(*(current->my_pred));\n if ( was_missing_predecessors ) {\n (void) my_join->decrement_port_count(); // may try to forward\n }\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n case rem_pred:\n if ( !my_predecessors.empty() ) {\n my_predecessors.remove(*(current->my_pred));\n if ( my_predecessors.empty() ) // was the last predecessor\n my_join->increment_port_count();\n }\n // TODO: consider returning failure if there were no predecessors to remove\n current->status.store( SUCCEEDED, std::memory_order_release );\n break;\n case res_item:\n if ( reserved ) {\n current->status.store( FAILED, std::memory_order_release);\n }\n else {\n bool reserve_result = false;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (current->metainfo) {\n reserve_result = my_predecessors.try_reserve(*(current->my_arg),\n *(current->metainfo));\n } else\n#endif\n {\n reserve_result = my_predecessors.try_reserve(*(current->my_arg));\n }\n if (reserve_result) {\n reserved = true;\n current->status.store( SUCCEEDED, std::memory_order_release);\n } else {\n if ( my_predecessors.empty() ) {\n my_join->increment_port_count();\n }\n current->status.store( FAILED, std::memory_order_release);\n }\n }\n break;\n case rel_res:\n reserved = false;\n my_predecessors.try_release( );\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n case con_res:\n reserved = false;\n my_predecessors.try_consume( );\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n }\n }\n }\n\n protected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n graph_task* try_put_task( const T & ) override {\n return nullptr;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T&, const message_metainfo&) override { return nullptr; }\n#endif\n\n graph& graph_reference() const override {\n return my_join->graph_ref;\n }\n\n public:\n\n //! Constructor\n reserving_port() : my_join(nullptr), my_predecessors(this), reserved(false) {\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n // copy constructor\n reserving_port(const reserving_port& /* other */) = delete;\n\n void set_join_node_pointer(reserving_forwarding_base *join) {\n my_join = join;\n }\n\n //! Add a predecessor\n bool register_predecessor( predecessor_type &src ) override {\n reserving_port_operation op_data(src, reg_pred);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n //! Remove a predecessor\n bool remove_predecessor( predecessor_type &src ) override {\n reserving_port_operation op_data(src, rem_pred);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n //! Reserve an item from the port\n bool reserve( T &v ) {\n reserving_port_operation op_data(v, res_item);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool reserve( T& v, message_metainfo& metainfo ) {\n reserving_port_operation op_data(v, res_item, metainfo);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n#endif\n\n //! Release the port\n void release( ) {\n reserving_port_operation op_data(rel_res);\n my_aggregator.execute(&op_data);\n }\n\n //! Complete use of the port\n void consume( ) {\n reserving_port_operation op_data(con_res);\n my_aggregator.execute(&op_data);\n }\n\n void reset_receiver( reset_flags f) {\n if(f & rf_clear_edges) my_predecessors.clear();\n else\n my_predecessors.reset();\n reserved = false;\n __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), \"port edges not removed\");\n }\n\n private:\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n friend class get_graph_helper;\n#endif\n\n reserving_forwarding_base *my_join;\n reservable_predecessor_cache< T, null_mutex > my_predecessors;\n bool reserved;\n }; // reserving_port\n\n //! queueing join_port\n template\n class queueing_port : public receiver, public item_buffer {\n public:\n typedef T input_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef queueing_port class_type;\n\n // ----------- Aggregator ------------\n private:\n enum op_type { get__item, res_port, try__put_task\n };\n\n class queueing_port_operation : public d1::aggregated_operation {\n public:\n char type;\n T my_val;\n T* my_arg;\n graph_task* bypass_t;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo;\n#endif\n // constructor for value parameter\n queueing_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info))\n : type(char(t)), my_val(e), my_arg(nullptr)\n , bypass_t(nullptr)\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info)))\n {}\n // constructor for pointer parameter\n queueing_port_operation(const T* p, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) :\n type(char(t)), my_arg(const_cast(p))\n , bypass_t(nullptr)\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info))\n {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n queueing_port_operation(const T* p, op_type t)\n : type(char(t)), my_arg(const_cast(p)), bypass_t(nullptr), metainfo(nullptr)\n {}\n#endif\n // constructor with no parameter\n queueing_port_operation(op_type t) : type(char(t)), my_arg(nullptr)\n , bypass_t(nullptr)\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr))\n {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator my_aggregator;\n\n void handle_operations(queueing_port_operation* op_list) {\n queueing_port_operation *current;\n bool was_empty;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n case try__put_task: {\n graph_task* rtask = nullptr;\n was_empty = this->buffer_empty();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(current->metainfo, nullptr);\n this->push_back(current->my_val, *(current->metainfo));\n#else\n this->push_back(current->my_val);\n#endif\n if (was_empty) rtask = my_join->decrement_port_count(false);\n else\n rtask = SUCCESSFULLY_ENQUEUED;\n current->bypass_t = rtask;\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n break;\n case get__item:\n if(!this->buffer_empty()) {\n __TBB_ASSERT(current->my_arg, nullptr);\n *(current->my_arg) = this->front();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (current->metainfo) {\n *(current->metainfo) = this->front_metainfo();\n }\n#endif\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n else {\n current->status.store( FAILED, std::memory_order_release);\n }\n break;\n case res_port:\n __TBB_ASSERT(this->my_item_valid(this->my_head), \"No item to reset\");\n this->destroy_front();\n if(this->my_item_valid(this->my_head)) {\n (void)my_join->decrement_port_count(true);\n }\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n }\n }\n }\n // ------------ End Aggregator ---------------\n\n protected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n\n private:\n graph_task* try_put_task_impl(const T& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n queueing_port_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n my_aggregator.execute(&op_data);\n __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, \"inconsistent return from aggregator\");\n if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED;\n return op_data.bypass_t;\n }\n\n protected:\n graph_task* try_put_task(const T &v) override {\n return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override {\n return try_put_task_impl(v, metainfo);\n }\n#endif\n\n graph& graph_reference() const override {\n return my_join->graph_ref;\n }\n\n public:\n\n //! Constructor\n queueing_port() : item_buffer() {\n my_join = nullptr;\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n //! copy constructor\n queueing_port(const queueing_port& /* other */) = delete;\n\n //! record parent for tallying available items\n void set_join_node_pointer(queueing_forwarding_base *join) {\n my_join = join;\n }\n\n bool get_item( T &v ) {\n queueing_port_operation op_data(&v, get__item);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool get_item( T& v, message_metainfo& metainfo ) {\n queueing_port_operation op_data(&v, get__item, metainfo);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n#endif\n\n // reset_port is called when item is accepted by successor, but\n // is initiated by join_node.\n void reset_port() {\n queueing_port_operation op_data(res_port);\n my_aggregator.execute(&op_data);\n return;\n }\n\n void reset_receiver(reset_flags) {\n item_buffer::reset();\n }\n\n private:\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n friend class get_graph_helper;\n#endif\n\n queueing_forwarding_base *my_join;\n }; // queueing_port\n\n#include \"_flow_graph_tagged_buffer_impl.h\"\n\n template\n struct count_element {\n K my_key;\n size_t my_value;\n };\n\n // method to access the key in the counting table\n // the ref has already been removed from K\n template< typename K >\n struct key_to_count_functor {\n typedef count_element table_item_type;\n const K& operator()(const table_item_type& v) { return v.my_key; }\n };\n\n template \n struct key_matching_port_base {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n using type = metainfo_hash_buffer;\n#else\n using type = hash_buffer;\n#endif\n };\n\n // the ports can have only one template parameter. We wrap the types needed in\n // a traits type\n template< class TraitsType >\n class key_matching_port :\n public receiver,\n public key_matching_port_base< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK,\n typename TraitsType::KHash >::type\n {\n public:\n typedef TraitsType traits;\n typedef key_matching_port class_type;\n typedef typename TraitsType::T input_type;\n typedef typename TraitsType::K key_type;\n typedef typename std::decay::type noref_key_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename TraitsType::TtoK type_to_key_func_type;\n typedef typename TraitsType::KHash hash_compare_type;\n typedef typename key_matching_port_base::type buffer_type;\n\n private:\n// ----------- Aggregator ------------\n private:\n enum op_type { try__put, get__item, res_port\n };\n\n class key_matching_port_operation : public d1::aggregated_operation {\n public:\n char type;\n input_type my_val;\n input_type *my_arg;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo = nullptr;\n#endif\n // constructor for value parameter\n key_matching_port_operation(const input_type& e, op_type t\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info))\n : type(char(t)), my_val(e), my_arg(nullptr)\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info))) {}\n\n // constructor for pointer parameter\n key_matching_port_operation(const input_type* p, op_type t\n __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info))\n : type(char(t)), my_arg(const_cast(p))\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {}\n\n // constructor with no parameter\n key_matching_port_operation(op_type t) : type(char(t)), my_arg(nullptr) {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator my_aggregator;\n\n void handle_operations(key_matching_port_operation* op_list) {\n key_matching_port_operation *current;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n case try__put: {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(current->metainfo, nullptr);\n bool was_inserted = this->insert_with_key(current->my_val, *(current->metainfo));\n#else\n bool was_inserted = this->insert_with_key(current->my_val);\n#endif\n // return failure if a duplicate insertion occurs\n current->status.store( was_inserted ? SUCCEEDED : FAILED, std::memory_order_release);\n }\n break;\n case get__item: {\n // use current_key from FE for item\n __TBB_ASSERT(current->my_arg, nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(current->metainfo, nullptr);\n bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg),\n *(current->metainfo));\n#else\n bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg));\n#endif\n#if TBB_USE_DEBUG\n if (!find_result) {\n __TBB_ASSERT(false, \"Failed to find item corresponding to current_key.\");\n }\n#else\n tbb::detail::suppress_unused_warning(find_result);\n#endif\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n break;\n case res_port:\n // use current_key from FE for item\n this->delete_with_key(my_join->current_key);\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n }\n }\n }\n// ------------ End Aggregator ---------------\n protected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n private:\n graph_task* try_put_task_impl(const input_type& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n key_matching_port_operation op_data(v, try__put __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n graph_task* rtask = nullptr;\n my_aggregator.execute(&op_data);\n if(op_data.status == SUCCEEDED) {\n rtask = my_join->increment_key_count((*(this->get_key_func()))(v)); // may spawn\n // rtask has to reflect the return status of the try_put\n if(!rtask) rtask = SUCCESSFULLY_ENQUEUED;\n }\n return rtask;\n }\n protected:\n graph_task* try_put_task(const input_type& v) override {\n return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override {\n return try_put_task_impl(v, metainfo);\n }\n#endif\n\n graph& graph_reference() const override {\n return my_join->graph_ref;\n }\n\n public:\n\n key_matching_port() : receiver(), buffer_type() {\n my_join = nullptr;\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n // copy constructor\n key_matching_port(const key_matching_port& /*other*/) = delete;\n#if __INTEL_COMPILER <= 2021\n // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited\n // class while the parent class has the virtual keyword for the destrocutor.\n virtual\n#endif\n ~key_matching_port() { }\n\n void set_join_node_pointer(forwarding_base *join) {\n my_join = dynamic_cast*>(join);\n }\n\n void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); }\n\n type_to_key_func_type* get_my_key_func() { return this->get_key_func(); }\n\n bool get_item( input_type &v ) {\n // aggregator uses current_key from FE for Key\n key_matching_port_operation op_data(&v, get__item);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool get_item( input_type& v, message_metainfo& metainfo ) {\n // aggregator uses current_key from FE for Key\n key_matching_port_operation op_data(&v, get__item, metainfo);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n#endif\n\n // reset_port is called when item is accepted by successor, but\n // is initiated by join_node.\n void reset_port() {\n key_matching_port_operation op_data(res_port);\n my_aggregator.execute(&op_data);\n return;\n }\n\n void reset_receiver(reset_flags ) {\n buffer_type::reset();\n }\n\n private:\n // my_join forwarding base used to count number of inputs that\n // received key.\n matching_forwarding_base *my_join;\n }; // key_matching_port\n\n using namespace graph_policy_namespace;\n\n template\n class join_node_base;\n\n //! join_node_FE : implements input port policy\n template\n class join_node_FE;\n\n template\n class join_node_FE : public reserving_forwarding_base {\n private:\n static const int N = std::tuple_size::value;\n typedef OutputTuple output_type;\n typedef InputTuple input_type;\n typedef join_node_base base_node_type; // for forwarding\n public:\n join_node_FE(graph &g) : reserving_forwarding_base(g), my_node(nullptr) {\n ports_with_no_inputs = N;\n join_helper::set_join_node_pointer(my_inputs, this);\n }\n\n join_node_FE(const join_node_FE& other) : reserving_forwarding_base((other.reserving_forwarding_base::graph_ref)), my_node(nullptr) {\n ports_with_no_inputs = N;\n join_helper::set_join_node_pointer(my_inputs, this);\n }\n\n void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; }\n\n void increment_port_count() override {\n ++ports_with_no_inputs;\n }\n\n // if all input_ports have predecessors, spawn forward to try and consume tuples\n graph_task* decrement_port_count() override {\n if(ports_with_no_inputs.fetch_sub(1) == 1) {\n if(is_graph_active(this->graph_ref)) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass task_type;\n graph_task* t = allocator.new_object(graph_ref, allocator, *my_node);\n spawn_in_graph_arena(this->graph_ref, *t);\n }\n }\n return nullptr;\n }\n\n input_type &input_ports() { return my_inputs; }\n\n protected:\n\n void reset( reset_flags f) {\n // called outside of parallel contexts\n ports_with_no_inputs = N;\n join_helper::reset_inputs(my_inputs, f);\n }\n\n // all methods on input ports should be called under mutual exclusion from join_node_base.\n\n bool tuple_build_may_succeed() {\n return !ports_with_no_inputs;\n }\n\n bool try_to_make_tuple(output_type &out) {\n if(ports_with_no_inputs) return false;\n return join_helper::reserve(my_inputs, out);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) {\n if (ports_with_no_inputs) return false;\n return join_helper::reserve(my_inputs, out, metainfo);\n }\n#endif\n\n void tuple_accepted() {\n join_helper::consume_reservations(my_inputs);\n }\n void tuple_rejected() {\n join_helper::release_reservations(my_inputs);\n }\n\n input_type my_inputs;\n base_node_type *my_node;\n std::atomic ports_with_no_inputs;\n }; // join_node_FE\n\n template\n class join_node_FE : public queueing_forwarding_base {\n public:\n static const int N = std::tuple_size::value;\n typedef OutputTuple output_type;\n typedef InputTuple input_type;\n typedef join_node_base base_node_type; // for forwarding\n\n join_node_FE(graph &g) : queueing_forwarding_base(g), my_node(nullptr) {\n ports_with_no_items = N;\n join_helper::set_join_node_pointer(my_inputs, this);\n }\n\n join_node_FE(const join_node_FE& other) : queueing_forwarding_base((other.queueing_forwarding_base::graph_ref)), my_node(nullptr) {\n ports_with_no_items = N;\n join_helper::set_join_node_pointer(my_inputs, this);\n }\n\n // needed for forwarding\n void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; }\n\n void reset_port_count() {\n ports_with_no_items = N;\n }\n\n // if all input_ports have items, spawn forward to try and consume tuples\n graph_task* decrement_port_count(bool handle_task) override\n {\n if(ports_with_no_items.fetch_sub(1) == 1) {\n if(is_graph_active(this->graph_ref)) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass task_type;\n graph_task* t = allocator.new_object(graph_ref, allocator, *my_node);\n if( !handle_task )\n return t;\n spawn_in_graph_arena(this->graph_ref, *t);\n }\n }\n return nullptr;\n }\n\n input_type &input_ports() { return my_inputs; }\n\n protected:\n\n void reset( reset_flags f) {\n reset_port_count();\n join_helper::reset_inputs(my_inputs, f );\n }\n\n // all methods on input ports should be called under mutual exclusion from join_node_base.\n\n bool tuple_build_may_succeed() {\n return !ports_with_no_items;\n }\n\n bool try_to_make_tuple(output_type &out) {\n if(ports_with_no_items) return false;\n return join_helper::get_items(my_inputs, out);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) {\n if(ports_with_no_items) return false;\n return join_helper::get_items(my_inputs, out, metainfo);\n }\n#endif\n\n void tuple_accepted() {\n reset_port_count();\n join_helper::reset_ports(my_inputs);\n }\n void tuple_rejected() {\n // nothing to do.\n }\n\n input_type my_inputs;\n base_node_type *my_node;\n std::atomic ports_with_no_items;\n }; // join_node_FE\n\n // key_matching join front-end.\n template\n class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base,\n // buffer of key value counts\n public hash_buffer< // typedefed below to key_to_count_buffer_type\n typename std::decay::type&, // force ref type on K\n count_element::type>,\n type_to_key_function_body<\n count_element::type>,\n typename std::decay::type& >,\n KHash >,\n // buffer of output items\n public item_buffer {\n public:\n static const int N = std::tuple_size::value;\n typedef OutputTuple output_type;\n typedef InputTuple input_type;\n typedef K key_type;\n typedef typename std::decay::type unref_key_type;\n typedef KHash key_hash_compare;\n // must use K without ref.\n typedef count_element count_element_type;\n // method that lets us refer to the key of this type.\n typedef key_to_count_functor key_to_count_func;\n typedef type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type;\n typedef type_to_key_function_body_leaf TtoK_function_body_leaf_type;\n // this is the type of the special table that keeps track of the number of discrete\n // elements corresponding to each key that we've seen.\n typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare >\n key_to_count_buffer_type;\n typedef item_buffer output_buffer_type;\n typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding\n typedef matching_forwarding_base forwarding_base_type;\n\n// ----------- Aggregator ------------\n // the aggregator is only needed to serialize the access to the hash table.\n // and the output_buffer_type base class\n private:\n enum op_type { res_count, inc_count, may_succeed, try_make };\n typedef join_node_FE, InputTuple, OutputTuple> class_type;\n\n class key_matching_FE_operation : public d1::aggregated_operation {\n public:\n char type;\n unref_key_type my_val;\n output_type* my_output;\n graph_task* bypass_t;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo = nullptr;\n#endif\n // constructor for value parameter\n key_matching_FE_operation(const unref_key_type& e , op_type t) : type(char(t)), my_val(e),\n my_output(nullptr), bypass_t(nullptr) {}\n key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(nullptr) {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n key_matching_FE_operation(output_type *p, op_type t, message_metainfo& info)\n : type(char(t)), my_output(p), bypass_t(nullptr), metainfo(&info) {}\n#endif\n // constructor with no parameter\n key_matching_FE_operation(op_type t) : type(char(t)), my_output(nullptr), bypass_t(nullptr) {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator my_aggregator;\n\n // called from aggregator, so serialized\n // returns a task pointer if the a task would have been enqueued but we asked that\n // it be returned. Otherwise returns nullptr.\n graph_task* fill_output_buffer(unref_key_type &t) {\n output_type l_out;\n graph_task* rtask = nullptr;\n bool do_fwd = this->buffer_empty() && is_graph_active(this->graph_ref);\n this->current_key = t;\n this->delete_with_key(this->current_key); // remove the key\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n if(join_helper::get_items(my_inputs, l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) { // <== call back\n this->push_back(l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass task_type;\n rtask = allocator.new_object(this->graph_ref, allocator, *my_node);\n do_fwd = false;\n }\n // retire the input values\n join_helper::reset_ports(my_inputs); // <== call back\n }\n else {\n __TBB_ASSERT(false, \"should have had something to push\");\n }\n return rtask;\n }\n\n void handle_operations(key_matching_FE_operation* op_list) {\n key_matching_FE_operation *current;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n case res_count: // called from BE\n {\n this->destroy_front();\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n break;\n case inc_count: { // called from input ports\n count_element_type *p = nullptr;\n unref_key_type &t = current->my_val;\n if(!(this->find_ref_with_key(t,p))) {\n count_element_type ev;\n ev.my_key = t;\n ev.my_value = 0;\n this->insert_with_key(ev);\n bool found = this->find_ref_with_key(t, p);\n __TBB_ASSERT_EX(found, \"should find key after inserting it\");\n }\n if(++(p->my_value) == size_t(N)) {\n current->bypass_t = fill_output_buffer(t);\n }\n }\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n case may_succeed: // called from BE\n current->status.store( this->buffer_empty() ? FAILED : SUCCEEDED, std::memory_order_release);\n break;\n case try_make: // called from BE\n if(this->buffer_empty()) {\n current->status.store( FAILED, std::memory_order_release);\n }\n else {\n *(current->my_output) = this->front();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (current->metainfo) {\n *(current->metainfo) = this->front_metainfo();\n }\n#endif\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n break;\n }\n }\n }\n// ------------ End Aggregator ---------------\n\n public:\n template\n join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(nullptr) {\n join_helper::set_join_node_pointer(my_inputs, this);\n join_helper::set_key_functors(my_inputs, TtoK_funcs);\n my_aggregator.initialize_handler(handler_type(this));\n TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func());\n this->set_key_func(cfb);\n }\n\n join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(),\n output_buffer_type() {\n my_node = nullptr;\n join_helper::set_join_node_pointer(my_inputs, this);\n join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs));\n my_aggregator.initialize_handler(handler_type(this));\n TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func());\n this->set_key_func(cfb);\n }\n\n // needed for forwarding\n void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; }\n\n void reset_port_count() { // called from BE\n key_matching_FE_operation op_data(res_count);\n my_aggregator.execute(&op_data);\n return;\n }\n\n // if all input_ports have items, spawn forward to try and consume tuples\n // return a task if we are asked and did create one.\n graph_task *increment_key_count(unref_key_type const & t) override { // called from input_ports\n key_matching_FE_operation op_data(t, inc_count);\n my_aggregator.execute(&op_data);\n return op_data.bypass_t;\n }\n\n input_type &input_ports() { return my_inputs; }\n\n protected:\n\n void reset( reset_flags f ) {\n // called outside of parallel contexts\n join_helper::reset_inputs(my_inputs, f);\n\n key_to_count_buffer_type::reset();\n output_buffer_type::reset();\n }\n\n // all methods on input ports should be called under mutual exclusion from join_node_base.\n\n bool tuple_build_may_succeed() { // called from back-end\n key_matching_FE_operation op_data(may_succeed);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n // cannot lock while calling back to input_ports. current_key will only be set\n // and reset under the aggregator, so it will remain consistent.\n bool try_to_make_tuple(output_type &out) {\n key_matching_FE_operation op_data(&out,try_make);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) {\n key_matching_FE_operation op_data(&out, try_make, metainfo);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n#endif\n\n void tuple_accepted() {\n reset_port_count(); // reset current_key after ports reset.\n }\n\n void tuple_rejected() {\n // nothing to do.\n }\n\n input_type my_inputs; // input ports\n base_node_type *my_node;\n }; // join_node_FE, InputTuple, OutputTuple>\n\n //! join_node_base\n template\n class join_node_base : public graph_node, public join_node_FE,\n public sender {\n protected:\n using graph_node::my_graph;\n public:\n typedef OutputTuple output_type;\n\n typedef typename sender::successor_type successor_type;\n typedef join_node_FE input_ports_type;\n using input_ports_type::tuple_build_may_succeed;\n using input_ports_type::try_to_make_tuple;\n using input_ports_type::tuple_accepted;\n using input_ports_type::tuple_rejected;\n\n private:\n // ----------- Aggregator ------------\n enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass\n };\n typedef join_node_base class_type;\n\n class join_node_base_operation : public d1::aggregated_operation {\n public:\n char type;\n union {\n output_type *my_arg;\n successor_type *my_succ;\n };\n graph_task* bypass_t;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo;\n#endif\n join_node_base_operation(const output_type& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info))\n : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr)\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n join_node_base_operation(const output_type& e, op_type t)\n : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr), metainfo(nullptr) {}\n#endif\n join_node_base_operation(const successor_type &s, op_type t) : type(char(t)),\n my_succ(const_cast(&s)), bypass_t(nullptr) {}\n join_node_base_operation(op_type t) : type(char(t)), bypass_t(nullptr) {}\n };\n\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n bool forwarder_busy;\n d1::aggregator my_aggregator;\n\n void handle_operations(join_node_base_operation* op_list) {\n join_node_base_operation *current;\n while(op_list) {\n current = op_list;\n op_list = op_list->next;\n switch(current->type) {\n case reg_succ: {\n my_successors.register_successor(*(current->my_succ));\n if(tuple_build_may_succeed() && !forwarder_busy && is_graph_active(my_graph)) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass< join_node_base > task_type;\n graph_task* t = allocator.new_object(my_graph, allocator, *this);\n spawn_in_graph_arena(my_graph, *t);\n forwarder_busy = true;\n }\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n break;\n case rem_succ:\n my_successors.remove_successor(*(current->my_succ));\n current->status.store( SUCCEEDED, std::memory_order_release);\n break;\n case try__get:\n if(tuple_build_may_succeed()) {\n bool make_tuple_result = false;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (current->metainfo) {\n make_tuple_result = try_to_make_tuple(*(current->my_arg), *(current->metainfo));\n } else\n#endif\n {\n make_tuple_result = try_to_make_tuple(*(current->my_arg));\n }\n if(make_tuple_result) {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (current->metainfo) {\n // Since elements would be removed from queues while calling to tuple_accepted\n // together with corresponding message_metainfo objects\n // we need to prolong the wait until the successor would create a task for removed elements\n for (auto waiter : current->metainfo->waiters()) {\n waiter->reserve(1);\n }\n }\n#endif\n tuple_accepted();\n current->status.store( SUCCEEDED, std::memory_order_release);\n }\n else current->status.store( FAILED, std::memory_order_release);\n }\n else current->status.store( FAILED, std::memory_order_release);\n break;\n case do_fwrd_bypass: {\n bool build_succeeded;\n graph_task *last_task = nullptr;\n output_type out;\n // forwarding must be exclusive, because try_to_make_tuple and tuple_accepted\n // are separate locked methods in the FE. We could conceivably fetch the front\n // of the FE queue, then be swapped out, have someone else consume the FE's\n // object, then come back, forward, and then try to remove it from the queue\n // again. Without reservation of the FE, the methods accessing it must be locked.\n // We could remember the keys of the objects we forwarded, and then remove\n // them from the input ports after forwarding is complete?\n if(tuple_build_may_succeed()) { // checks output queue of FE\n do {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n // fetch front_end of queue\n build_succeeded = try_to_make_tuple(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n if(build_succeeded) {\n graph_task *new_task =\n my_successors.try_put_task(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n last_task = combine_tasks(my_graph, last_task, new_task);\n if(new_task) {\n tuple_accepted();\n }\n else {\n tuple_rejected();\n build_succeeded = false;\n }\n }\n } while(build_succeeded);\n }\n current->bypass_t = last_task;\n current->status.store( SUCCEEDED, std::memory_order_release);\n forwarder_busy = false;\n }\n break;\n }\n }\n }\n // ---------- end aggregator -----------\n public:\n join_node_base(graph &g)\n : graph_node(g), input_ports_type(g), forwarder_busy(false), my_successors(this)\n {\n input_ports_type::set_my_node(this);\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n join_node_base(const join_node_base& other) :\n graph_node(other.graph_node::my_graph), input_ports_type(other),\n sender(), forwarder_busy(false), my_successors(this)\n {\n input_ports_type::set_my_node(this);\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n template\n join_node_base(graph &g, FunctionTuple f)\n : graph_node(g), input_ports_type(g, f), forwarder_busy(false), my_successors(this)\n {\n input_ports_type::set_my_node(this);\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n bool register_successor(successor_type &r) override {\n join_node_base_operation op_data(r, reg_succ);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n bool remove_successor( successor_type &r) override {\n join_node_base_operation op_data(r, rem_succ);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n bool try_get( output_type &v) override {\n join_node_base_operation op_data(v, try__get);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_get( output_type &v, message_metainfo& metainfo) override {\n join_node_base_operation op_data(v, try__get, metainfo);\n my_aggregator.execute(&op_data);\n return op_data.status == SUCCEEDED;\n }\n#endif\n\n protected:\n void reset_node(reset_flags f) override {\n input_ports_type::reset(f);\n if(f & rf_clear_edges) my_successors.clear();\n }\n\n private:\n broadcast_cache my_successors;\n\n friend class forward_task_bypass< join_node_base >;\n graph_task *forward_task() {\n join_node_base_operation op_data(do_fwrd_bypass);\n my_aggregator.execute(&op_data);\n return op_data.bypass_t;\n }\n\n }; // join_node_base\n\n // join base class type generator\n template class PT, typename OutputTuple, typename JP>\n struct join_base {\n typedef join_node_base::type, OutputTuple> type;\n };\n\n template\n struct join_base > {\n typedef key_matching key_traits_type;\n typedef K key_type;\n typedef KHash key_hash_compare;\n typedef join_node_base< key_traits_type,\n // ports type\n typename wrap_key_tuple_elements::type,\n OutputTuple > type;\n };\n\n //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type\n // using tuple_element. The class PT is the port type (reserving_port, queueing_port, key_matching_port)\n // and should match the typename.\n\n template class PT, typename OutputTuple, typename JP>\n class unfolded_join_node : public join_base::type {\n public:\n typedef typename wrap_tuple_elements::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base base_type;\n public:\n unfolded_join_node(graph &g) : base_type(g) {}\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n template \n struct key_from_message_body {\n K operator()(const T& t) const {\n return key_from_message(t);\n }\n };\n // Adds const to reference type\n template \n struct key_from_message_body {\n const K& operator()(const T& t) const {\n return key_from_message(t);\n }\n };\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n // key_matching unfolded_join_node. This must be a separate specialization because the constructors\n // differ.\n\n template\n class unfolded_join_node<2,key_matching_port,OutputTuple,key_matching > : public\n join_base<2,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n public:\n typedef typename wrap_key_tuple_elements<2,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base, input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef std::tuple< f0_p, f1_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1)\n ) ) {\n static_assert(std::tuple_size::value == 2, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n\n template\n class unfolded_join_node<3,key_matching_port,OutputTuple,key_matching > : public\n join_base<3,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n public:\n typedef typename wrap_key_tuple_elements<3,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base, input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef std::tuple< f0_p, f1_p, f2_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2)\n ) ) {\n static_assert(std::tuple_size::value == 3, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n\n template\n class unfolded_join_node<4,key_matching_port,OutputTuple,key_matching > : public\n join_base<4,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n public:\n typedef typename wrap_key_tuple_elements<4,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base, input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3)\n ) ) {\n static_assert(std::tuple_size::value == 4, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n\n template\n class unfolded_join_node<5,key_matching_port,OutputTuple,key_matching > : public\n join_base<5,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n public:\n typedef typename wrap_key_tuple_elements<5,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4)\n ) ) {\n static_assert(std::tuple_size::value == 5, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n\n#if __TBB_VARIADIC_MAX >= 6\n template\n class unfolded_join_node<6,key_matching_port,OutputTuple,key_matching > : public\n join_base<6,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n typedef typename std::tuple_element<5, OutputTuple>::type T5;\n public:\n typedef typename wrap_key_tuple_elements<6,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef type_to_key_function_body *f5_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, Body5 body5)\n : base_type(g, func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4),\n new type_to_key_function_body_leaf(body5)\n ) ) {\n static_assert(std::tuple_size::value == 6, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n#endif\n\n#if __TBB_VARIADIC_MAX >= 7\n template\n class unfolded_join_node<7,key_matching_port,OutputTuple,key_matching > : public\n join_base<7,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n typedef typename std::tuple_element<5, OutputTuple>::type T5;\n typedef typename std::tuple_element<6, OutputTuple>::type T6;\n public:\n typedef typename wrap_key_tuple_elements<7,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef type_to_key_function_body *f5_p;\n typedef type_to_key_function_body *f6_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4,\n Body5 body5, Body6 body6) : base_type(g, func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4),\n new type_to_key_function_body_leaf(body5),\n new type_to_key_function_body_leaf(body6)\n ) ) {\n static_assert(std::tuple_size::value == 7, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n#endif\n\n#if __TBB_VARIADIC_MAX >= 8\n template\n class unfolded_join_node<8,key_matching_port,OutputTuple,key_matching > : public\n join_base<8,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n typedef typename std::tuple_element<5, OutputTuple>::type T5;\n typedef typename std::tuple_element<6, OutputTuple>::type T6;\n typedef typename std::tuple_element<7, OutputTuple>::type T7;\n public:\n typedef typename wrap_key_tuple_elements<8,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef type_to_key_function_body *f5_p;\n typedef type_to_key_function_body *f6_p;\n typedef type_to_key_function_body *f7_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4,\n Body5 body5, Body6 body6, Body7 body7) : base_type(g, func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4),\n new type_to_key_function_body_leaf(body5),\n new type_to_key_function_body_leaf(body6),\n new type_to_key_function_body_leaf(body7)\n ) ) {\n static_assert(std::tuple_size::value == 8, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n#endif\n\n#if __TBB_VARIADIC_MAX >= 9\n template\n class unfolded_join_node<9,key_matching_port,OutputTuple,key_matching > : public\n join_base<9,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n typedef typename std::tuple_element<5, OutputTuple>::type T5;\n typedef typename std::tuple_element<6, OutputTuple>::type T6;\n typedef typename std::tuple_element<7, OutputTuple>::type T7;\n typedef typename std::tuple_element<8, OutputTuple>::type T8;\n public:\n typedef typename wrap_key_tuple_elements<9,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef type_to_key_function_body *f5_p;\n typedef type_to_key_function_body *f6_p;\n typedef type_to_key_function_body *f7_p;\n typedef type_to_key_function_body *f8_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4,\n Body5 body5, Body6 body6, Body7 body7, Body8 body8) : base_type(g, func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4),\n new type_to_key_function_body_leaf(body5),\n new type_to_key_function_body_leaf(body6),\n new type_to_key_function_body_leaf(body7),\n new type_to_key_function_body_leaf(body8)\n ) ) {\n static_assert(std::tuple_size::value == 9, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n#endif\n\n#if __TBB_VARIADIC_MAX >= 10\n template\n class unfolded_join_node<10,key_matching_port,OutputTuple,key_matching > : public\n join_base<10,key_matching_port,OutputTuple,key_matching >::type {\n typedef typename std::tuple_element<0, OutputTuple>::type T0;\n typedef typename std::tuple_element<1, OutputTuple>::type T1;\n typedef typename std::tuple_element<2, OutputTuple>::type T2;\n typedef typename std::tuple_element<3, OutputTuple>::type T3;\n typedef typename std::tuple_element<4, OutputTuple>::type T4;\n typedef typename std::tuple_element<5, OutputTuple>::type T5;\n typedef typename std::tuple_element<6, OutputTuple>::type T6;\n typedef typename std::tuple_element<7, OutputTuple>::type T7;\n typedef typename std::tuple_element<8, OutputTuple>::type T8;\n typedef typename std::tuple_element<9, OutputTuple>::type T9;\n public:\n typedef typename wrap_key_tuple_elements<10,key_matching_port,key_matching,OutputTuple>::type input_ports_type;\n typedef OutputTuple output_type;\n private:\n typedef join_node_base , input_ports_type, output_type > base_type;\n typedef type_to_key_function_body *f0_p;\n typedef type_to_key_function_body *f1_p;\n typedef type_to_key_function_body *f2_p;\n typedef type_to_key_function_body *f3_p;\n typedef type_to_key_function_body *f4_p;\n typedef type_to_key_function_body *f5_p;\n typedef type_to_key_function_body *f6_p;\n typedef type_to_key_function_body *f7_p;\n typedef type_to_key_function_body *f8_p;\n typedef type_to_key_function_body *f9_p;\n typedef std::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p, f9_p > func_initializer_type;\n public:\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n unfolded_join_node(graph &g) : base_type(g,\n func_initializer_type(\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body()),\n new type_to_key_function_body_leaf >(key_from_message_body())\n ) ) {\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n template\n unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4,\n Body5 body5, Body6 body6, Body7 body7, Body8 body8, Body9 body9) : base_type(g, func_initializer_type(\n new type_to_key_function_body_leaf(body0),\n new type_to_key_function_body_leaf(body1),\n new type_to_key_function_body_leaf(body2),\n new type_to_key_function_body_leaf(body3),\n new type_to_key_function_body_leaf(body4),\n new type_to_key_function_body_leaf(body5),\n new type_to_key_function_body_leaf(body6),\n new type_to_key_function_body_leaf(body7),\n new type_to_key_function_body_leaf(body8),\n new type_to_key_function_body_leaf(body9)\n ) ) {\n static_assert(std::tuple_size::value == 10, \"wrong number of body initializers\");\n }\n unfolded_join_node(const unfolded_join_node &other) : base_type(other) {}\n };\n#endif\n\n //! templated function to refer to input ports of the join node\n template\n typename std::tuple_element::type &input_port(JNT &jn) {\n return std::get(jn.input_ports());\n }\n\n#endif // __TBB__flow_graph_join_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_node_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_node_impl_H\n#define __TBB__flow_graph_node_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n#include \"_flow_graph_item_buffer_impl.h\"\n\ntemplate< typename T, typename A >\nclass function_input_queue : public item_buffer {\npublic:\n bool empty() const {\n return this->buffer_empty();\n }\n\n const T& front() const {\n return this->item_buffer::front();\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n const message_metainfo& front_metainfo() const {\n return this->item_buffer::front_metainfo();\n }\n#endif\n\n void pop() {\n this->destroy_front();\n }\n\n bool push( T& t ) {\n return this->push_back( t );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool push( T& t, const message_metainfo& metainfo ) {\n return this->push_back(t, metainfo);\n }\n#endif\n};\n\n//! Input and scheduling for a function node that takes a type Input as input\n// The only up-ref is apply_body_impl, which should implement the function\n// call and any handling of the result.\ntemplate< typename Input, typename Policy, typename A, typename ImplType >\nclass function_input_base : public receiver, no_assign {\n enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency\n };\n typedef function_input_base class_type;\n\npublic:\n\n //! The input type of this receiver\n typedef Input input_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef predecessor_cache predecessor_cache_type;\n typedef function_input_queue input_queue_type;\n typedef typename allocator_traits::template rebind_alloc allocator_type;\n static_assert(!has_policy::value || !has_policy::value, \"\");\n\n //! Constructor for function_input_base\n function_input_base( graph &g, size_t max_concurrency, node_priority_t a_priority, bool is_no_throw )\n : my_graph_ref(g), my_max_concurrency(max_concurrency)\n , my_concurrency(0), my_priority(a_priority), my_is_no_throw(is_no_throw)\n , my_queue(!has_policy::value ? new input_queue_type() : nullptr)\n , my_predecessors(this)\n , forwarder_busy(false)\n {\n my_aggregator.initialize_handler(handler_type(this));\n }\n\n //! Copy constructor\n function_input_base( const function_input_base& src )\n : function_input_base(src.my_graph_ref, src.my_max_concurrency, src.my_priority, src.my_is_no_throw) {}\n\n //! Destructor\n // The queue is allocated by the constructor for {multi}function_node.\n // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead.\n // This would be an interface-breaking change.\n virtual ~function_input_base() {\n delete my_queue;\n my_queue = nullptr;\n }\n\n graph_task* try_put_task( const input_type& t) override {\n return try_put_task_base(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task( const input_type& t, const message_metainfo& metainfo ) override {\n return try_put_task_base(t, metainfo);\n }\n#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n\n //! Adds src to the list of cached predecessors.\n bool register_predecessor( predecessor_type &src ) override {\n operation_type op_data(reg_pred);\n op_data.r = &src;\n my_aggregator.execute(&op_data);\n return true;\n }\n\n //! Removes src from the list of cached predecessors.\n bool remove_predecessor( predecessor_type &src ) override {\n operation_type op_data(rem_pred);\n op_data.r = &src;\n my_aggregator.execute(&op_data);\n return true;\n }\n\nprotected:\n\n void reset_function_input_base( reset_flags f) {\n my_concurrency = 0;\n if(my_queue) {\n my_queue->reset();\n }\n reset_receiver(f);\n forwarder_busy = false;\n }\n\n graph& my_graph_ref;\n const size_t my_max_concurrency;\n size_t my_concurrency;\n node_priority_t my_priority;\n const bool my_is_no_throw;\n input_queue_type *my_queue;\n predecessor_cache my_predecessors;\n\n void reset_receiver( reset_flags f) {\n if( f & rf_clear_edges) my_predecessors.clear();\n else\n my_predecessors.reset();\n __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), \"function_input_base reset failed\");\n }\n\n graph& graph_reference() const override {\n return my_graph_ref;\n }\n\n graph_task* try_get_postponed_task(const input_type& i) {\n operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item\n my_aggregator.execute(&op_data);\n return op_data.bypass_t;\n }\n\nprivate:\n\n friend class apply_body_task_bypass< class_type, input_type >;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n friend class apply_body_task_bypass< class_type, input_type, trackable_messages_graph_task >;\n#endif\n friend class forward_task_bypass< class_type >;\n\n class operation_type : public d1::aggregated_operation< operation_type > {\n public:\n char type;\n union {\n input_type *elem;\n predecessor_type *r;\n };\n graph_task* bypass_t;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo;\n#endif\n operation_type(const input_type& e, op_type t) :\n type(char(t)), elem(const_cast(&e)), bypass_t(nullptr)\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n , metainfo(nullptr)\n#endif\n {}\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n operation_type(const input_type& e, op_type t, const message_metainfo& info) :\n type(char(t)), elem(const_cast(&e)), bypass_t(nullptr),\n metainfo(const_cast(&info)) {}\n#endif\n operation_type(op_type t) : type(char(t)), r(nullptr), bypass_t(nullptr) {}\n };\n\n bool forwarder_busy;\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator< handler_type, operation_type > my_aggregator;\n\n graph_task* perform_queued_requests() {\n graph_task* new_task = nullptr;\n if(my_queue) {\n if(!my_queue->empty()) {\n ++my_concurrency;\n // TODO: consider removing metainfo from the queue using move semantics to avoid\n // ref counter increase\n new_task = create_body_task(my_queue->front()\n __TBB_FLOW_GRAPH_METAINFO_ARG(my_queue->front_metainfo()));\n\n my_queue->pop();\n }\n }\n else {\n input_type i;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n if(my_predecessors.get_item(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) {\n ++my_concurrency;\n new_task = create_body_task(i __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo)));\n }\n }\n return new_task;\n }\n void handle_operations(operation_type *op_list) {\n operation_type* tmp;\n while (op_list) {\n tmp = op_list;\n op_list = op_list->next;\n switch (tmp->type) {\n case reg_pred:\n my_predecessors.add(*(tmp->r));\n tmp->status.store(SUCCEEDED, std::memory_order_release);\n if (!forwarder_busy) {\n forwarder_busy = true;\n spawn_forward_task();\n }\n break;\n case rem_pred:\n my_predecessors.remove(*(tmp->r));\n tmp->status.store(SUCCEEDED, std::memory_order_release);\n break;\n case app_body_bypass: {\n tmp->bypass_t = nullptr;\n __TBB_ASSERT(my_max_concurrency != 0, nullptr);\n --my_concurrency;\n if(my_concurrencybypass_t = perform_queued_requests();\n tmp->status.store(SUCCEEDED, std::memory_order_release);\n }\n break;\n case tryput_bypass: internal_try_put_task(tmp); break;\n case try_fwd: internal_forward(tmp); break;\n case occupy_concurrency:\n if (my_concurrency < my_max_concurrency) {\n ++my_concurrency;\n tmp->status.store(SUCCEEDED, std::memory_order_release);\n } else {\n tmp->status.store(FAILED, std::memory_order_release);\n }\n break;\n }\n }\n }\n\n //! Put to the node, but return the task instead of enqueueing it\n void internal_try_put_task(operation_type *op) {\n __TBB_ASSERT(my_max_concurrency != 0, nullptr);\n if (my_concurrency < my_max_concurrency) {\n ++my_concurrency;\n graph_task* new_task = create_body_task(*(op->elem)\n __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo)));\n op->bypass_t = new_task;\n op->status.store(SUCCEEDED, std::memory_order_release);\n } else if ( my_queue && my_queue->push(*(op->elem)\n __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo))) )\n {\n op->bypass_t = SUCCESSFULLY_ENQUEUED;\n op->status.store(SUCCEEDED, std::memory_order_release);\n } else {\n op->bypass_t = nullptr;\n op->status.store(FAILED, std::memory_order_release);\n }\n }\n\n //! Creates tasks for postponed messages if available and if concurrency allows\n void internal_forward(operation_type *op) {\n op->bypass_t = nullptr;\n if (my_concurrency < my_max_concurrency)\n op->bypass_t = perform_queued_requests();\n if(op->bypass_t)\n op->status.store(SUCCEEDED, std::memory_order_release);\n else {\n forwarder_busy = false;\n op->status.store(FAILED, std::memory_order_release);\n }\n }\n\n graph_task* internal_try_put_bypass( const input_type& t\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n operation_type op_data(t, tryput_bypass __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n my_aggregator.execute(&op_data);\n if( op_data.status == SUCCEEDED ) {\n return op_data.bypass_t;\n }\n return nullptr;\n }\n\n graph_task* try_put_task_base(const input_type& t\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n if ( my_is_no_throw )\n return try_put_task_impl(t, has_policy()\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n else\n return try_put_task_impl(t, std::false_type()\n __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n\n graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::true_type\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n if( my_max_concurrency == 0 ) {\n return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n } else {\n operation_type check_op(t, occupy_concurrency);\n my_aggregator.execute(&check_op);\n if( check_op.status == SUCCEEDED ) {\n return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n }\n\n graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::false_type\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n if( my_max_concurrency == 0 ) {\n return create_body_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n } else {\n return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n }\n\n //! Applies the body to the provided input\n // then decides if more work is available\n graph_task* apply_body_bypass( const input_type &i\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n\n {\n return static_cast(this)->apply_body_impl_bypass(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n }\n\n //! allocates a task to apply a body\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n graph_task* create_body_task( const input_type &input, Metainfo&& metainfo )\n#else\n graph_task* create_body_task( const input_type &input )\n#endif\n {\n if (!is_graph_active(my_graph_ref)) {\n return nullptr;\n }\n // TODO revamp: extract helper for common graph task allocation part\n d1::small_object_allocator allocator{};\n graph_task* t = nullptr;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (!metainfo.empty()) {\n using task_type = apply_body_task_bypass;\n t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority, std::forward(metainfo));\n } else\n#endif\n {\n using task_type = apply_body_task_bypass;\n t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority);\n }\n return t;\n }\n\n //! This is executed by an enqueued task, the \"forwarder\"\n graph_task* forward_task() {\n operation_type op_data(try_fwd);\n graph_task* rval = nullptr;\n do {\n op_data.status = WAIT;\n my_aggregator.execute(&op_data);\n if(op_data.status == SUCCEEDED) {\n graph_task* ttask = op_data.bypass_t;\n __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, nullptr);\n rval = combine_tasks(my_graph_ref, rval, ttask);\n }\n } while (op_data.status == SUCCEEDED);\n return rval;\n }\n\n inline graph_task* create_forward_task() {\n if (!is_graph_active(my_graph_ref)) {\n return nullptr;\n }\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass task_type;\n graph_task* t = allocator.new_object( graph_reference(), allocator, *this, my_priority );\n return t;\n }\n\n //! Spawns a task that calls forward()\n inline void spawn_forward_task() {\n graph_task* tp = create_forward_task();\n if(tp) {\n spawn_in_graph_arena(graph_reference(), *tp);\n }\n }\n\n node_priority_t priority() const override { return my_priority; }\n}; // function_input_base\n\n//! Implements methods for a function node that takes a type Input as input and sends\n// a type Output to its successors.\ntemplate< typename Input, typename Output, typename Policy, typename A>\nclass function_input : public function_input_base > {\npublic:\n typedef Input input_type;\n typedef Output output_type;\n typedef function_body function_body_type;\n typedef function_input my_class;\n typedef function_input_base base_type;\n typedef function_input_queue input_queue_type;\n\n // constructor\n template\n function_input(\n graph &g, size_t max_concurrency, Body& body, node_priority_t a_priority )\n : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type())))\n , my_body( new function_body_leaf< input_type, output_type, Body>(body) )\n , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) {\n }\n\n //! Copy constructor\n function_input( const function_input& src ) :\n base_type(src),\n my_body( src.my_init_body->clone() ),\n my_init_body(src.my_init_body->clone() ) {\n }\n#if __INTEL_COMPILER <= 2021\n // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited\n // class while the parent class has the virtual keyword for the destrocutor.\n virtual\n#endif\n ~function_input() {\n delete my_body;\n delete my_init_body;\n }\n\n template< typename Body >\n Body copy_function_object() {\n function_body_type &body_ref = *this->my_body;\n return dynamic_cast< function_body_leaf & >(body_ref).get_body();\n }\n\n output_type apply_body_impl( const input_type& i) {\n // There is an extra copied needed to capture the\n // body execution without the try_put\n fgt_begin_body( my_body );\n output_type v = tbb::detail::invoke(*my_body, i);\n fgt_end_body( my_body );\n return v;\n }\n\n //TODO: consider moving into the base class\n graph_task* apply_body_impl_bypass( const input_type &i\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo))\n {\n output_type v = apply_body_impl(i);\n graph_task* postponed_task = nullptr;\n if( base_type::my_max_concurrency != 0 ) {\n postponed_task = base_type::try_get_postponed_task(i);\n __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, nullptr);\n }\n if( postponed_task ) {\n // make the task available for other workers since we do not know successors'\n // execution policy\n spawn_in_graph_arena(base_type::graph_reference(), *postponed_task);\n }\n graph_task* successor_task = successors().try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n#if _MSC_VER && !__INTEL_COMPILER\n// #pragma warning (push)\n// #pragma warning (disable: 4127) /* suppress conditional expression is constant */\n#endif\n if(has_policy::value) {\n#if _MSC_VER && !__INTEL_COMPILER\n// #pragma warning (pop)\n#endif\n if(!successor_task) {\n // Return confirmative status since current\n // node's body has been executed anyway\n successor_task = SUCCESSFULLY_ENQUEUED;\n }\n }\n return successor_task;\n }\n\nprotected:\n\n void reset_function_input(reset_flags f) {\n base_type::reset_function_input_base(f);\n if(f & rf_reset_bodies) {\n function_body_type *tmp = my_init_body->clone();\n delete my_body;\n my_body = tmp;\n }\n }\n\n function_body_type *my_body;\n function_body_type *my_init_body;\n virtual broadcast_cache &successors() = 0;\n\n}; // function_input\n\n\n// helper templates to clear the successor edges of the output ports of an multifunction_node\ntemplate struct clear_element {\n template static void clear_this(P &p) {\n (void)std::get(p).successors().clear();\n clear_element::clear_this(p);\n }\n#if TBB_USE_ASSERT\n template static bool this_empty(P &p) {\n if(std::get(p).successors().empty())\n return clear_element::this_empty(p);\n return false;\n }\n#endif\n};\n\ntemplate<> struct clear_element<1> {\n template static void clear_this(P &p) {\n (void)std::get<0>(p).successors().clear();\n }\n#if TBB_USE_ASSERT\n template static bool this_empty(P &p) {\n return std::get<0>(p).successors().empty();\n }\n#endif\n};\n\ntemplate \nstruct init_output_ports {\n template \n static OutputTuple call(graph& g, const std::tuple&) {\n return OutputTuple(Args(g)...);\n }\n}; // struct init_output_ports\n\n//! Implements methods for a function node that takes a type Input as input\n// and has a tuple of output ports specified.\ntemplate< typename Input, typename OutputPortSet, typename Policy, typename A>\nclass multifunction_input : public function_input_base > {\npublic:\n static const int N = std::tuple_size::value;\n typedef Input input_type;\n typedef OutputPortSet output_ports_type;\n typedef multifunction_body multifunction_body_type;\n typedef multifunction_input my_class;\n typedef function_input_base base_type;\n typedef function_input_queue input_queue_type;\n\n // constructor\n template\n multifunction_input(graph &g, size_t max_concurrency,Body& body, node_priority_t a_priority )\n : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type(), my_output_ports)))\n , my_body( new multifunction_body_leaf(body) )\n , my_init_body( new multifunction_body_leaf(body) )\n , my_output_ports(init_output_ports::call(g, my_output_ports)){\n }\n\n //! Copy constructor\n multifunction_input( const multifunction_input& src ) :\n base_type(src),\n my_body( src.my_init_body->clone() ),\n my_init_body(src.my_init_body->clone() ),\n my_output_ports( init_output_ports::call(src.my_graph_ref, my_output_ports) ) {\n }\n\n ~multifunction_input() {\n delete my_body;\n delete my_init_body;\n }\n\n template< typename Body >\n Body copy_function_object() {\n multifunction_body_type &body_ref = *this->my_body;\n return *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr());\n }\n\n // for multifunction nodes we do not have a single successor as such. So we just tell\n // the task we were successful.\n //TODO: consider moving common parts with implementation in function_input into separate function\n graph_task* apply_body_impl_bypass( const input_type &i\n __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&) )\n {\n fgt_begin_body( my_body );\n (*my_body)(i, my_output_ports);\n fgt_end_body( my_body );\n graph_task* ttask = nullptr;\n if(base_type::my_max_concurrency != 0) {\n ttask = base_type::try_get_postponed_task(i);\n }\n return ttask ? ttask : SUCCESSFULLY_ENQUEUED;\n }\n\n output_ports_type &output_ports(){ return my_output_ports; }\n\nprotected:\n\n void reset(reset_flags f) {\n base_type::reset_function_input_base(f);\n if(f & rf_clear_edges)clear_element::clear_this(my_output_ports);\n if(f & rf_reset_bodies) {\n multifunction_body_type* tmp = my_init_body->clone();\n delete my_body;\n my_body = tmp;\n }\n __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), \"multifunction_node reset failed\");\n }\n\n multifunction_body_type *my_body;\n multifunction_body_type *my_init_body;\n output_ports_type my_output_ports;\n\n}; // multifunction_input\n\n// template to refer to an output port of a multifunction_node\ntemplate\ntypename std::tuple_element::type &output_port(MOP &op) {\n return std::get(op.output_ports());\n}\n\ninline void check_task_and_spawn(graph& g, graph_task* t) {\n if (t && t != SUCCESSFULLY_ENQUEUED) {\n spawn_in_graph_arena(g, *t);\n }\n}\n\n// helper structs for split_node\ntemplate\nstruct emit_element {\n template\n static graph_task* emit_this(graph& g, const T &t, P &p) {\n // TODO: consider to collect all the tasks in task_list and spawn them all at once\n graph_task* last_task = std::get(p).try_put_task(std::get(t));\n check_task_and_spawn(g, last_task);\n return emit_element::emit_this(g,t,p);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static graph_task* emit_this(graph& g, const TupleType& t, PortsType& p,\n const message_metainfo& metainfo)\n {\n // TODO: consider to collect all the tasks in task_list and spawn them all at once\n graph_task* last_task = std::get(p).try_put_task(std::get(t), metainfo);\n check_task_and_spawn(g, last_task);\n return emit_element::emit_this(g, t, p, metainfo);\n }\n#endif\n};\n\ntemplate<>\nstruct emit_element<1> {\n template\n static graph_task* emit_this(graph& g, const T &t, P &p) {\n graph_task* last_task = std::get<0>(p).try_put_task(std::get<0>(t));\n check_task_and_spawn(g, last_task);\n return SUCCESSFULLY_ENQUEUED;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n template \n static graph_task* emit_this(graph& g, const TupleType& t, PortsType& ports,\n const message_metainfo& metainfo)\n {\n graph_task* last_task = std::get<0>(ports).try_put_task(std::get<0>(t), metainfo);\n check_task_and_spawn(g, last_task);\n return SUCCESSFULLY_ENQUEUED;\n }\n#endif\n};\n\n//! Implements methods for an executable node that takes continue_msg as input\ntemplate< typename Output, typename Policy>\nclass continue_input : public continue_receiver {\npublic:\n\n //! The input type of this receiver\n typedef continue_msg input_type;\n\n //! The output type of this receiver\n typedef Output output_type;\n typedef function_body function_body_type;\n typedef continue_input class_type;\n\n template< typename Body >\n continue_input( graph &g, Body& body, node_priority_t a_priority )\n : continue_receiver(/*number_of_predecessors=*/0, a_priority)\n , my_graph_ref(g)\n , my_body( new function_body_leaf< input_type, output_type, Body>(body) )\n , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) )\n { }\n\n template< typename Body >\n continue_input( graph &g, int number_of_predecessors,\n Body& body, node_priority_t a_priority )\n : continue_receiver( number_of_predecessors, a_priority )\n , my_graph_ref(g)\n , my_body( new function_body_leaf< input_type, output_type, Body>(body) )\n , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) )\n { }\n\n continue_input( const continue_input& src ) : continue_receiver(src),\n my_graph_ref(src.my_graph_ref),\n my_body( src.my_init_body->clone() ),\n my_init_body( src.my_init_body->clone() ) {}\n\n ~continue_input() {\n delete my_body;\n delete my_init_body;\n }\n\n template< typename Body >\n Body copy_function_object() {\n function_body_type &body_ref = *my_body;\n return dynamic_cast< function_body_leaf & >(body_ref).get_body();\n }\n\n void reset_receiver( reset_flags f) override {\n continue_receiver::reset_receiver(f);\n if(f & rf_reset_bodies) {\n function_body_type *tmp = my_init_body->clone();\n delete my_body;\n my_body = tmp;\n }\n }\n\nprotected:\n\n graph& my_graph_ref;\n function_body_type *my_body;\n function_body_type *my_init_body;\n\n virtual broadcast_cache &successors() = 0;\n\n friend class apply_body_task_bypass< class_type, continue_msg >;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n friend class apply_body_task_bypass< class_type, continue_msg, trackable_messages_graph_task >;\n#endif\n\n //! Applies the body to the provided input\n graph_task* apply_body_bypass( input_type __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) {\n // There is an extra copied needed to capture the\n // body execution without the try_put\n fgt_begin_body( my_body );\n output_type v = (*my_body)( continue_msg() );\n fgt_end_body( my_body );\n return successors().try_put_task( v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* execute(const message_metainfo& metainfo) override {\n#else\n graph_task* execute() override {\n#endif\n if(!is_graph_active(my_graph_ref)) {\n return nullptr;\n }\n#if _MSC_VER && !__INTEL_COMPILER\n// #pragma warning (push)\n// #pragma warning (disable: 4127) /* suppress conditional expression is constant */\n#endif\n if(has_policy::value) {\n#if _MSC_VER && !__INTEL_COMPILER\n// #pragma warning (pop)\n#endif\n return apply_body_bypass( continue_msg() __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) );\n }\n else {\n d1::small_object_allocator allocator{};\n graph_task* t = nullptr;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (!metainfo.empty()) {\n using task_type = apply_body_task_bypass;\n t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority, metainfo );\n } else\n#endif\n {\n using task_type = apply_body_task_bypass;\n t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority );\n }\n return t;\n }\n }\n\n graph& graph_reference() const override {\n return my_graph_ref;\n }\n}; // continue_input\n\n//! Implements methods for both executable and function nodes that puts Output to its successors\ntemplate< typename Output >\nclass function_output : public sender {\npublic:\n\n template friend struct clear_element;\n typedef Output output_type;\n typedef typename sender::successor_type successor_type;\n typedef broadcast_cache broadcast_cache_type;\n\n function_output(graph& g) : my_successors(this), my_graph_ref(g) {}\n function_output(const function_output& other) = delete;\n\n //! Adds a new successor to this node\n bool register_successor( successor_type &r ) override {\n successors().register_successor( r );\n return true;\n }\n\n //! Removes a successor from this node\n bool remove_successor( successor_type &r ) override {\n successors().remove_successor( r );\n return true;\n }\n\n broadcast_cache_type &successors() { return my_successors; }\n\n graph& graph_reference() const { return my_graph_ref; }\nprotected:\n broadcast_cache_type my_successors;\n graph& my_graph_ref;\n}; // function_output\n\ntemplate< typename Output >\nclass multifunction_output : public function_output {\npublic:\n typedef Output output_type;\n typedef function_output base_type;\n using base_type::my_successors;\n\n multifunction_output(graph& g) : base_type(g) {}\n multifunction_output(const multifunction_output& other) : base_type(other.my_graph_ref) {}\n\n bool try_put(const output_type &i) {\n graph_task *res = try_put_task(i);\n if( !res ) return false;\n if( res != SUCCESSFULLY_ENQUEUED ) {\n // wrapping in task_arena::execute() is not needed since the method is called from\n // inside task::execute()\n spawn_in_graph_arena(graph_reference(), *res);\n }\n return true;\n }\n\n using base_type::graph_reference;\n\nprotected:\n\n graph_task* try_put_task(const output_type &i) {\n return my_successors.try_put_task(i);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const output_type& i, const message_metainfo& metainfo) {\n return my_successors.try_put_task(i, metainfo);\n }\n#endif\n\n template friend struct emit_element;\n\n}; // multifunction_output\n\n//composite_node\ntemplate\nvoid add_nodes_impl(CompositeType*, bool) {}\n\ntemplate< typename CompositeType, typename NodeType1, typename... NodeTypes >\nvoid add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) {\n void *addr = const_cast(&n1);\n\n fgt_alias_port(c_node, addr, visible);\n add_nodes_impl(c_node, visible, n...);\n}\n\n#endif // __TBB__flow_graph_node_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_node_set_impl.h", "content": "/*\n Copyright (c) 2020-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_flow_graph_node_set_impl_H\n#define __TBB_flow_graph_node_set_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// Included in namespace tbb::detail::d2 (in flow_graph.h)\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n// Visual Studio 2019 reports an error while calling predecessor_selector::get and successor_selector::get\n// Seems like the well-formed expression in trailing decltype is treated as ill-formed\n// TODO: investigate problems with decltype in trailing return types or find the cross-platform solution\n#define __TBB_MSVC_DISABLE_TRAILING_DECLTYPE (_MSC_VER >= 1900)\n\nnamespace order {\nstruct undefined {};\nstruct following {};\nstruct preceding {};\n}\n\nclass get_graph_helper {\npublic:\n // TODO: consider making graph_reference() public and consistent interface to get a reference to the graph\n // and remove get_graph_helper\n template \n static graph& get(const T& object) {\n return get_impl(object, std::is_base_of());\n }\n\nprivate:\n // Get graph from the object of type derived from graph_node\n template \n static graph& get_impl(const T& object, std::true_type) {\n return static_cast(&object)->my_graph;\n }\n\n template \n static graph& get_impl(const T& object, std::false_type) {\n return object.graph_reference();\n }\n};\n\ntemplate\nstruct node_set {\n typedef Order order_type;\n\n std::tuple nodes;\n node_set(Nodes&... ns) : nodes(ns...) {}\n\n template \n node_set(const node_set& set) : nodes(set.nodes) {}\n\n graph& graph_reference() const {\n return get_graph_helper::get(std::get<0>(nodes));\n }\n};\n\nnamespace alias_helpers {\ntemplate using output_type = typename T::output_type;\ntemplate using output_ports_type = typename T::output_ports_type;\ntemplate using input_type = typename T::input_type;\ntemplate using input_ports_type = typename T::input_ports_type;\n} // namespace alias_helpers\n\ntemplate \nusing has_output_type = supports;\n\ntemplate \nusing has_input_type = supports;\n\ntemplate \nusing has_input_ports_type = supports;\n\ntemplate \nusing has_output_ports_type = supports;\n\ntemplate\nstruct is_sender : std::is_base_of, T> {};\n\ntemplate\nstruct is_receiver : std::is_base_of, T> {};\n\ntemplate \nstruct is_async_node : std::false_type {};\n\ntemplate \nstruct is_async_node> : std::true_type {};\n\ntemplate\nnode_set\nfollows(FirstPredecessor& first_predecessor, Predecessors&... predecessors) {\n static_assert((conjunction,\n has_output_type...>::value),\n \"Not all node's predecessors has output_type typedef\");\n static_assert((conjunction, is_sender...>::value),\n \"Not all node's predecessors are senders\");\n return node_set(first_predecessor, predecessors...);\n}\n\ntemplate\nnode_set\nfollows(node_set& predecessors_set) {\n static_assert((conjunction...>::value),\n \"Not all nodes in the set has output_type typedef\");\n static_assert((conjunction...>::value),\n \"Not all nodes in the set are senders\");\n return node_set(predecessors_set);\n}\n\ntemplate\nnode_set\nprecedes(FirstSuccessor& first_successor, Successors&... successors) {\n static_assert((conjunction,\n has_input_type...>::value),\n \"Not all node's successors has input_type typedef\");\n static_assert((conjunction, is_receiver...>::value),\n \"Not all node's successors are receivers\");\n return node_set(first_successor, successors...);\n}\n\ntemplate\nnode_set\nprecedes(node_set& successors_set) {\n static_assert((conjunction...>::value),\n \"Not all nodes in the set has input_type typedef\");\n static_assert((conjunction...>::value),\n \"Not all nodes in the set are receivers\");\n return node_set(successors_set);\n}\n\ntemplate \nnode_set\nmake_node_set(Node& first_node, Nodes&... nodes) {\n return node_set(first_node, nodes...);\n}\n\ntemplate\nclass successor_selector {\n template \n static auto get_impl(NodeType& node, std::true_type) -> decltype(input_port(node)) {\n return input_port(node);\n }\n\n template \n static NodeType& get_impl(NodeType& node, std::false_type) { return node; }\n\npublic:\n template \n#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE\n static auto& get(NodeType& node)\n#else\n static auto get(NodeType& node) -> decltype(get_impl(node, has_input_ports_type()))\n#endif\n {\n return get_impl(node, has_input_ports_type());\n }\n};\n\ntemplate\nclass predecessor_selector {\n template \n static auto internal_get(NodeType& node, std::true_type) -> decltype(output_port(node)) {\n return output_port(node);\n }\n\n template \n static NodeType& internal_get(NodeType& node, std::false_type) { return node;}\n\n template \n#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE\n static auto& get_impl(NodeType& node, std::false_type)\n#else\n static auto get_impl(NodeType& node, std::false_type) -> decltype(internal_get(node, has_output_ports_type()))\n#endif\n {\n return internal_get(node, has_output_ports_type());\n }\n\n template \n static AsyncNode& get_impl(AsyncNode& node, std::true_type) { return node; }\n\npublic:\n template \n#if __TBB_MSVC_DISABLE_TRAILING_DECLTYPE\n static auto& get(NodeType& node)\n#else\n static auto get(NodeType& node) -> decltype(get_impl(node, is_async_node()))\n#endif\n {\n return get_impl(node, is_async_node());\n }\n};\n\ntemplate\nclass make_edges_helper {\npublic:\n template\n static void connect_predecessors(PredecessorsTuple& predecessors, NodeType& node) {\n make_edge(std::get(predecessors), successor_selector::get(node));\n make_edges_helper::connect_predecessors(predecessors, node);\n }\n\n template\n static void connect_successors(NodeType& node, SuccessorsTuple& successors) {\n make_edge(predecessor_selector::get(node), std::get(successors));\n make_edges_helper::connect_successors(node, successors);\n }\n};\n\ntemplate<>\nstruct make_edges_helper<0> {\n template\n static void connect_predecessors(PredecessorsTuple& predecessors, NodeType& node) {\n make_edge(std::get<0>(predecessors), successor_selector<0>::get(node));\n }\n\n template\n static void connect_successors(NodeType& node, SuccessorsTuple& successors) {\n make_edge(predecessor_selector<0>::get(node), std::get<0>(successors));\n }\n};\n\n// TODO: consider adding an overload for making edges between node sets\ntemplate\nvoid make_edges(const node_set& s, NodeType& node) {\n const std::size_t SetSize = std::tuple_size::value;\n make_edges_helper::connect_predecessors(s.nodes, node);\n}\n\ntemplate \nvoid make_edges(NodeType& node, const node_set& s) {\n const std::size_t SetSize = std::tuple_size::value;\n make_edges_helper::connect_successors(node, s.nodes);\n}\n\ntemplate \nvoid make_edges_in_order(const node_set& ns, NodeType& node) {\n make_edges(ns, node);\n}\n\ntemplate \nvoid make_edges_in_order(const node_set& ns, NodeType& node) {\n make_edges(node, ns);\n}\n\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n#endif // __TBB_flow_graph_node_set_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_nodes_deduction.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_flow_graph_nodes_deduction_H\n#define __TBB_flow_graph_nodes_deduction_H\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate \nstruct declare_body_types {\n using input_type = Input;\n using output_type = Output;\n};\n\nstruct NoInputBody {};\n\ntemplate \nstruct declare_body_types {\n using output_type = Output;\n};\n\ntemplate struct body_types;\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nstruct body_types : declare_body_types {};\n\ntemplate \nusing input_t = typename body_types::input_type;\n\ntemplate \nusing output_t = typename body_types::output_type;\n\ntemplate \nauto decide_on_operator_overload(Output (T::*name)(const Input&) const)->decltype(name);\n\ntemplate \nauto decide_on_operator_overload(Output (T::*name)(const Input&))->decltype(name);\n\ntemplate \nauto decide_on_operator_overload(Output (T::*name)(Input&) const)->decltype(name);\n\ntemplate \nauto decide_on_operator_overload(Output (T::*name)(Input&))->decltype(name);\n\ntemplate \nauto decide_on_operator_overload(Output (*name)(const Input&))->decltype(name);\n\ntemplate \nauto decide_on_operator_overload(Output (*name)(Input&))->decltype(name);\n\ntemplate \ndecltype(decide_on_operator_overload(&Body::operator())) decide_on_callable_type(int);\n\ntemplate \ndecltype(decide_on_operator_overload(std::declval())) decide_on_callable_type(...);\n\n// Deduction guides for Flow Graph nodes\n\ntemplate \ninput_node(GraphOrSet&&, Body)\n->input_node(0))>>;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\ntemplate \nstruct decide_on_set;\n\ntemplate \nstruct decide_on_set> {\n using type = typename Node::output_type;\n};\n\ntemplate \nstruct decide_on_set> {\n using type = typename Node::input_type;\n};\n\ntemplate \nusing decide_on_set_t = typename decide_on_set>::type;\n\ntemplate \nbroadcast_node(const NodeSet&)\n->broadcast_node>;\n\ntemplate \nbuffer_node(const NodeSet&)\n->buffer_node>;\n\ntemplate \nqueue_node(const NodeSet&)\n->queue_node>;\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\ntemplate \nsequencer_node(GraphOrProxy&&, Sequencer)\n->sequencer_node(0))>>;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\ntemplate \npriority_queue_node(const NodeSet&, const Compare&)\n->priority_queue_node, Compare>;\n\ntemplate \npriority_queue_node(const NodeSet&)\n->priority_queue_node, std::less>>;\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\ntemplate \nstruct join_key {\n using type = Key;\n};\n\ntemplate \nstruct join_key {\n using type = T&;\n};\n\ntemplate \nusing join_key_t = typename join_key::type;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\ntemplate \njoin_node(const node_set&, Policy)\n->join_node,\n Policy>;\n\ntemplate \njoin_node(const node_set&, Policy)\n->join_node;\n\ntemplate \njoin_node(const node_set)\n->join_node,\n queueing>;\n\ntemplate \njoin_node(const node_set)\n->join_node;\n#endif\n\ntemplate \njoin_node(GraphOrProxy&&, Body, Bodies...)\n->join_node(0))>,\n input_t(0))>...>,\n key_matching(0))>>>>;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\ntemplate \nindexer_node(const node_set&)\n->indexer_node;\n#endif\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\ntemplate \nlimiter_node(const NodeSet&, size_t)\n->limiter_node>;\n\ntemplate \nsplit_node(const node_set&)\n->split_node;\n\ntemplate \nsplit_node(const node_set&)\n->split_node>;\n\n#endif\n\ntemplate \nfunction_node(GraphOrSet&&,\n size_t, Body,\n Policy, node_priority_t = no_priority)\n->function_node(0))>,\n output_t(0))>,\n Policy>;\n\ntemplate \nfunction_node(GraphOrSet&&, size_t,\n Body, node_priority_t = no_priority)\n->function_node(0))>,\n output_t(0))>,\n queueing>;\n\ntemplate \nstruct continue_output {\n using type = Output;\n};\n\ntemplate <>\nstruct continue_output {\n using type = continue_msg;\n};\n\ntemplate \nusing continue_output_t = typename continue_output::type;\n\ntemplate \ncontinue_node(GraphOrSet&&, Body,\n Policy, node_priority_t = no_priority)\n->continue_node>,\n Policy>;\n\ntemplate \ncontinue_node(GraphOrSet&&,\n int, Body,\n Policy, node_priority_t = no_priority)\n->continue_node>,\n Policy>;\n\ntemplate \ncontinue_node(GraphOrSet&&,\n Body, node_priority_t = no_priority)\n->continue_node>, Policy>;\n\ntemplate \ncontinue_node(GraphOrSet&&, int,\n Body, node_priority_t = no_priority)\n->continue_node>,\n Policy>;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\ntemplate \noverwrite_node(const NodeSet&)\n->overwrite_node>;\n\ntemplate \nwrite_once_node(const NodeSet&)\n->write_once_node>;\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n} // namespace d2\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\n#endif // __TBB_flow_graph_nodes_deduction_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// a hash table buffer that can expand, and can support as many deletions as\n// additions, list-based, with elements of list held in array (for destruction\n// management), multiplicative hashing (like ets). No synchronization built-in.\n//\n\n#ifndef __TBB__flow_graph_hash_buffer_impl_H\n#define __TBB__flow_graph_hash_buffer_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included in namespace tbb::flow::interfaceX::internal\n\n// elements in the table are a simple list; we need pointer to next element to\n// traverse the chain\n\ntemplate \nstruct hash_buffer_element : public aligned_pair {\n using key_type = Key;\n using value_type = ValueType;\n\n value_type* get_value_ptr() { return reinterpret_cast(this->first); }\n hash_buffer_element* get_next() { return reinterpret_cast(this->second); }\n void set_next(hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); }\n\n void create_element(const value_type& v) {\n ::new(this->first) value_type(v);\n }\n\n void create_element(hash_buffer_element&& other) {\n ::new(this->first) value_type(std::move(*other.get_value_ptr()));\n }\n\n void destroy_element() {\n get_value_ptr()->~value_type();\n }\n};\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\ntemplate \nstruct metainfo_hash_buffer_element : public aligned_triple {\n using key_type = Key;\n using value_type = ValueType;\n\n value_type* get_value_ptr() { return reinterpret_cast(this->first); }\n metainfo_hash_buffer_element* get_next() {\n return reinterpret_cast(this->second);\n }\n void set_next(metainfo_hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); }\n message_metainfo& get_metainfo() { return this->third; }\n\n void create_element(const value_type& v, const message_metainfo& metainfo) {\n __TBB_ASSERT(this->third.empty(), nullptr);\n ::new(this->first) value_type(v);\n this->third = metainfo;\n\n for (auto waiter : metainfo.waiters()) {\n waiter->reserve(1);\n }\n }\n\n void create_element(metainfo_hash_buffer_element&& other) {\n __TBB_ASSERT(this->third.empty(), nullptr);\n ::new(this->first) value_type(std::move(*other.get_value_ptr()));\n this->third = std::move(other.get_metainfo());\n }\n\n void destroy_element() {\n get_value_ptr()->~value_type();\n\n for (auto waiter : get_metainfo().waiters()) {\n waiter->release(1);\n }\n get_metainfo() = message_metainfo{};\n }\n};\n#endif\n\ntemplate\n <\n typename ElementType,\n typename ValueToKey, // abstract method that returns \"const Key\" or \"const Key&\" given ValueType\n typename HashCompare, // has hash and equal\n typename Allocator=tbb::cache_aligned_allocator\n >\nclass hash_buffer_impl : public HashCompare {\npublic:\n static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table\n typedef typename ElementType::key_type key_type;\n typedef typename ElementType::value_type value_type;\n typedef ElementType element_type;\n typedef value_type *pointer_type;\n typedef element_type *list_array_type; // array we manage manually\n typedef list_array_type *pointer_array_type;\n typedef typename std::allocator_traits::template rebind_alloc pointer_array_allocator_type;\n typedef typename std::allocator_traits::template rebind_alloc elements_array_allocator;\n typedef typename std::decay::type Knoref;\n\nprivate:\n ValueToKey *my_key;\n size_t my_size;\n size_t nelements;\n pointer_array_type pointer_array; // pointer_array[my_size]\n list_array_type elements_array; // elements_array[my_size / 2]\n element_type* free_list;\n\n size_t mask() { return my_size - 1; }\n\n void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) {\n for(size_t i=0; i < sz - 1; ++i ) { // construct free list\n la[i].set_next(&(la[i + 1]));\n }\n la[sz - 1].set_next(nullptr);\n *p_free_list = (element_type *)&(la[0]);\n }\n\n // cleanup for exceptions\n struct DoCleanup {\n pointer_array_type *my_pa;\n list_array_type *my_elements;\n size_t my_size;\n\n DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) :\n my_pa(&pa), my_elements(&my_els), my_size(sz) { }\n ~DoCleanup() {\n if(my_pa) {\n size_t dont_care = 0;\n internal_free_buffer(*my_pa, *my_elements, my_size, dont_care);\n }\n }\n };\n\n // exception-safety requires we do all the potentially-throwing operations first\n void grow_array() {\n size_t new_size = my_size*2;\n size_t new_nelements = nelements; // internal_free_buffer zeroes this\n list_array_type new_elements_array = nullptr;\n pointer_array_type new_pointer_array = nullptr;\n list_array_type new_free_list = nullptr;\n {\n DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size);\n new_elements_array = elements_array_allocator().allocate(my_size);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (std::size_t i = 0; i < my_size; ++i) {\n ::new(new_elements_array + i) element_type();\n }\n#endif\n new_pointer_array = pointer_array_allocator_type().allocate(new_size);\n for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = nullptr;\n set_up_free_list(&new_free_list, new_elements_array, my_size );\n\n for(size_t i=0; i < my_size; ++i) {\n for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->get_next())) {\n internal_insert_with_key(new_pointer_array, new_size, new_free_list, std::move(*op));\n }\n }\n my_cleanup.my_pa = nullptr;\n my_cleanup.my_elements = nullptr;\n }\n\n internal_free_buffer(pointer_array, elements_array, my_size, nelements);\n free_list = new_free_list;\n pointer_array = new_pointer_array;\n elements_array = new_elements_array;\n my_size = new_size;\n nelements = new_nelements;\n }\n\n // v should have perfect forwarding if std::move implemented.\n // we use this method to move elements in grow_array, so can't use class fields\n template \n const value_type& get_value_from_pack(const Value& value, const Args&...) {\n return value;\n }\n\n template \n const value_type& get_value_from_pack(Element&& element) {\n return *(element.get_value_ptr());\n }\n\n template \n void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list,\n Args&&... args) {\n size_t l_mask = p_sz-1;\n __TBB_ASSERT(my_key, \"Error: value-to-key functor not provided\");\n size_t h = this->hash(tbb::detail::invoke(*my_key, get_value_from_pack(args...))) & l_mask;\n __TBB_ASSERT(p_free_list, \"Error: free list not set up.\");\n element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->get_next());\n my_elem->create_element(std::forward(args)...);\n my_elem->set_next(p_pointer_array[h]);\n p_pointer_array[h] = my_elem;\n }\n\n void internal_initialize_buffer() {\n pointer_array = pointer_array_allocator_type().allocate(my_size);\n for(size_t i = 0; i < my_size; ++i) pointer_array[i] = nullptr;\n elements_array = elements_array_allocator().allocate(my_size / 2);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (std::size_t i = 0; i < my_size / 2; ++i) {\n ::new(elements_array + i) element_type();\n }\n#endif\n set_up_free_list(&free_list, elements_array, my_size / 2);\n }\n\n // made static so an enclosed class can use to properly dispose of the internals\n static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) {\n if(pa) {\n for(size_t i = 0; i < sz; ++i ) {\n element_type *p_next;\n for( element_type *p = pa[i]; p; p = p_next) {\n p_next = p->get_next();\n p->destroy_element();\n }\n }\n pointer_array_allocator_type().deallocate(pa, sz);\n pa = nullptr;\n }\n // Separate test (if allocation of pa throws, el may be allocated.\n // but no elements will be constructed.)\n if(el) {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (std::size_t i = 0; i < sz / 2; ++i) {\n (el + i)->~element_type();\n }\n#endif\n elements_array_allocator().deallocate(el, sz / 2);\n el = nullptr;\n }\n sz = INITIAL_SIZE;\n ne = 0;\n }\n\npublic:\n hash_buffer_impl() : my_key(nullptr), my_size(INITIAL_SIZE), nelements(0) {\n internal_initialize_buffer();\n }\n\n ~hash_buffer_impl() {\n internal_free_buffer(pointer_array, elements_array, my_size, nelements);\n delete my_key;\n my_key = nullptr;\n }\n hash_buffer_impl(const hash_buffer_impl&) = delete;\n hash_buffer_impl& operator=(const hash_buffer_impl&) = delete;\n\n void reset() {\n internal_free_buffer(pointer_array, elements_array, my_size, nelements);\n internal_initialize_buffer();\n }\n\n // Take ownership of func object allocated with new.\n // This method is only used internally, so can't be misused by user.\n void set_key_func(ValueToKey *vtk) { my_key = vtk; }\n // pointer is used to clone()\n ValueToKey* get_key_func() { return my_key; }\n\n template \n bool insert_with_key(const value_type &v, Args&&... args) {\n element_type* p = nullptr;\n __TBB_ASSERT(my_key, \"Error: value-to-key functor not provided\");\n if(find_element_ref_with_key(tbb::detail::invoke(*my_key, v), p)) {\n p->destroy_element();\n p->create_element(v, std::forward(args)...);\n return false;\n }\n ++nelements;\n if(nelements*2 > my_size) grow_array();\n internal_insert_with_key(pointer_array, my_size, free_list, v, std::forward(args)...);\n return true;\n }\n\n bool find_element_ref_with_key(const Knoref& k, element_type*& v) {\n size_t i = this->hash(k) & mask();\n for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->get_next())) {\n __TBB_ASSERT(my_key, \"Error: value-to-key functor not provided\");\n if(this->equal(tbb::detail::invoke(*my_key, *p->get_value_ptr()), k)) {\n v = p;\n return true;\n }\n }\n return false;\n }\n\n // returns true and sets v to array element if found, else returns false.\n bool find_ref_with_key(const Knoref& k, pointer_type &v) {\n element_type* element_ptr = nullptr;\n bool res = find_element_ref_with_key(k, element_ptr);\n v = element_ptr->get_value_ptr();\n return res;\n }\n\n bool find_with_key( const Knoref& k, value_type &v) {\n value_type *p;\n if(find_ref_with_key(k, p)) {\n v = *p;\n return true;\n }\n else\n return false;\n }\n\n void delete_with_key(const Knoref& k) {\n size_t h = this->hash(k) & mask();\n element_type* prev = nullptr;\n for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->get_next())) {\n value_type *vp = p->get_value_ptr();\n __TBB_ASSERT(my_key, \"Error: value-to-key functor not provided\");\n if(this->equal(tbb::detail::invoke(*my_key, *vp), k)) {\n p->destroy_element();\n if(prev) prev->set_next(p->get_next());\n else pointer_array[h] = (element_type *)(p->get_next());\n p->set_next(free_list);\n free_list = p;\n --nelements;\n return;\n }\n }\n __TBB_ASSERT(false, \"key not found for delete\");\n }\n};\n\ntemplate\n <\n typename Key, // type of key within ValueType\n typename ValueType,\n typename ValueToKey, // abstract method that returns \"const Key\" or \"const Key&\" given ValueType\n typename HashCompare, // has hash and equal\n typename Allocator=tbb::cache_aligned_allocator>\n >\nusing hash_buffer = hash_buffer_impl,\n ValueToKey, HashCompare, Allocator>;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\ntemplate\n <\n typename Key, // type of key within ValueType\n typename ValueType,\n typename ValueToKey, // abstract method that returns \"const Key\" or \"const Key&\" given ValueType\n typename HashCompare, // has hash and equal\n typename Allocator=tbb::cache_aligned_allocator>\n >\nstruct metainfo_hash_buffer : public hash_buffer_impl,\n ValueToKey, HashCompare, Allocator>\n{\nprivate:\n using base_type = hash_buffer_impl,\n ValueToKey, HashCompare, Allocator>;\npublic:\n bool find_with_key(const typename base_type::Knoref& k,\n typename base_type::value_type& v, message_metainfo& metainfo)\n {\n typename base_type::element_type* p = nullptr;\n bool result = this->find_element_ref_with_key(k, p);\n if (result) {\n v = *(p->get_value_ptr());\n metainfo = p->get_metainfo();\n }\n return result;\n }\n};\n#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n#endif // __TBB__flow_graph_hash_buffer_impl_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_trace_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _FGT_GRAPH_TRACE_IMPL_H\n#define _FGT_GRAPH_TRACE_IMPL_H\n\n#include \"../profiling.h\"\n#if (_MSC_VER >= 1900)\n #include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\ntemplate< typename T > class sender;\ntemplate< typename T > class receiver;\n\n#if TBB_USE_PROFILING_TOOLS\n #if __TBB_FLOW_TRACE_CODEPTR\n #if (_MSC_VER >= 1900)\n #define CODEPTR() (_ReturnAddress())\n #elif __TBB_GCC_VERSION >= 40800\n #define CODEPTR() ( __builtin_return_address(0))\n #else\n #define CODEPTR() nullptr\n #endif\n #else\n #define CODEPTR() nullptr\n #endif /* __TBB_FLOW_TRACE_CODEPTR */\n\nstatic inline void fgt_alias_port(void *node, void *p, bool visible) {\n if(visible)\n itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE );\n else\n itt_relation_add( d1::ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE );\n}\n\nstatic inline void fgt_composite ( void* codeptr, void *node, void *graph ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE );\n suppress_unused_warning( codeptr );\n#if __TBB_FLOW_TRACE_CODEPTR\n if (codeptr != nullptr) {\n register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr);\n }\n#endif\n}\n\nstatic inline void fgt_internal_alias_input_port( void *node, void *p, string_resource_index name_index ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index );\n itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT );\n}\n\nstatic inline void fgt_internal_alias_output_port( void *node, void *p, string_resource_index name_index ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index );\n itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT );\n}\n\ntemplate\nvoid alias_input_port(void *node, receiver* port, string_resource_index name_index) {\n // TODO: Make fgt_internal_alias_input_port a function template?\n fgt_internal_alias_input_port( node, port, name_index);\n}\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_input_alias_helper {\n static void alias_port( void *node, PortsTuple &ports ) {\n alias_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) );\n fgt_internal_input_alias_helper::alias_port( node, ports );\n }\n};\n\ntemplate < typename PortsTuple >\nstruct fgt_internal_input_alias_helper {\n static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { }\n};\n\ntemplate\nvoid alias_output_port(void *node, sender* port, string_resource_index name_index) {\n // TODO: Make fgt_internal_alias_output_port a function template?\n fgt_internal_alias_output_port( node, static_cast(port), name_index);\n}\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_output_alias_helper {\n static void alias_port( void *node, PortsTuple &ports ) {\n alias_output_port( node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) );\n fgt_internal_output_alias_helper::alias_port( node, ports );\n }\n};\n\ntemplate < typename PortsTuple >\nstruct fgt_internal_output_alias_helper {\n static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) {\n }\n};\n\nstatic inline void fgt_internal_create_input_port( void *node, void *p, string_resource_index name_index ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index );\n}\n\nstatic inline void fgt_internal_create_output_port( void* codeptr, void *node, void *p, string_resource_index name_index ) {\n itt_make_task_group(d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index);\n suppress_unused_warning( codeptr );\n#if __TBB_FLOW_TRACE_CODEPTR\n if (codeptr != nullptr) {\n register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr);\n }\n#endif\n}\n\ntemplate\nvoid register_input_port(void *node, receiver* port, string_resource_index name_index) {\n // TODO: Make fgt_internal_create_input_port a function template?\n fgt_internal_create_input_port(node, static_cast(port), name_index);\n}\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_input_helper {\n static void register_port( void *node, PortsTuple &ports ) {\n register_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) );\n fgt_internal_input_helper::register_port( node, ports );\n }\n};\n\ntemplate < typename PortsTuple >\nstruct fgt_internal_input_helper {\n static void register_port( void *node, PortsTuple &ports ) {\n register_input_port( node, &(std::get<0>(ports)), FLOW_INPUT_PORT_0 );\n }\n};\n\ntemplate\nvoid register_output_port(void* codeptr, void *node, sender* port, string_resource_index name_index) {\n // TODO: Make fgt_internal_create_output_port a function template?\n fgt_internal_create_output_port( codeptr, node, static_cast(port), name_index);\n}\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_output_helper {\n static void register_port( void* codeptr, void *node, PortsTuple &ports ) {\n register_output_port( codeptr, node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) );\n fgt_internal_output_helper::register_port( codeptr, node, ports );\n }\n};\n\ntemplate < typename PortsTuple >\nstruct fgt_internal_output_helper {\n static void register_port( void* codeptr, void *node, PortsTuple &ports ) {\n register_output_port( codeptr, node, &(std::get<0>(ports)), FLOW_OUTPUT_PORT_0 );\n }\n};\n\ntemplate< typename NodeType >\nvoid fgt_multioutput_node_desc( const NodeType *node, const char *desc ) {\n void *addr = (void *)( static_cast< receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) );\n itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc );\n}\n\ntemplate< typename NodeType >\nvoid fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) {\n void *addr = const_cast(node);\n itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc );\n}\n\ntemplate< typename NodeType >\nstatic inline void fgt_node_desc( const NodeType *node, const char *desc ) {\n void *addr = (void *)( static_cast< sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) );\n itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc );\n}\n\nstatic inline void fgt_graph_desc( const void *g, const char *desc ) {\n void *addr = const_cast< void *>(g);\n itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_GRAPH, FLOW_OBJECT_NAME, desc );\n}\n\nstatic inline void fgt_body( void *node, void *body ) {\n itt_relation_add( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE );\n}\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multioutput_node(void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t );\n fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 );\n fgt_internal_output_helper::register_port(codeptr, input_port, ports );\n}\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multioutput_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports, void *body ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t );\n fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 );\n fgt_internal_output_helper::register_port( codeptr, input_port, ports );\n fgt_body( input_port, body );\n}\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multiinput_node( void* codeptr, string_resource_index t, void *g, PortsTuple &ports, void *output_port) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t );\n fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 );\n fgt_internal_input_helper::register_port( output_port, ports );\n}\n\nstatic inline void fgt_multiinput_multioutput_node( void* codeptr, string_resource_index t, void *n, void *g ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t );\n suppress_unused_warning( codeptr );\n#if __TBB_FLOW_TRACE_CODEPTR\n if (codeptr != nullptr) {\n register_node_addr(d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, CODE_ADDRESS, &codeptr);\n }\n#endif\n}\n\nstatic inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *output_port ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t );\n fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 );\n}\n\nstatic void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *output_port, void *body ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t );\n fgt_internal_create_output_port(codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 );\n fgt_body( output_port, body );\n}\n\nstatic inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port ) {\n fgt_node( codeptr, t, g, output_port );\n fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 );\n}\n\nstatic inline void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port, void *body ) {\n fgt_node_with_body( codeptr, t, g, output_port, body );\n fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 );\n}\n\n\nstatic inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *decrement_port, void *output_port ) {\n fgt_node( codeptr, t, g, input_port, output_port );\n fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 );\n}\n\nstatic inline void fgt_make_edge( void *output_port, void *input_port ) {\n itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT);\n}\n\nstatic inline void fgt_remove_edge( void *output_port, void *input_port ) {\n itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT);\n}\n\nstatic inline void fgt_graph( void *g ) {\n itt_make_task_group( d1::ITT_DOMAIN_FLOW, g, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_GRAPH );\n}\n\nstatic inline void fgt_begin_body( void *body ) {\n itt_task_begin( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, nullptr, FLOW_NULL, FLOW_BODY );\n}\n\nstatic inline void fgt_end_body( void * ) {\n itt_task_end( d1::ITT_DOMAIN_FLOW );\n}\n\nstatic inline void fgt_async_try_put_begin( void *node, void *port ) {\n itt_task_begin( d1::ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT );\n}\n\nstatic inline void fgt_async_try_put_end( void *, void * ) {\n itt_task_end( d1::ITT_DOMAIN_FLOW );\n}\n\nstatic inline void fgt_async_reserve( void *node, void *graph ) {\n itt_region_begin( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL );\n}\n\nstatic inline void fgt_async_commit( void *node, void * /*graph*/) {\n itt_region_end( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE );\n}\n\nstatic inline void fgt_reserve_wait( void *graph ) {\n itt_region_begin( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_NULL );\n}\n\nstatic inline void fgt_release_wait( void *graph ) {\n itt_region_end( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH );\n}\n\n#else // TBB_USE_PROFILING_TOOLS\n\n#define CODEPTR() nullptr\n\nstatic inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { }\n\nstatic inline void fgt_composite ( void* /*codeptr*/, void * /*node*/, void * /*graph*/ ) { }\n\nstatic inline void fgt_graph( void * /*g*/ ) { }\n\ntemplate< typename NodeType >\nstatic inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { }\n\ntemplate< typename NodeType >\nstatic inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { }\n\nstatic inline void fgt_graph_desc( const void * /*g*/, const char * /*desc*/ ) { }\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { }\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multioutput_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { }\n\ntemplate< int N, typename PortsTuple >\nstatic inline void fgt_multiinput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { }\n\nstatic inline void fgt_multiinput_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*node*/, void * /*graph*/ ) { }\n\nstatic inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { }\nstatic inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { }\n\nstatic inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { }\nstatic inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { }\n\nstatic inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { }\nstatic inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { }\n\nstatic inline void fgt_begin_body( void * /*body*/ ) { }\nstatic inline void fgt_end_body( void * /*body*/) { }\n\nstatic inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { }\nstatic inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { }\nstatic inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { }\nstatic inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { }\nstatic inline void fgt_reserve_wait( void * /*graph*/ ) { }\nstatic inline void fgt_release_wait( void * /*graph*/ ) { }\n\ntemplate< typename NodeType >\nvoid fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { }\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_input_alias_helper {\n static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { }\n};\n\ntemplate < typename PortsTuple, int N >\nstruct fgt_internal_output_alias_helper {\n static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { }\n};\n\n#endif // TBB_USE_PROFILING_TOOLS\n\n} // d2\n} // namespace detail\n} // namespace tbb\n\n#endif // _FGT_GRAPH_TRACE_IMPL_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_flow_graph_types_impl.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__flow_graph_types_impl_H\n#define __TBB__flow_graph_types_impl_H\n\n#ifndef __TBB_flow_graph_H\n#error Do not #include this internal file directly; use public TBB headers instead.\n#endif\n\n// included in namespace tbb::detail::d2\n\n// the change to key_matching (adding a K and KHash template parameter, making it a class)\n// means we have to pass this data to the key_matching_port. All the ports have only one\n// template parameter, so we have to wrap the following types in a trait:\n//\n// . K == key_type\n// . KHash == hash and compare for Key\n// . TtoK == function_body that given an object of T, returns its K\n// . T == type accepted by port, and stored in the hash table\n//\n// The port will have an additional parameter on node construction, which is a function_body\n// that accepts a const T& and returns a K which is the field in T which is its K.\ntemplate\nstruct KeyTrait {\n typedef Kp K;\n typedef Tp T;\n typedef type_to_key_function_body TtoK;\n typedef KHashp KHash;\n};\n\n// wrap each element of a tuple in a template, and make a tuple of the result.\ntemplate class PT, typename TypeTuple>\nstruct wrap_tuple_elements;\n\n// A wrapper that generates the traits needed for each port of a key-matching join,\n// and the type of the tuple of input ports.\ntemplate class PT, typename KeyTraits, typename TypeTuple>\nstruct wrap_key_tuple_elements;\n\ntemplate class PT, typename... Args>\nstruct wrap_tuple_elements >{\n typedef typename std::tuple... > type;\n};\n\ntemplate class PT, typename KeyTraits, typename... Args>\nstruct wrap_key_tuple_elements > {\n typedef typename KeyTraits::key_type K;\n typedef typename KeyTraits::hash_compare_type KHash;\n typedef typename std::tuple >... > type;\n};\n\ntemplate< int... S > class sequence {};\n\ntemplate< int N, int... S >\nstruct make_sequence : make_sequence < N - 1, N - 1, S... > {};\n\ntemplate< int... S >\nstruct make_sequence < 0, S... > {\n typedef sequence type;\n};\n\ntemplate struct alignment_of {\n typedef struct { char t; U padded; } test_alignment;\n static const size_t value = sizeof(test_alignment) - sizeof(U);\n};\n\ntemplate \nstruct max_alignment_helper;\n\ntemplate \nstruct max_alignment_helper {\n using type = typename max_alignment_helper::type>::type;\n};\n\ntemplate \nstruct max_alignment_helper {\n using type = typename std::conditional::type;\n};\n\ntemplate \nusing max_alignment_helper_t = typename max_alignment_helper::type;\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(push)\n// #pragma warning(disable: 4324) // warning C4324: structure was padded due to alignment specifier\n#endif\n\n// T1, T2 are actual types stored. The space defined for T1 in the type returned\n// is a char array of the correct size. Type T2 should be trivially-constructible,\n// T1 must be explicitly managed.\n\ntemplate \nstruct alignas(alignof(max_alignment_helper_t)) aligned_pair {\n char first[sizeof(T1)];\n T2 second;\n};\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\ntemplate \nstruct alignas(alignof(max_alignment_helper_t)) aligned_triple {\n char first[sizeof(T1)];\n T2 second;\n T3 third;\n};\n#endif\n\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(pop) // warning 4324 is back\n#endif\n\n// support for variant type\n// type we use when we're not storing a value\nstruct default_constructed { };\n\n// type which contains another type, tests for what type is contained, and references to it.\n// Wrapper\n// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace\n\n// struct to allow us to copy and test the type of objects\nstruct WrapperBase {\n virtual ~WrapperBase() {}\n virtual void CopyTo(void* /*newSpace*/) const = 0;\n};\n\n// Wrapper contains a T, with the ability to test what T is. The Wrapper can be\n// constructed from a T, can be copy-constructed from another Wrapper, and can be\n// examined via value(), but not modified.\ntemplate\nstruct Wrapper: public WrapperBase {\n typedef T value_type;\n typedef T* pointer_type;\nprivate:\n T value_space;\npublic:\n const value_type &value() const { return value_space; }\n\nprivate:\n Wrapper();\n\n // on exception will ensure the Wrapper will contain only a trivially-constructed object\n struct _unwind_space {\n pointer_type space;\n _unwind_space(pointer_type p) : space(p) {}\n ~_unwind_space() {\n if(space) (void) new (space) Wrapper(default_constructed());\n }\n };\npublic:\n explicit Wrapper( const T& other ) : value_space(other) { }\n explicit Wrapper(const Wrapper& other) = delete;\n\n void CopyTo(void* newSpace) const override {\n _unwind_space guard((pointer_type)newSpace);\n (void) new(newSpace) Wrapper(value_space);\n guard.space = nullptr;\n }\n ~Wrapper() { }\n};\n\n// specialization for array objects\ntemplate\nstruct Wrapper : public WrapperBase {\n typedef T value_type;\n typedef T* pointer_type;\n // space must be untyped.\n typedef T ArrayType[N];\nprivate:\n // The space is not of type T[N] because when copy-constructing, it would be\n // default-initialized and then copied to in some fashion, resulting in two\n // constructions and one destruction per element. If the type is char[ ], we\n // placement new into each element, resulting in one construction per element.\n static const size_t space_size = sizeof(ArrayType);\n char value_space[space_size];\n\n\n // on exception will ensure the already-built objects will be destructed\n // (the value_space is a char array, so it is already trivially-destructible.)\n struct _unwind_class {\n pointer_type space;\n int already_built;\n _unwind_class(pointer_type p) : space(p), already_built(0) {}\n ~_unwind_class() {\n if(space) {\n for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type();\n (void) new(space) Wrapper(default_constructed());\n }\n }\n };\npublic:\n const ArrayType &value() const {\n char *vp = const_cast(value_space);\n return reinterpret_cast(*vp);\n }\n\nprivate:\n Wrapper();\npublic:\n // have to explicitly construct because other decays to a const value_type*\n explicit Wrapper(const ArrayType& other) {\n _unwind_class guard((pointer_type)value_space);\n pointer_type vp = reinterpret_cast(&value_space);\n for(size_t i = 0; i < N; ++i ) {\n (void) new(vp++) value_type(other[i]);\n ++(guard.already_built);\n }\n guard.space = nullptr;\n }\n explicit Wrapper(const Wrapper& other) : WrapperBase() {\n // we have to do the heavy lifting to copy contents\n _unwind_class guard((pointer_type)value_space);\n pointer_type dp = reinterpret_cast(value_space);\n pointer_type sp = reinterpret_cast(const_cast(other.value_space));\n for(size_t i = 0; i < N; ++i, ++dp, ++sp) {\n (void) new(dp) value_type(*sp);\n ++(guard.already_built);\n }\n guard.space = nullptr;\n }\n\n void CopyTo(void* newSpace) const override {\n (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor\n }\n\n ~Wrapper() {\n // have to destroy explicitly in reverse order\n pointer_type vp = reinterpret_cast(&value_space);\n for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type();\n }\n};\n\n// given a tuple, return the type of the element that has the maximum alignment requirement.\n// Given a tuple and that type, return the number of elements of the object with the max\n// alignment requirement that is at least as big as the largest object in the tuple.\n\ntemplate struct pick_one;\ntemplate struct pick_one { typedef T1 type; };\ntemplate struct pick_one { typedef T2 type; };\n\ntemplate< template class Selector, typename T1, typename T2 >\nstruct pick_max {\n typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type;\n};\n\ntemplate struct size_of { static const int value = sizeof(T); };\n\ntemplate< size_t N, class Tuple, template class Selector > struct pick_tuple_max {\n typedef typename pick_tuple_max::type LeftMaxType;\n typedef typename std::tuple_element::type ThisType;\n typedef typename pick_max::type type;\n};\n\ntemplate< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> {\n typedef typename std::tuple_element<0, Tuple>::type type;\n};\n\n// is the specified type included in a tuple?\ntemplate\nstruct is_element_of {\n typedef typename std::tuple_element::type T_i;\n static const bool value = std::is_same::value || is_element_of::value;\n};\n\ntemplate\nstruct is_element_of {\n typedef typename std::tuple_element<0, Tuple>::type T_i;\n static const bool value = std::is_same::value;\n};\n\n// allow the construction of types that are listed tuple. If a disallowed type\n// construction is written, a method involving this type is created. The\n// type has no definition, so a syntax error is generated.\ntemplate struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple;\n\ntemplate struct do_if;\ntemplate\nstruct do_if {\n static void construct(void *mySpace, const T& x) {\n (void) new(mySpace) Wrapper(x);\n }\n};\ntemplate\nstruct do_if {\n static void construct(void * /*mySpace*/, const T& x) {\n // This method is instantiated when the type T does not match any of the\n // element types in the Tuple in variant.\n ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x);\n }\n};\n\n// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in\n// Wrapper, and how big Wrapper is.\n//\n// the object can only be tested for type, and a read-only reference can be fetched by cast_to().\n\nusing tbb::detail::punned_cast;\nstruct tagged_null_type {};\ntemplate\nclass tagged_msg {\n typedef std::tuple= 6\n , T5\n #endif\n #if __TBB_VARIADIC_MAX >= 7\n , T6\n #endif\n #if __TBB_VARIADIC_MAX >= 8\n , T7\n #endif\n #if __TBB_VARIADIC_MAX >= 9\n , T8\n #endif\n #if __TBB_VARIADIC_MAX >= 10\n , T9\n #endif\n > Tuple;\n\nprivate:\n class variant {\n static const size_t N = std::tuple_size::value;\n typedef typename pick_tuple_max::type AlignType;\n typedef typename pick_tuple_max::type MaxSizeType;\n static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1);\n static const size_t MaxNElements = MaxNBytes/sizeof(AlignType);\n typedef aligned_space SpaceType;\n SpaceType my_space;\n static const size_t MaxSize = sizeof(SpaceType);\n\n public:\n variant() { (void) new(&my_space) Wrapper(default_constructed()); }\n\n template\n variant( const T& x ) {\n do_if::value>::construct(&my_space,x);\n }\n\n variant(const variant& other) {\n const WrapperBase * h = punned_cast(&(other.my_space));\n h->CopyTo(&my_space);\n }\n\n // assignment must destroy and re-create the Wrapper type, as there is no way\n // to create a Wrapper-to-Wrapper assign even if we find they agree in type.\n void operator=( const variant& rhs ) {\n if(&rhs != this) {\n WrapperBase *h = punned_cast(&my_space);\n h->~WrapperBase();\n const WrapperBase *ch = punned_cast(&(rhs.my_space));\n ch->CopyTo(&my_space);\n }\n }\n\n template\n const U& variant_cast_to() const {\n const Wrapper *h = dynamic_cast*>(punned_cast(&my_space));\n if(!h) {\n throw_exception(exception_id::bad_tagged_msg_cast);\n }\n return h->value();\n }\n template\n bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != nullptr; }\n\n bool variant_is_default_constructed() const {return variant_is_a();}\n\n ~variant() {\n WrapperBase *h = punned_cast(&my_space);\n h->~WrapperBase();\n }\n }; //class variant\n\n TagType my_tag;\n variant my_msg;\n\npublic:\n tagged_msg(): my_tag(TagType(~0)), my_msg(){}\n\n template\n tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {}\n\n template\n tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {}\n\n void set_tag(TagType const &index) {my_tag = index;}\n TagType tag() const {return my_tag;}\n\n template\n const V& cast_to() const {return my_msg.template variant_cast_to();}\n\n template\n bool is_a() const {return my_msg.template variant_is_a();}\n\n bool is_default_constructed() const {return my_msg.variant_is_default_constructed();}\n}; //class tagged_msg\n\n// template to simplify cast and test for tagged_msg in template contexts\ntemplate\nconst V& cast_to(T const &t) { return t.template cast_to(); }\n\ntemplate\nbool is_a(T const &t) { return t.template is_a(); }\n\nenum op_stat { WAIT = 0, SUCCEEDED, FAILED };\n\n#endif /* __TBB__flow_graph_types_impl_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_hash_compare.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__hash_compare_H\n#define __TBB_detail__hash_compare_H\n\n#include \n\n#include \"_containers_helpers.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\ntemplate \nclass hash_compare {\n using is_transparent_hash = has_transparent_key_equal;\npublic:\n using hasher = Hash;\n using key_equal = typename is_transparent_hash::type;\n\n hash_compare() = default;\n hash_compare( hasher hash, key_equal equal ) : my_hasher(hash), my_equal(equal) {}\n\n std::size_t operator()( const Key& key ) const {\n return std::size_t(my_hasher(key));\n }\n\n bool operator()( const Key& key1, const Key& key2 ) const {\n return my_equal(key1, key2);\n }\n\n template ::type>\n std::size_t operator()( const K& key ) const {\n return std::size_t(my_hasher(key));\n }\n\n template ::type>\n bool operator()( const K1& key1, const K2& key2 ) const {\n return my_equal(key1, key2);\n }\n\n hasher hash_function() const {\n return my_hasher;\n }\n\n key_equal key_eq() const {\n return my_equal;\n }\n\n\nprivate:\n hasher my_hasher;\n key_equal my_equal;\n}; // class hash_compare\n\n//! hash_compare that is default argument for concurrent_hash_map\ntemplate \nclass tbb_hash_compare {\npublic:\n std::size_t hash( const Key& a ) const { return my_hash_func(a); }\n#if defined(_MSC_VER) && _MSC_VER <= 1900\n// #pragma warning (push)\n// MSVC 2015 throws a strange warning: 'std::size_t': forcing value to bool 'true' or 'false'\n// #pragma warning (disable: 4800)\n#endif\n bool equal( const Key& a, const Key& b ) const { return my_key_equal(a, b); }\n#if defined(_MSC_VER) && _MSC_VER <= 1900\n// #pragma warning (pop)\n#endif\nprivate:\n std::hash my_hash_func;\n std::equal_to my_key_equal;\n};\n\n} // namespace d1\n#if __TBB_CPP20_CONCEPTS_PRESENT\ninline namespace d0 {\n\ntemplate \nconcept hash_compare = std::copy_constructible &&\n requires( const std::remove_reference_t& hc, const Key& key1, const Key& key2 ) {\n { hc.hash(key1) } -> std::same_as;\n { hc.equal(key1, key2) } -> std::convertible_to;\n };\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n} // namespace detail\n} // namespace tbb\n\n#if TBB_DEFINE_STD_HASH_SPECIALIZATIONS\n\nnamespace std {\n\ntemplate \nstruct hash> {\npublic:\n std::size_t operator()( const std::pair& p ) const {\n return first_hash(p.first) ^ second_hash(p.second);\n }\n\nprivate:\n std::hash first_hash;\n std::hash second_hash;\n}; // struct hash\n\n// Apple clang and MSVC defines their own specializations for std::hash>\n#if !(_LIBCPP_VERSION) && !(_CPPLIB_VER)\n\ntemplate \nstruct hash> {\npublic:\n std::size_t operator()( const std::basic_string& s ) const {\n std::size_t h = 0;\n for ( const CharT* c = s.c_str(); *c; ++c ) {\n h = h * hash_multiplier ^ char_hash(*c);\n }\n return h;\n }\n\nprivate:\n static constexpr std::size_t hash_multiplier = tbb::detail::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value;\n\n std::hash char_hash;\n}; // struct hash\n\n#endif // !(_LIBCPP_VERSION || _CPPLIB_VER)\n\n} // namespace std\n\n#endif // TBB_DEFINE_STD_HASH_SPECIALIZATIONS\n\n#endif // __TBB_detail__hash_compare_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_intrusive_list_node.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_detail__intrusive_list_node_H\n#define _TBB_detail__intrusive_list_node_H\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! Data structure to be inherited by the types that can form intrusive lists.\n/** Intrusive list is formed by means of the member_intrusive_list template class.\n Note that type T must derive from intrusive_list_node either publicly or\n declare instantiation member_intrusive_list as a friend.\n This class implements a limited subset of std::list interface. **/\nstruct intrusive_list_node {\n intrusive_list_node* my_prev_node{};\n intrusive_list_node* my_next_node{};\n#if TBB_USE_ASSERT\n intrusive_list_node() { my_prev_node = my_next_node = this; }\n#endif /* TBB_USE_ASSERT */\n};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_detail__intrusive_list_node_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_machine.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__machine_H\n#define __TBB_detail__machine_H\n\n#include \"_config.h\"\n#include \"_assert.h\"\n\n#include \n#include \n#include \n#include \n\n#ifdef _WIN32\n#include \n#ifdef __TBBMALLOC_BUILD\n#define WIN32_LEAN_AND_MEAN\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif\n#include // SwitchToThread()\n#endif\n#ifdef _MSC_VER\n#if __TBB_x86_64 || __TBB_x86_32\n#pragma intrinsic(__rdtsc)\n#endif\n#endif\n#endif\n#if __TBB_x86_64 || __TBB_x86_32\n#include // _mm_pause\n#endif\n#if (_WIN32)\n#include // _control87\n#endif\n\n#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN\n#include // sched_yield\n#else\n#include // std::this_thread::yield()\n#endif\n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n//--------------------------------------------------------------------------------------------------\n// Yield implementation\n//--------------------------------------------------------------------------------------------------\n\n#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN\nstatic inline void yield() {\n int err = sched_yield();\n __TBB_ASSERT_EX(err == 0, \"sched_yield has failed\");\n}\n#elif __TBBMALLOC_BUILD && _WIN32\n// Use Windows API for yield in tbbmalloc to avoid dependency on C++ runtime with some implementations.\nstatic inline void yield() {\n SwitchToThread();\n}\n#else\nusing std::this_thread::yield;\n#endif\n\n//--------------------------------------------------------------------------------------------------\n// atomic_fence_seq_cst implementation\n//--------------------------------------------------------------------------------------------------\n\nstatic inline void atomic_fence_seq_cst() {\n#if (__TBB_x86_64 || __TBB_x86_32) && defined(__GNUC__) && __GNUC__ < 11\n unsigned char dummy = 0u;\n __asm__ __volatile__ (\"lock; notb %0\" : \"+m\" (dummy) :: \"memory\");\n#else\n std::atomic_thread_fence(std::memory_order_seq_cst);\n#endif\n}\n\n//--------------------------------------------------------------------------------------------------\n// Pause implementation\n//--------------------------------------------------------------------------------------------------\n\nstatic inline void machine_pause(int32_t delay) {\n#if __TBB_x86_64 || __TBB_x86_32\n while (delay-- > 0) { _mm_pause(); }\n#elif __ARM_ARCH_7A__ || __aarch64__\n while (delay-- > 0) { __asm__ __volatile__(\"isb sy\" ::: \"memory\"); }\n#else /* Generic */\n (void)delay; // suppress without including _template_helpers.h\n yield();\n#endif\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// tbb::detail::log2() implementation\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// TODO: Use log2p1() function that will be available in C++20 standard\n\n#if defined(__GNUC__) || defined(__clang__)\nnamespace gnu_builtins {\n inline uintptr_t clz(unsigned int x) { return static_cast(__builtin_clz(x)); }\n inline uintptr_t clz(unsigned long int x) { return static_cast(__builtin_clzl(x)); }\n inline uintptr_t clz(unsigned long long int x) { return static_cast(__builtin_clzll(x)); }\n}\n#elif defined(_MSC_VER)\n#pragma intrinsic(__TBB_W(_BitScanReverse))\nnamespace msvc_intrinsics {\n static inline uintptr_t bit_scan_reverse(uintptr_t i) {\n unsigned long j;\n __TBB_W(_BitScanReverse)( &j, i );\n return j;\n }\n}\n#endif\n\ntemplate \nconstexpr std::uintptr_t number_of_bits() {\n return sizeof(T) * CHAR_BIT;\n}\n\n// logarithm is the index of the most significant non-zero bit\nstatic inline uintptr_t machine_log2(uintptr_t x) {\n#if defined(__GNUC__) || defined(__clang__)\n // If P is a power of 2 and x() - 1) ^ gnu_builtins::clz(x);\n#elif defined(_MSC_VER)\n return msvc_intrinsics::bit_scan_reverse(x);\n#elif __i386__ || __i386 /*for Sun OS*/ || __MINGW32__\n uintptr_t j, i = x;\n __asm__(\"bsr %1,%0\" : \"=r\"(j) : \"r\"(i));\n return j;\n#elif __powerpc__ || __POWERPC__\n #if __TBB_WORDSIZE==8\n __asm__ __volatile__ (\"cntlzd %0,%0\" : \"+r\"(x));\n return 63 - static_cast(x);\n #else\n __asm__ __volatile__ (\"cntlzw %0,%0\" : \"+r\"(x));\n return 31 - static_cast(x);\n #endif /*__TBB_WORDSIZE*/\n#elif __sparc\n uint64_t count;\n // one hot encode\n x |= (x >> 1);\n x |= (x >> 2);\n x |= (x >> 4);\n x |= (x >> 8);\n x |= (x >> 16);\n x |= (x >> 32);\n // count 1's\n __asm__ (\"popc %1, %0\" : \"=r\"(count) : \"r\"(x) );\n return count - 1;\n#else\n intptr_t result = 0;\n\n if( sizeof(x) > 4 && (uintptr_t tmp = x >> 32) ) { x = tmp; result += 32; }\n if( uintptr_t tmp = x >> 16 ) { x = tmp; result += 16; }\n if( uintptr_t tmp = x >> 8 ) { x = tmp; result += 8; }\n if( uintptr_t tmp = x >> 4 ) { x = tmp; result += 4; }\n if( uintptr_t tmp = x >> 2 ) { x = tmp; result += 2; }\n\n return (x & 2) ? result + 1 : result;\n#endif\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// tbb::detail::reverse_bits() implementation\n////////////////////////////////////////////////////////////////////////////////////////////////////\n#if TBB_USE_CLANG_BITREVERSE_BUILTINS\nnamespace llvm_builtins {\n inline uint8_t builtin_bitreverse(uint8_t x) { return __builtin_bitreverse8 (x); }\n inline uint16_t builtin_bitreverse(uint16_t x) { return __builtin_bitreverse16(x); }\n inline uint32_t builtin_bitreverse(uint32_t x) { return __builtin_bitreverse32(x); }\n inline uint64_t builtin_bitreverse(uint64_t x) { return __builtin_bitreverse64(x); }\n}\n#else // generic\ntemplate\nstruct reverse {\n static const T byte_table[256];\n};\n\ntemplate\nconst T reverse::byte_table[256] = {\n 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,\n 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,\n 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,\n 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,\n 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,\n 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,\n 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,\n 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,\n 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,\n 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,\n 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,\n 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,\n 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,\n 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,\n 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,\n 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF\n};\n\ninline unsigned char reverse_byte(unsigned char src) {\n return reverse::byte_table[src];\n}\n#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS\n\ntemplate\nT machine_reverse_bits(T src) {\n#if TBB_USE_CLANG_BITREVERSE_BUILTINS\n return builtin_bitreverse(fixed_width_cast(src));\n#else /* Generic */\n T dst;\n unsigned char *original = reinterpret_cast(&src);\n unsigned char *reversed = reinterpret_cast(&dst);\n\n for ( int i = sizeof(T) - 1; i >= 0; i-- ) {\n reversed[i] = reverse_byte( original[sizeof(T) - i - 1] );\n }\n\n return dst;\n#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS\n}\n\n} // inline namespace d0\n\nnamespace d1 {\n\n#if (_WIN32)\n// API to retrieve/update FPU control setting\n#define __TBB_CPU_CTL_ENV_PRESENT 1\nstruct cpu_ctl_env {\n unsigned int x87cw{};\n#if (__TBB_x86_64)\n // Changing the infinity mode or the floating-point precision is not supported on x64.\n // The attempt causes an assertion. See\n // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/control87-controlfp-control87-2\n static constexpr unsigned int X87CW_CONTROL_MASK = _MCW_DN | _MCW_EM | _MCW_RC;\n#else\n static constexpr unsigned int X87CW_CONTROL_MASK = ~0U;\n#endif\n#if (__TBB_x86_32 || __TBB_x86_64)\n unsigned int mxcsr{};\n static constexpr unsigned int MXCSR_CONTROL_MASK = ~0x3fu; /* all except last six status bits */\n#endif\n\n bool operator!=( const cpu_ctl_env& ctl ) const {\n return\n#if (__TBB_x86_32 || __TBB_x86_64)\n mxcsr != ctl.mxcsr ||\n#endif\n x87cw != ctl.x87cw;\n }\n void get_env() {\n x87cw = _control87(0, 0);\n#if (__TBB_x86_32 || __TBB_x86_64)\n mxcsr = _mm_getcsr();\n#endif\n }\n void set_env() const {\n _control87(x87cw, X87CW_CONTROL_MASK);\n#if (__TBB_x86_32 || __TBB_x86_64)\n _mm_setcsr(mxcsr & MXCSR_CONTROL_MASK);\n#endif\n }\n};\n#elif (__TBB_x86_32 || __TBB_x86_64)\n// API to retrieve/update FPU control setting\n#define __TBB_CPU_CTL_ENV_PRESENT 1\nstruct cpu_ctl_env {\n int mxcsr{};\n short x87cw{};\n static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */\n\n bool operator!=(const cpu_ctl_env& ctl) const {\n return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw;\n }\n void get_env() {\n __asm__ __volatile__(\n \"stmxcsr %0\\n\\t\"\n \"fstcw %1\"\n : \"=m\"(mxcsr), \"=m\"(x87cw)\n );\n mxcsr &= MXCSR_CONTROL_MASK;\n }\n void set_env() const {\n __asm__ __volatile__(\n \"ldmxcsr %0\\n\\t\"\n \"fldcw %1\"\n : : \"m\"(mxcsr), \"m\"(x87cw)\n );\n }\n};\n#endif\n\n} // namespace d1\n\n} // namespace detail\n} // namespace tbb\n\n#if !__TBB_CPU_CTL_ENV_PRESENT\n#include \n\n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nvoid* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size);\nvoid __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p);\n} // namespace r1\n\nnamespace d1 {\n\nclass cpu_ctl_env {\n fenv_t *my_fenv_ptr;\npublic:\n cpu_ctl_env() : my_fenv_ptr(nullptr) {}\n ~cpu_ctl_env() {\n if ( my_fenv_ptr )\n r1::cache_aligned_deallocate( (void*)my_fenv_ptr );\n }\n // It is possible not to copy memory but just to copy pointers but the following issues should be addressed:\n // 1. The arena lifetime and the context lifetime are independent;\n // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside\n // dispatch loop may become invalid.\n // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation\n // with a platform specific implementation.\n cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(nullptr) {\n *this = src;\n }\n cpu_ctl_env& operator=( const cpu_ctl_env &src ) {\n __TBB_ASSERT( src.my_fenv_ptr, nullptr);\n if ( !my_fenv_ptr )\n my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t));\n *my_fenv_ptr = *src.my_fenv_ptr;\n return *this;\n }\n bool operator!=( const cpu_ctl_env &ctl ) const {\n __TBB_ASSERT( my_fenv_ptr, \"cpu_ctl_env is not initialized.\" );\n __TBB_ASSERT( ctl.my_fenv_ptr, \"cpu_ctl_env is not initialized.\" );\n return std::memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) );\n }\n void get_env () {\n if ( !my_fenv_ptr )\n my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t));\n fegetenv( my_fenv_ptr );\n }\n const cpu_ctl_env& set_env () const {\n __TBB_ASSERT( my_fenv_ptr, \"cpu_ctl_env is not initialized.\" );\n fesetenv( my_fenv_ptr );\n return *this;\n }\n};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif /* !__TBB_CPU_CTL_ENV_PRESENT */\n\n#endif // __TBB_detail__machine_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_mutex_common.h", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__mutex_common_H\n#define __TBB_detail__mutex_common_H\n\n#include \"_config.h\"\n#include \"_utils.h\"\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\ntemplate \nconcept mutex_scoped_lock = std::default_initializable &&\n std::constructible_from &&\n requires( Lock& lock, Mutex& mutex ) {\n lock.acquire(mutex);\n { lock.try_acquire(mutex) } -> adaptive_same_as;\n lock.release();\n };\n\ntemplate \nconcept rw_mutex_scoped_lock = mutex_scoped_lock &&\n std::constructible_from &&\n requires( Lock& lock, Mutex& mutex ) {\n lock.acquire(mutex, false);\n { lock.try_acquire(mutex, false) } -> adaptive_same_as;\n { lock.upgrade_to_writer() } -> adaptive_same_as;\n { lock.downgrade_to_reader() } -> adaptive_same_as;\n };\n\ntemplate \nconcept scoped_lockable = mutex_scoped_lock;\n\ntemplate \nconcept rw_scoped_lockable = scoped_lockable &&\n rw_mutex_scoped_lock;\n\n} // namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n#endif // __TBB_detail__mutex_common_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_namespace_injection.h", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// All public entities of the OneAPI Spec are available under oneapi namespace\n\n// Define tbb namespace first as it might not be known yet\nnamespace tbb {}\n\nnamespace oneapi {\nnamespace tbb = ::tbb;\n}\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_node_handle.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__node_handle_H\n#define __TBB_detail__node_handle_H\n\n#include \"_allocator_traits.h\"\n#include \"_assert.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n// A structure to access private node handle methods in internal TBB classes\n// Regular friend declaration is not convenient because classes which use node handle\n// can be placed in the different versioning namespaces.\nstruct node_handle_accessor {\n template \n static typename NodeHandleType::node* get_node_ptr( NodeHandleType& nh ) {\n return nh.get_node_ptr();\n }\n\n template \n static NodeHandleType construct( typename NodeHandleType::node* node_ptr ) {\n return NodeHandleType{node_ptr};\n }\n\n template \n static void deactivate( NodeHandleType& nh ) {\n nh.deactivate();\n }\n}; // struct node_handle_accessor\n\ntemplate\nclass node_handle_base {\npublic:\n using allocator_type = Allocator;\nprotected:\n using node = Node;\n using allocator_traits_type = tbb::detail::allocator_traits;\npublic:\n\n node_handle_base() : my_node(nullptr), my_allocator() {}\n node_handle_base(node_handle_base&& nh) : my_node(nh.my_node),\n my_allocator(std::move(nh.my_allocator)) {\n nh.my_node = nullptr;\n }\n\n __TBB_nodiscard bool empty() const { return my_node == nullptr; }\n explicit operator bool() const { return my_node != nullptr; }\n\n ~node_handle_base() { internal_destroy(); }\n\n node_handle_base& operator=( node_handle_base&& nh ) {\n internal_destroy();\n my_node = nh.my_node;\n move_assign_allocators(my_allocator, nh.my_allocator);\n nh.deactivate();\n return *this;\n }\n\n void swap( node_handle_base& nh ) {\n using std::swap;\n swap(my_node, nh.my_node);\n swap_allocators(my_allocator, nh.my_allocator);\n }\n\n allocator_type get_allocator() const {\n return my_allocator;\n }\n\nprotected:\n node_handle_base( node* n ) : my_node(n) {}\n\n void internal_destroy() {\n if(my_node != nullptr) {\n allocator_traits_type::destroy(my_allocator, my_node->storage());\n typename allocator_traits_type::template rebind_alloc node_allocator(my_allocator);\n node_allocator.deallocate(my_node, 1);\n }\n }\n\n node* get_node_ptr() { return my_node; }\n\n void deactivate() { my_node = nullptr; }\n\n node* my_node;\n allocator_type my_allocator;\n};\n\n// node handle for maps\ntemplate\nclass node_handle : public node_handle_base {\n using base_type = node_handle_base;\npublic:\n using key_type = Key;\n using mapped_type = typename Value::second_type;\n using allocator_type = typename base_type::allocator_type;\n\n node_handle() = default;\n\n key_type& key() const {\n __TBB_ASSERT(!this->empty(), \"Cannot get key from the empty node_type object\");\n return *const_cast(&(this->my_node->value().first));\n }\n\n mapped_type& mapped() const {\n __TBB_ASSERT(!this->empty(), \"Cannot get mapped value from the empty node_type object\");\n return this->my_node->value().second;\n }\n\nprivate:\n friend struct node_handle_accessor;\n\n node_handle( typename base_type::node* n ) : base_type(n) {}\n}; // class node_handle\n\n// node handle for sets\ntemplate\nclass node_handle : public node_handle_base {\n using base_type = node_handle_base;\npublic:\n using value_type = Key;\n using allocator_type = typename base_type::allocator_type;\n\n node_handle() = default;\n\n value_type& value() const {\n __TBB_ASSERT(!this->empty(), \"Cannot get value from the empty node_type object\");\n return *const_cast(&(this->my_node->value()));\n }\n\nprivate:\n friend struct node_handle_accessor;\n\n node_handle( typename base_type::node* n ) : base_type(n) {}\n}; // class node_handle\n\ntemplate \nvoid swap( node_handle& lhs,\n node_handle& rhs ) {\n return lhs.swap(rhs);\n}\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__node_handle_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_pipeline_filters.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_filters_H\n#define __TBB_parallel_filters_H\n\n#include \"_config.h\"\n#include \"_task.h\"\n#include \"_pipeline_filters_deduction.h\"\n#include \"../tbb_allocator.h\"\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass base_filter;\n}\n\nnamespace d2 {\ntemplate \n__TBB_requires(std::copyable)\nclass input_node;\n}\n\nnamespace r1 {\nTBB_EXPORT void __TBB_EXPORTED_FUNC set_end_of_input(d1::base_filter&);\nclass pipeline;\nclass stage_task;\nclass input_buffer;\n}\n\nnamespace d1 {\nclass filter_node;\n\n//! A stage in a pipeline.\n/** @ingroup algorithms */\nclass base_filter{\nprivate:\n //! Value used to mark \"not in pipeline\"\n static base_filter* not_in_pipeline() { return reinterpret_cast(std::intptr_t(-1)); }\npublic:\n //! The lowest bit 0 is for parallel vs serial\n static constexpr unsigned int filter_is_serial = 0x1;\n\n //! 2nd bit distinguishes ordered vs unordered filters.\n static constexpr unsigned int filter_is_out_of_order = 0x1<<1;\n\n //! 3rd bit marks input filters emitting small objects\n static constexpr unsigned int filter_may_emit_null = 0x1<<2;\n\n base_filter(const base_filter&) = delete;\n base_filter& operator=(const base_filter&) = delete;\n\nprotected:\n explicit base_filter( unsigned int m ) :\n next_filter_in_pipeline(not_in_pipeline()),\n my_input_buffer(nullptr),\n my_filter_mode(m),\n my_pipeline(nullptr)\n {}\n\n // signal end-of-input for concrete_filters\n void set_end_of_input() {\n r1::set_end_of_input(*this);\n }\n\npublic:\n //! True if filter is serial.\n bool is_serial() const {\n return bool( my_filter_mode & filter_is_serial );\n }\n\n //! True if filter must receive stream in order.\n bool is_ordered() const {\n return (my_filter_mode & filter_is_serial) && !(my_filter_mode & filter_is_out_of_order);\n }\n\n //! true if an input filter can emit null\n bool object_may_be_null() {\n return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null;\n }\n\n //! Operate on an item from the input stream, and return item for output stream.\n /** Returns nullptr if filter is a sink. */\n virtual void* operator()( void* item ) = 0;\n\n //! Destroy filter.\n virtual ~base_filter() {};\n\n //! Destroys item if pipeline was cancelled.\n /** Required to prevent memory leaks.\n Note it can be called concurrently even for serial filters.*/\n virtual void finalize( void* /*item*/ ) {}\n\nprivate:\n //! Pointer to next filter in the pipeline.\n base_filter* next_filter_in_pipeline;\n\n //! Buffer for incoming tokens, or nullptr if not required.\n /** The buffer is required if the filter is serial. */\n r1::input_buffer* my_input_buffer;\n\n friend class r1::stage_task;\n friend class r1::pipeline;\n friend void r1::set_end_of_input(d1::base_filter&);\n\n //! Storage for filter mode and dynamically checked implementation version.\n const unsigned int my_filter_mode;\n\n //! Pointer to the pipeline.\n r1::pipeline* my_pipeline;\n};\n\ntemplate\nclass concrete_filter;\n\n//! input_filter control to signal end-of-input for parallel_pipeline\nclass flow_control {\n bool is_pipeline_stopped = false;\n flow_control() = default;\n template friend class concrete_filter;\n template\n __TBB_requires(std::copyable)\n friend class d2::input_node;\npublic:\n void stop() { is_pipeline_stopped = true; }\n};\n\n// Emulate std::is_trivially_copyable (false positives not allowed, false negatives suboptimal but safe).\n#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT\ntemplate using tbb_trivially_copyable = std::is_trivially_copyable;\n#else\ntemplate struct tbb_trivially_copyable { enum { value = false }; };\ntemplate struct tbb_trivially_copyable < T* > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < bool > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < char > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < signed char > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < short > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < int > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < long > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < long long> { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < float > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < double > { enum { value = true }; };\ntemplate<> struct tbb_trivially_copyable < long double > { enum { value = true }; };\n#endif // __TBB_CPP11_TYPE_PROPERTIES_PRESENT\n\ntemplate\nstruct use_allocator {\n static constexpr bool value = sizeof(T) > sizeof(void *) || !tbb_trivially_copyable::value;\n};\n\n// A helper class to customize how a type is passed between filters.\n// Usage: token_helper::value>\ntemplate struct token_helper;\n\n// using tbb_allocator\ntemplate\nstruct token_helper {\n using pointer = T*;\n using value_type = T;\n static pointer create_token(value_type && source) {\n return new (r1::allocate_memory(sizeof(T))) T(std::move(source));\n }\n static value_type & token(pointer & t) { return *t; }\n static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); }\n static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); }\n static void destroy_token(pointer token) {\n token->~value_type();\n r1::deallocate_memory(token);\n }\n};\n\n// pointer specialization\ntemplate\nstruct token_helper {\n using pointer = T*;\n using value_type = T*;\n static pointer create_token(const value_type & source) { return source; }\n static value_type & token(pointer & t) { return t; }\n static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); }\n static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); }\n static void destroy_token( pointer /*token*/) {}\n};\n\n// converting type to and from void*, passing objects directly\ntemplate\nstruct token_helper {\n typedef union {\n T actual_value;\n void * void_overlay;\n } type_to_void_ptr_map;\n using pointer = T; // not really a pointer in this case.\n using value_type = T;\n static pointer create_token(const value_type & source) { return source; }\n static value_type & token(pointer & t) { return t; }\n static void * cast_to_void_ptr(pointer ref) {\n type_to_void_ptr_map mymap;\n mymap.void_overlay = nullptr;\n mymap.actual_value = ref;\n return mymap.void_overlay;\n }\n static pointer cast_from_void_ptr(void * ref) {\n type_to_void_ptr_map mymap;\n mymap.void_overlay = ref;\n return mymap.actual_value;\n }\n static void destroy_token( pointer /*token*/) {}\n};\n\n// intermediate\ntemplate\nclass concrete_filter: public base_filter {\n const Body& my_body;\n using input_helper = token_helper::value>;\n using input_pointer = typename input_helper::pointer;\n using output_helper = token_helper::value>;\n using output_pointer = typename output_helper::pointer;\n\n void* operator()(void* input) override {\n input_pointer temp_input = input_helper::cast_from_void_ptr(input);\n output_pointer temp_output = output_helper::create_token(tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input))));\n input_helper::destroy_token(temp_input);\n return output_helper::cast_to_void_ptr(temp_output);\n }\n\n void finalize(void * input) override {\n input_pointer temp_input = input_helper::cast_from_void_ptr(input);\n input_helper::destroy_token(temp_input);\n }\n\npublic:\n concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {}\n};\n\n// input\ntemplate\nclass concrete_filter: public base_filter {\n const Body& my_body;\n using output_helper = token_helper::value>;\n using output_pointer = typename output_helper::pointer;\n\n void* operator()(void*) override {\n flow_control control;\n output_pointer temp_output = output_helper::create_token(my_body(control));\n if(control.is_pipeline_stopped) {\n output_helper::destroy_token(temp_output);\n set_end_of_input();\n return nullptr;\n }\n return output_helper::cast_to_void_ptr(temp_output);\n }\n\npublic:\n concrete_filter(unsigned int m, const Body& body) :\n base_filter(m | filter_may_emit_null),\n my_body(body)\n {}\n};\n\n// output\ntemplate\nclass concrete_filter: public base_filter {\n const Body& my_body;\n using input_helper = token_helper::value>;\n using input_pointer = typename input_helper::pointer;\n\n void* operator()(void* input) override {\n input_pointer temp_input = input_helper::cast_from_void_ptr(input);\n tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input)));\n input_helper::destroy_token(temp_input);\n return nullptr;\n }\n void finalize(void* input) override {\n input_pointer temp_input = input_helper::cast_from_void_ptr(input);\n input_helper::destroy_token(temp_input);\n }\n\npublic:\n concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {}\n};\n\ntemplate\nclass concrete_filter: public base_filter {\n const Body& my_body;\n\n void* operator()(void*) override {\n flow_control control;\n my_body(control);\n void* output = control.is_pipeline_stopped ? nullptr : (void*)(std::intptr_t)-1;\n return output;\n }\npublic:\n concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {}\n};\n\nclass filter_node_ptr {\n filter_node * my_node;\n\npublic:\n filter_node_ptr() : my_node(nullptr) {}\n filter_node_ptr(filter_node *);\n ~filter_node_ptr();\n filter_node_ptr(const filter_node_ptr &);\n filter_node_ptr(filter_node_ptr &&);\n void operator=(filter_node *);\n void operator=(const filter_node_ptr &);\n void operator=(filter_node_ptr &&);\n filter_node& operator*() const;\n operator bool() const;\n};\n\n//! Abstract base class that represents a node in a parse tree underlying a filter class.\n/** These nodes are always heap-allocated and can be shared by filter objects. */\nclass filter_node {\n /** Count must be atomic because it is hidden state for user, but might be shared by threads. */\n std::atomic ref_count;\npublic:\n filter_node_ptr left;\n filter_node_ptr right;\nprotected:\n filter_node() : ref_count(0), left(nullptr), right(nullptr) {\n#ifdef __TBB_TEST_FILTER_NODE_COUNT\n ++(__TBB_TEST_FILTER_NODE_COUNT);\n#endif\n }\npublic:\n filter_node(const filter_node_ptr& x, const filter_node_ptr& y) : filter_node(){\n left = x;\n right = y;\n }\n filter_node(const filter_node&) = delete;\n filter_node& operator=(const filter_node&) = delete;\n\n //! Add concrete_filter to pipeline\n virtual base_filter* create_filter() const {\n __TBB_ASSERT(false, \"method of non-leaf was called\");\n return nullptr;\n }\n\n //! Increment reference count\n void add_ref() { ref_count.fetch_add(1, std::memory_order_relaxed); }\n\n //! Decrement reference count and delete if it becomes zero.\n void remove_ref() {\n __TBB_ASSERT(ref_count>0,\"ref_count underflow\");\n if( ref_count.fetch_sub(1, std::memory_order_relaxed) == 1 ) {\n this->~filter_node();\n r1::deallocate_memory(this);\n }\n }\n\n virtual ~filter_node() {\n#ifdef __TBB_TEST_FILTER_NODE_COUNT\n --(__TBB_TEST_FILTER_NODE_COUNT);\n#endif\n }\n};\n\ninline filter_node_ptr::filter_node_ptr(filter_node * nd) : my_node(nd) {\n if (my_node) {\n my_node->add_ref();\n }\n}\n\ninline filter_node_ptr::~filter_node_ptr() {\n if (my_node) {\n my_node->remove_ref();\n }\n}\n\ninline filter_node_ptr::filter_node_ptr(const filter_node_ptr & rhs) : my_node(rhs.my_node) {\n if (my_node) {\n my_node->add_ref();\n }\n}\n\ninline filter_node_ptr::filter_node_ptr(filter_node_ptr && rhs) : my_node(rhs.my_node) {\n rhs.my_node = nullptr;\n}\n\ninline void filter_node_ptr::operator=(filter_node * rhs) {\n // Order of operations below carefully chosen so that reference counts remain correct\n // in unlikely event that remove_ref throws exception.\n filter_node* old = my_node;\n my_node = rhs;\n if (my_node) {\n my_node->add_ref();\n }\n if (old) {\n old->remove_ref();\n }\n}\n\ninline void filter_node_ptr::operator=(const filter_node_ptr & rhs) {\n *this = rhs.my_node;\n}\n\ninline void filter_node_ptr::operator=(filter_node_ptr && rhs) {\n filter_node* old = my_node;\n my_node = rhs.my_node;\n rhs.my_node = nullptr;\n if (old) {\n old->remove_ref();\n }\n}\n\ninline filter_node& filter_node_ptr::operator*() const{\n __TBB_ASSERT(my_node,\"nullptr node is used\");\n return *my_node;\n}\n\ninline filter_node_ptr::operator bool() const {\n return my_node != nullptr;\n}\n\n//! Node in parse tree representing result of make_filter.\ntemplate\nclass filter_node_leaf: public filter_node {\n const unsigned int my_mode;\n const Body my_body;\n base_filter* create_filter() const override {\n return new(r1::allocate_memory(sizeof(concrete_filter))) concrete_filter(my_mode,my_body);\n }\npublic:\n filter_node_leaf( unsigned int m, const Body& b ) : my_mode(m), my_body(b) {}\n};\n\n\ntemplate ::input_type>\nusing filter_input = typename std::conditional::value, void, Input>::type;\n\ntemplate \nusing filter_output = typename filter_body_types::output_type;\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n\n#endif /* __TBB_parallel_filters_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_pipeline_filters_deduction.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__pipeline_filters_deduction_H\n#define __TBB__pipeline_filters_deduction_H\n\n#include \"_config.h\"\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\ntemplate \nstruct declare_filter_types {\n using input_type = typename std::remove_const::type>::type;\n using output_type = typename std::remove_const::type>::type;\n};\n\ntemplate struct filter_body_types;\n\ntemplate \nstruct filter_body_types : declare_filter_types {};\n\ntemplate \nstruct filter_body_types : declare_filter_types {};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB__pipeline_filters_deduction_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_range_common.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__range_common_H\n#define __TBB_detail__range_common_H\n\n#include \"_config.h\"\n#include \"_utils.h\"\n#if __TBB_CPP20_CONCEPTS_PRESENT\n#include \n#endif\n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n//! Dummy type that distinguishes splitting constructor from copy constructor.\n/**\n * See description of parallel_for and parallel_reduce for example usages.\n * @ingroup algorithms\n */\nclass split {};\n\n//! Type enables transmission of splitting proportion from partitioners to range objects\n/**\n * In order to make use of such facility Range objects must implement\n * splitting constructor with this type passed.\n */\nclass proportional_split : no_assign {\npublic:\n proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { }\n\n size_t left() const { return my_left; }\n size_t right() const { return my_right; }\n\n // used when range does not support proportional split\n explicit operator split() const { return split(); }\n\nprivate:\n size_t my_left, my_right;\n};\n\ntemplate \nstruct range_split_object_provider {\n template \n static split get( PartitionerSplitType& ) { return split(); }\n};\n\ntemplate \nstruct range_split_object_provider::value>::type> {\n template \n static PartitionerSplitType& get( PartitionerSplitType& split_obj ) { return split_obj; }\n};\n\ntemplate \nauto get_range_split_object( PartitionerSplitType& split_obj )\n-> decltype(range_split_object_provider::get(split_obj)) {\n return range_split_object_provider::get(split_obj);\n}\n\ntemplate \nusing range_iterator_type = decltype(std::begin(std::declval()));\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \nusing iterator_reference_type = typename std::iterator_traits::reference;\n\ntemplate \nusing range_reference_type = iterator_reference_type>;\n\ntemplate \nconcept blocked_range_value = std::copyable &&\n requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) {\n { lhs < rhs } -> relaxed_convertible_to;\n { lhs - rhs } -> std::convertible_to;\n { lhs + (rhs - lhs) } -> std::convertible_to;\n };\n\ntemplate \nconcept splittable = std::constructible_from;\n\ntemplate \nconcept tbb_range = std::copy_constructible &&\n splittable &&\n requires( const std::remove_reference_t& range ) {\n { range.empty() } -> relaxed_convertible_to;\n { range.is_divisible() } -> relaxed_convertible_to;\n };\n\ntemplate \nconstexpr bool iterator_concept_helper( std::input_iterator_tag ) {\n return std::input_iterator;\n}\n\ntemplate \nconstexpr bool iterator_concept_helper( std::random_access_iterator_tag ) {\n return std::random_access_iterator;\n}\n\ntemplate \nconcept iterator_satisfies = requires (IteratorTag tag) {\n requires iterator_concept_helper(tag);\n};\n\ntemplate \nconcept container_based_sequence = requires( Sequence& seq ) {\n { std::begin(seq) } -> iterator_satisfies;\n { std::end(seq) } -> iterator_satisfies;\n};\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n} // namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__range_common_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_rtm_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__rtm_mutex_impl_H\n#define __TBB__rtm_mutex_impl_H\n\n#include \"_assert.h\"\n#include \"_utils.h\"\n#include \"../spin_mutex.h\"\n\n#include \"../profiling.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\nstruct rtm_mutex_impl;\n}\nnamespace d1 {\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Suppress warning: structure was padded due to alignment specifier\n // #pragma warning (push)\n // #pragma warning (disable: 4324)\n#endif\n\n/** A rtm_mutex is an speculation-enabled spin mutex.\n It should be used for locking short critical sections where the lock is\n contended but the data it protects are not. If zero-initialized, the\n mutex is considered unheld.\n @ingroup synchronization */\nclass alignas(max_nfs_size) rtm_mutex : private spin_mutex {\nprivate:\n enum class rtm_state {\n rtm_none,\n rtm_transacting,\n rtm_real\n };\npublic:\n //! Constructors\n rtm_mutex() noexcept {\n create_itt_sync(this, \"tbb::speculative_spin_mutex\", \"\");\n }\n\n //! Destructor\n ~rtm_mutex() = default;\n\n //! Represents acquisition of a mutex.\n class scoped_lock {\n public:\n friend class rtm_mutex;\n //! Construct lock that has not acquired a mutex.\n /** Equivalent to zero-initialization of *this. */\n constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) {}\n\n //! Acquire lock on given mutex.\n scoped_lock(rtm_mutex& m) : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) {\n acquire(m);\n }\n\n //! Release lock (if lock is held).\n ~scoped_lock() {\n if(m_transaction_state != rtm_state::rtm_none) {\n release();\n }\n }\n\n //! No Copy\n scoped_lock(const scoped_lock&) = delete;\n scoped_lock& operator=(const scoped_lock&) = delete;\n\n //! Acquire lock on given mutex.\n void acquire(rtm_mutex& m);\n\n //! Try acquire lock on given mutex.\n bool try_acquire(rtm_mutex& m);\n\n //! Release lock\n void release();\n\n private:\n rtm_mutex* m_mutex;\n rtm_state m_transaction_state;\n friend r1::rtm_mutex_impl;\n };\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = false;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\nprivate:\n friend r1::rtm_mutex_impl;\n}; // end of rtm_mutex\n} // namespace d1\n\nnamespace r1 {\n //! Internal acquire lock.\n // only_speculate == true if we're doing a try_lock, else false.\n TBB_EXPORT void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&, bool only_speculate = false);\n //! Internal try_acquire lock.\n TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&);\n //! Internal release lock.\n TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock&);\n} // namespace r1\n\nnamespace d1 {\n//! Acquire lock on given mutex.\ninline void rtm_mutex::scoped_lock::acquire(rtm_mutex& m) {\n __TBB_ASSERT(!m_mutex, \"lock is already acquired\");\n r1::acquire(m, *this);\n}\n\n//! Try acquire lock on given mutex.\ninline bool rtm_mutex::scoped_lock::try_acquire(rtm_mutex& m) {\n __TBB_ASSERT(!m_mutex, \"lock is already acquired\");\n return r1::try_acquire(m, *this);\n}\n\n//! Release lock\ninline void rtm_mutex::scoped_lock::release() {\n __TBB_ASSERT(m_mutex, \"lock is not acquired\");\n __TBB_ASSERT(m_transaction_state != rtm_state::rtm_none, \"lock is not acquired\");\n return r1::release(*this);\n}\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning (pop) // 4324 warning\n#endif\n\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(rtm_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(rtm_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif // WIN\n#else\ninline void set_name(rtm_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(rtm_mutex&, const wchar_t*) {}\n#endif // WIN\n#endif\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB__rtm_mutex_impl_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_rtm_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__rtm_rw_mutex_H\n#define __TBB_detail__rtm_rw_mutex_H\n\n#include \"_assert.h\"\n#include \"_utils.h\"\n#include \"../spin_rw_mutex.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nstruct rtm_rw_mutex_impl;\n}\n\nnamespace d1 {\n\nconstexpr std::size_t speculation_granularity = 64;\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Suppress warning: structure was padded due to alignment specifier\n // #pragma warning (push)\n // #pragma warning (disable: 4324)\n#endif\n\n//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and writer-preference\n/** @ingroup synchronization */\nclass alignas(max_nfs_size) rtm_rw_mutex : private spin_rw_mutex {\n friend struct r1::rtm_rw_mutex_impl;\nprivate:\n enum class rtm_type {\n rtm_not_in_mutex,\n rtm_transacting_reader,\n rtm_transacting_writer,\n rtm_real_reader,\n rtm_real_writer\n };\npublic:\n //! Constructors\n rtm_rw_mutex() noexcept : write_flag(false) {\n create_itt_sync(this, \"tbb::speculative_spin_rw_mutex\", \"\");\n }\n\n //! Destructor\n ~rtm_rw_mutex() = default;\n\n //! Represents acquisition of a mutex.\n class scoped_lock {\n friend struct r1::rtm_rw_mutex_impl;\n public:\n //! Construct lock that has not acquired a mutex.\n /** Equivalent to zero-initialization of *this. */\n constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) {}\n\n //! Acquire lock on given mutex.\n scoped_lock(rtm_rw_mutex& m, bool write = true) : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) {\n acquire(m, write);\n }\n\n //! Release lock (if lock is held).\n ~scoped_lock() {\n if(m_transaction_state != rtm_type::rtm_not_in_mutex) {\n release();\n }\n }\n\n //! No Copy\n scoped_lock(const scoped_lock&) = delete;\n scoped_lock& operator=(const scoped_lock&) = delete;\n\n //! Acquire lock on given mutex.\n inline void acquire(rtm_rw_mutex& m, bool write = true);\n\n //! Try acquire lock on given mutex.\n inline bool try_acquire(rtm_rw_mutex& m, bool write = true);\n\n //! Release lock\n inline void release();\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n inline bool upgrade_to_writer();\n\n //! Downgrade writer to become a reader.\n inline bool downgrade_to_reader();\n\n inline bool is_writer() const;\n private:\n rtm_rw_mutex* m_mutex;\n rtm_type m_transaction_state;\n };\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = true;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\n\nprivate:\n alignas(speculation_granularity) std::atomic write_flag;\n};\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning (pop) // 4324 warning\n#endif\n\n} // namespace d1\n\nnamespace r1 {\n //! Internal acquire write lock.\n // only_speculate == true if we're doing a try_lock, else false.\n TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false);\n //! Internal acquire read lock.\n // only_speculate == true if we're doing a try_lock, else false.\n TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false);\n //! Internal upgrade reader to become a writer.\n TBB_EXPORT bool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock&);\n //! Internal downgrade writer to become a reader.\n TBB_EXPORT bool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock&);\n //! Internal try_acquire write lock.\n TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&);\n //! Internal try_acquire read lock.\n TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&);\n //! Internal release lock.\n TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock&);\n}\n\nnamespace d1 {\n//! Acquire lock on given mutex.\nvoid rtm_rw_mutex::scoped_lock::acquire(rtm_rw_mutex& m, bool write) {\n __TBB_ASSERT(!m_mutex, \"lock is already acquired\");\n if (write) {\n r1::acquire_writer(m, *this);\n } else {\n r1::acquire_reader(m, *this);\n }\n}\n\n//! Try acquire lock on given mutex.\nbool rtm_rw_mutex::scoped_lock::try_acquire(rtm_rw_mutex& m, bool write) {\n __TBB_ASSERT(!m_mutex, \"lock is already acquired\");\n if (write) {\n return r1::try_acquire_writer(m, *this);\n } else {\n return r1::try_acquire_reader(m, *this);\n }\n}\n\n//! Release lock\nvoid rtm_rw_mutex::scoped_lock::release() {\n __TBB_ASSERT(m_mutex, \"lock is not acquired\");\n __TBB_ASSERT(m_transaction_state != rtm_type::rtm_not_in_mutex, \"lock is not acquired\");\n return r1::release(*this);\n}\n\n//! Upgrade reader to become a writer.\n/** Returns whether the upgrade happened without releasing and re-acquiring the lock */\nbool rtm_rw_mutex::scoped_lock::upgrade_to_writer() {\n __TBB_ASSERT(m_mutex, \"lock is not acquired\");\n if (m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer) {\n return true; // Already a writer\n }\n return r1::upgrade(*this);\n}\n\n//! Downgrade writer to become a reader.\nbool rtm_rw_mutex::scoped_lock::downgrade_to_reader() {\n __TBB_ASSERT(m_mutex, \"lock is not acquired\");\n if (m_transaction_state == rtm_type::rtm_transacting_reader || m_transaction_state == rtm_type::rtm_real_reader) {\n return true; // Already a reader\n }\n return r1::downgrade(*this);\n}\n\nbool rtm_rw_mutex::scoped_lock::is_writer() const {\n __TBB_ASSERT(m_mutex, \"lock is not acquired\");\n return m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer;\n}\n\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(rtm_rw_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(rtm_rw_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif // WIN\n#else\ninline void set_name(rtm_rw_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(rtm_rw_mutex&, const wchar_t*) {}\n#endif // WIN\n#endif\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__rtm_rw_mutex_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_scoped_lock.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail_scoped_lock_H\n#define __TBB_detail_scoped_lock_H\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n// unique_scoped_lock supposes that Mutex operations never throw\ntemplate \nclass unique_scoped_lock {\n //! Points to currently held Mutex, or nullptr if no lock is held.\n Mutex* m_mutex{};\n\npublic:\n //! Construct without acquiring a Mutex.\n constexpr unique_scoped_lock() noexcept : m_mutex(nullptr) {}\n\n //! Construct and acquire lock on a Mutex.\n unique_scoped_lock(Mutex& m) {\n acquire(m);\n }\n\n //! No Copy\n unique_scoped_lock(const unique_scoped_lock&) = delete;\n unique_scoped_lock& operator=(const unique_scoped_lock&) = delete;\n\n //! Acquire lock.\n void acquire(Mutex& m) {\n __TBB_ASSERT(m_mutex == nullptr, \"The mutex is already acquired\");\n m_mutex = &m;\n m.lock();\n }\n\n //! Try acquiring lock (non-blocking)\n /** Return true if lock acquired; false otherwise. */\n bool try_acquire(Mutex& m) {\n __TBB_ASSERT(m_mutex == nullptr, \"The mutex is already acquired\");\n bool succeed = m.try_lock();\n if (succeed) {\n m_mutex = &m;\n }\n return succeed;\n }\n\n //! Release lock\n void release() {\n __TBB_ASSERT(m_mutex, \"release on Mutex::unique_scoped_lock that is not holding a lock\");\n m_mutex->unlock();\n m_mutex = nullptr;\n }\n\n //! Destroy lock. If holding a lock, releases the lock first.\n ~unique_scoped_lock() {\n if (m_mutex) {\n release();\n }\n }\n};\n\n// rw_scoped_lock supposes that Mutex operations never throw\ntemplate \nclass rw_scoped_lock {\npublic:\n //! Construct lock that has not acquired a mutex.\n /** Equivalent to zero-initialization of *this. */\n constexpr rw_scoped_lock() noexcept {}\n\n //! Acquire lock on given mutex.\n rw_scoped_lock(Mutex& m, bool write = true) {\n acquire(m, write);\n }\n\n //! Release lock (if lock is held).\n ~rw_scoped_lock() {\n if (m_mutex) {\n release();\n }\n }\n\n //! No Copy\n rw_scoped_lock(const rw_scoped_lock&) = delete;\n rw_scoped_lock& operator=(const rw_scoped_lock&) = delete;\n\n //! Acquire lock on given mutex.\n void acquire(Mutex& m, bool write = true) {\n __TBB_ASSERT(m_mutex == nullptr, \"The mutex is already acquired\");\n m_is_writer = write;\n m_mutex = &m;\n if (write) {\n m_mutex->lock();\n } else {\n m_mutex->lock_shared();\n }\n }\n\n //! Try acquire lock on given mutex.\n bool try_acquire(Mutex& m, bool write = true) {\n bool succeed = write ? m.try_lock() : m.try_lock_shared();\n if (succeed) {\n m_mutex = &m;\n m_is_writer = write;\n }\n return succeed;\n }\n\n //! Release lock.\n void release() {\n __TBB_ASSERT(m_mutex != nullptr, \"The mutex is not acquired\");\n Mutex* m = m_mutex;\n m_mutex = nullptr;\n\n if (m_is_writer) {\n m->unlock();\n } else {\n m->unlock_shared();\n }\n }\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n bool upgrade_to_writer() {\n __TBB_ASSERT(m_mutex != nullptr, \"The mutex is not acquired\");\n if (m_is_writer) {\n return true; // Already a writer\n }\n m_is_writer = true;\n return m_mutex->upgrade();\n }\n\n //! Downgrade writer to become a reader.\n bool downgrade_to_reader() {\n __TBB_ASSERT(m_mutex != nullptr, \"The mutex is not acquired\");\n if (m_is_writer) {\n m_mutex->downgrade();\n m_is_writer = false;\n }\n return true;\n }\n\n bool is_writer() const {\n __TBB_ASSERT(m_mutex != nullptr, \"The mutex is not acquired\");\n return m_is_writer;\n }\n\nprotected:\n //! The pointer to the current mutex that is held, or nullptr if no mutex is held.\n Mutex* m_mutex {nullptr};\n\n //! If mutex != nullptr, then is_writer is true if holding a writer lock, false if holding a reader lock.\n /** Not defined if not holding a lock. */\n bool m_is_writer {false};\n};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail_scoped_lock_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_segment_table.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__segment_table_H\n#define __TBB_detail__segment_table_H\n\n#include \"_config.h\"\n#include \"_allocator_traits.h\"\n#include \"_template_helpers.h\"\n#include \"_utils.h\"\n#include \"_assert.h\"\n#include \"_exception.h\"\n#include \n#include \n#include \n#include \n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(push)\n// #pragma warning(disable: 4127) // warning C4127: conditional expression is constant\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\ntemplate \nclass segment_table {\npublic:\n using value_type = T;\n using segment_type = T*;\n using atomic_segment = std::atomic;\n using segment_table_type = atomic_segment*;\n\n using size_type = std::size_t;\n using segment_index_type = std::size_t;\n\n using allocator_type = Allocator;\n\n using allocator_traits_type = tbb::detail::allocator_traits;\n using segment_table_allocator_type = typename allocator_traits_type::template rebind_alloc;\nprotected:\n using segment_table_allocator_traits = tbb::detail::allocator_traits;\n using derived_type = DerivedType;\n\n static constexpr size_type pointers_per_embedded_table = PointersPerEmbeddedTable;\n static constexpr size_type pointers_per_long_table = sizeof(size_type) * 8;\npublic:\n segment_table( const allocator_type& alloc = allocator_type() )\n : my_segment_table_allocator(alloc), my_segment_table(nullptr)\n , my_first_block{}, my_size{}, my_segment_table_allocation_failed{}\n {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n }\n\n segment_table( const segment_table& other )\n : my_segment_table_allocator(segment_table_allocator_traits::\n select_on_container_copy_construction(other.my_segment_table_allocator))\n , my_segment_table(nullptr), my_first_block{}, my_size{}, my_segment_table_allocation_failed{}\n {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n try_call( [&] {\n internal_transfer(other, copy_segment_body_type{*this});\n } ).on_exception( [&] {\n clear();\n });\n }\n\n segment_table( const segment_table& other, const allocator_type& alloc )\n : my_segment_table_allocator(alloc), my_segment_table(nullptr)\n , my_first_block{}, my_size{}, my_segment_table_allocation_failed{}\n {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n try_call( [&] {\n internal_transfer(other, copy_segment_body_type{*this});\n } ).on_exception( [&] {\n clear();\n });\n }\n\n segment_table( segment_table&& other )\n : my_segment_table_allocator(std::move(other.my_segment_table_allocator)), my_segment_table(nullptr)\n , my_first_block{}, my_size{}, my_segment_table_allocation_failed{}\n {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n internal_move(std::move(other));\n }\n\n segment_table( segment_table&& other, const allocator_type& alloc )\n : my_segment_table_allocator(alloc), my_segment_table(nullptr), my_first_block{}\n , my_size{}, my_segment_table_allocation_failed{}\n {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n using is_equal_type = typename segment_table_allocator_traits::is_always_equal;\n internal_move_construct_with_allocator(std::move(other), alloc, is_equal_type());\n }\n\n ~segment_table() {\n clear();\n }\n\n segment_table& operator=( const segment_table& other ) {\n if (this != &other) {\n copy_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator);\n internal_transfer(other, copy_segment_body_type{*this});\n }\n return *this;\n }\n\n segment_table& operator=( segment_table&& other )\n noexcept(derived_type::is_noexcept_assignment)\n {\n using pocma_type = typename segment_table_allocator_traits::propagate_on_container_move_assignment;\n using is_equal_type = typename segment_table_allocator_traits::is_always_equal;\n\n if (this != &other) {\n move_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator);\n internal_move_assign(std::move(other), tbb::detail::disjunction());\n }\n return *this;\n }\n\n void swap( segment_table& other )\n noexcept(derived_type::is_noexcept_swap)\n {\n using is_equal_type = typename segment_table_allocator_traits::is_always_equal;\n using pocs_type = typename segment_table_allocator_traits::propagate_on_container_swap;\n\n if (this != &other) {\n swap_allocators(my_segment_table_allocator, other.my_segment_table_allocator);\n internal_swap(other, tbb::detail::disjunction());\n }\n }\n\n segment_type get_segment( segment_index_type index ) const {\n return get_table()[index] + segment_base(index);\n }\n\n value_type& operator[]( size_type index ) {\n return internal_subscript(index);\n }\n\n const value_type& operator[]( size_type index ) const {\n return const_cast(this)->internal_subscript(index);\n }\n\n const segment_table_allocator_type& get_allocator() const {\n return my_segment_table_allocator;\n }\n\n segment_table_allocator_type& get_allocator() {\n return my_segment_table_allocator;\n }\n\n void enable_segment( segment_type& segment, segment_table_type table, segment_index_type seg_index, size_type index ) {\n // Allocate new segment\n segment_type new_segment = self()->create_segment(table, seg_index, index);\n if (new_segment != nullptr) {\n // Store (new_segment - segment_base) into the segment table to allow access to the table by index via\n // my_segment_table[segment_index_of(index)][index]\n segment_type disabled_segment = nullptr;\n if (!table[seg_index].compare_exchange_strong(disabled_segment, new_segment - segment_base(seg_index))) {\n // compare_exchange failed => some other thread has already enabled this segment\n // Deallocate the memory\n self()->deallocate_segment(new_segment, seg_index);\n }\n }\n\n segment = table[seg_index].load(std::memory_order_acquire);\n __TBB_ASSERT(segment != nullptr, \"If create_segment returned nullptr, the element should be stored in the table\");\n }\n\n void delete_segment( segment_index_type seg_index ) {\n segment_type segment_to_delete = self()->nullify_segment(get_table(), seg_index);\n if (segment_to_delete == segment_allocation_failure_tag) {\n return;\n }\n\n segment_to_delete += segment_base(seg_index);\n\n // Deallocate the segment\n self()->destroy_segment(segment_to_delete, seg_index);\n }\n\n size_type number_of_segments( segment_table_type table ) const {\n // Check for an active table, if it is embedded table - return the number of embedded segments\n // Otherwise - return the maximum number of segments\n return table == my_embedded_table ? pointers_per_embedded_table : pointers_per_long_table;\n }\n\n size_type capacity() const noexcept {\n segment_table_type table = get_table();\n size_type num_segments = number_of_segments(table);\n for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) {\n // Check if the pointer is valid (allocated)\n if (table[seg_index].load(std::memory_order_relaxed) <= segment_allocation_failure_tag) {\n return segment_base(seg_index);\n }\n }\n return segment_base(num_segments);\n }\n\n size_type find_last_allocated_segment( segment_table_type table ) const noexcept {\n size_type end = 0;\n size_type num_segments = number_of_segments(table);\n for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) {\n // Check if the pointer is valid (allocated)\n if (table[seg_index].load(std::memory_order_relaxed) > segment_allocation_failure_tag) {\n end = seg_index + 1;\n }\n }\n return end;\n }\n\n void reserve( size_type n ) {\n if (n > allocator_traits_type::max_size(my_segment_table_allocator)) {\n throw_exception(exception_id::reservation_length_error);\n }\n\n size_type size = my_size.load(std::memory_order_relaxed);\n segment_index_type start_seg_idx = size == 0 ? 0 : segment_index_of(size - 1) + 1;\n for (segment_index_type seg_idx = start_seg_idx; segment_base(seg_idx) < n; ++seg_idx) {\n size_type first_index = segment_base(seg_idx);\n internal_subscript(first_index);\n }\n }\n\n void clear() {\n clear_segments();\n clear_table();\n my_size.store(0, std::memory_order_relaxed);\n my_first_block.store(0, std::memory_order_relaxed);\n }\n\n void clear_segments() {\n segment_table_type current_segment_table = get_table();\n for (size_type i = number_of_segments(current_segment_table); i != 0; --i) {\n if (current_segment_table[i - 1].load(std::memory_order_relaxed) != nullptr) {\n // If the segment was enabled - disable and deallocate it\n delete_segment(i - 1);\n }\n }\n }\n\n void clear_table() {\n segment_table_type current_segment_table = get_table();\n if (current_segment_table != my_embedded_table) {\n // If the active table is not the embedded one - deallocate the active table\n for (size_type i = 0; i != pointers_per_long_table; ++i) {\n segment_table_allocator_traits::destroy(my_segment_table_allocator, ¤t_segment_table[i]);\n }\n\n segment_table_allocator_traits::deallocate(my_segment_table_allocator, current_segment_table, pointers_per_long_table);\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n zero_table(my_embedded_table, pointers_per_embedded_table);\n }\n }\n\n void extend_table_if_necessary(segment_table_type& table, size_type start_index, size_type end_index) {\n // extend_segment_table if an active table is an embedded table\n // and the requested index is not in the embedded table\n if (table == my_embedded_table && end_index > embedded_table_size) {\n if (start_index <= embedded_table_size) {\n try_call([&] {\n table = self()->allocate_long_table(my_embedded_table, start_index);\n // It is possible that the table was extended by the thread that allocated first_block.\n // In this case it is necessary to re-read the current table.\n\n if (table) {\n my_segment_table.store(table, std::memory_order_release);\n } else {\n table = my_segment_table.load(std::memory_order_acquire);\n }\n }).on_exception([&] {\n my_segment_table_allocation_failed.store(true, std::memory_order_relaxed);\n });\n } else {\n atomic_backoff backoff;\n do {\n if (my_segment_table_allocation_failed.load(std::memory_order_relaxed)) {\n throw_exception(exception_id::bad_alloc);\n }\n backoff.pause();\n table = my_segment_table.load(std::memory_order_acquire);\n } while (table == my_embedded_table);\n }\n }\n }\n\n // Return the segment where index is stored\n static constexpr segment_index_type segment_index_of( size_type index ) {\n return size_type(tbb::detail::log2(uintptr_t(index|1)));\n }\n\n // Needed to calculate the offset in segment\n static constexpr size_type segment_base( size_type index ) {\n return size_type(1) << index & ~size_type(1);\n }\n\n // Return size of the segment\n static constexpr size_type segment_size( size_type index ) {\n return index == 0 ? 2 : size_type(1) << index;\n }\n\nprivate:\n\n derived_type* self() {\n return static_cast(this);\n }\n\n struct copy_segment_body_type {\n void operator()( segment_index_type index, segment_type from, segment_type to ) const {\n my_instance.self()->copy_segment(index, from, to);\n }\n segment_table& my_instance;\n };\n\n struct move_segment_body_type {\n void operator()( segment_index_type index, segment_type from, segment_type to ) const {\n my_instance.self()->move_segment(index, from, to);\n }\n segment_table& my_instance;\n };\n\n // Transgers all segments from the other table\n template \n void internal_transfer( const segment_table& other, TransferBody transfer_segment ) {\n static_cast(this)->destroy_elements();\n\n assign_first_block_if_necessary(other.my_first_block.load(std::memory_order_relaxed));\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n\n segment_table_type other_table = other.get_table();\n size_type end_segment_size = segment_size(other.find_last_allocated_segment(other_table));\n\n // If an exception occurred in other, then the size may be greater than the size of the end segment.\n size_type other_size = end_segment_size < other.my_size.load(std::memory_order_relaxed) ?\n other.my_size.load(std::memory_order_relaxed) : end_segment_size;\n other_size = my_segment_table_allocation_failed ? embedded_table_size : other_size;\n\n for (segment_index_type i = 0; segment_base(i) < other_size; ++i) {\n // If the segment in other table is enabled - transfer it\n if (other_table[i].load(std::memory_order_relaxed) == segment_allocation_failure_tag)\n {\n my_size = segment_base(i);\n break;\n } else if (other_table[i].load(std::memory_order_relaxed) != nullptr) {\n internal_subscript(segment_base(i));\n transfer_segment(i, other.get_table()[i].load(std::memory_order_relaxed) + segment_base(i),\n get_table()[i].load(std::memory_order_relaxed) + segment_base(i));\n }\n }\n }\n\n // Moves the other segment table\n // Only equal allocators are allowed\n void internal_move( segment_table&& other ) {\n // NOTE: allocators should be equal\n clear();\n my_first_block.store(other.my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n // If an active table in other is embedded - restore all of the embedded segments\n if (other.get_table() == other.my_embedded_table) {\n for ( size_type i = 0; i != pointers_per_embedded_table; ++i ) {\n segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed);\n my_embedded_table[i].store(other_segment, std::memory_order_relaxed);\n other.my_embedded_table[i].store(nullptr, std::memory_order_relaxed);\n }\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n } else {\n my_segment_table.store(other.my_segment_table, std::memory_order_relaxed);\n other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed);\n zero_table(other.my_embedded_table, pointers_per_embedded_table);\n }\n other.my_size.store(0, std::memory_order_relaxed);\n }\n\n // Move construct the segment table with the allocator object\n // if any instances of allocator_type are always equal\n void internal_move_construct_with_allocator( segment_table&& other, const allocator_type&,\n /*is_always_equal = */ std::true_type ) {\n internal_move(std::move(other));\n }\n\n // Move construct the segment table with the allocator object\n // if any instances of allocator_type are always equal\n void internal_move_construct_with_allocator( segment_table&& other, const allocator_type& alloc,\n /*is_always_equal = */ std::false_type ) {\n if (other.my_segment_table_allocator == alloc) {\n // If allocators are equal - restore pointers\n internal_move(std::move(other));\n } else {\n // If allocators are not equal - perform per element move with reallocation\n try_call( [&] {\n internal_transfer(other, move_segment_body_type{*this});\n } ).on_exception( [&] {\n clear();\n });\n }\n }\n\n // Move assigns the segment table to other is any instances of allocator_type are always equal\n // or propagate_on_container_move_assignment is true\n void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::true_type ) {\n internal_move(std::move(other));\n }\n\n // Move assigns the segment table to other is any instances of allocator_type are not always equal\n // and propagate_on_container_move_assignment is false\n void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::false_type ) {\n if (my_segment_table_allocator == other.my_segment_table_allocator) {\n // If allocators are equal - restore pointers\n internal_move(std::move(other));\n } else {\n // If allocators are not equal - perform per element move with reallocation\n internal_transfer(other, move_segment_body_type{*this});\n }\n }\n\n // Swaps two segment tables if any instances of allocator_type are always equal\n // or propagate_on_container_swap is true\n void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::true_type ) {\n internal_swap_fields(other);\n }\n\n // Swaps two segment tables if any instances of allocator_type are not always equal\n // and propagate_on_container_swap is false\n // According to the C++ standard, swapping of two containers with unequal allocators\n // is an undefined behavior scenario\n void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::false_type ) {\n __TBB_ASSERT(my_segment_table_allocator == other.my_segment_table_allocator,\n \"Swapping with unequal allocators is not allowed\");\n internal_swap_fields(other);\n }\n\n void internal_swap_fields( segment_table& other ) {\n // If an active table in either *this segment table or other is an embedded one - swaps the embedded tables\n if (get_table() == my_embedded_table ||\n other.get_table() == other.my_embedded_table) {\n\n for (size_type i = 0; i != pointers_per_embedded_table; ++i) {\n segment_type current_segment = my_embedded_table[i].load(std::memory_order_relaxed);\n segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed);\n\n my_embedded_table[i].store(other_segment, std::memory_order_relaxed);\n other.my_embedded_table[i].store(current_segment, std::memory_order_relaxed);\n }\n }\n\n segment_table_type current_segment_table = get_table();\n segment_table_type other_segment_table = other.get_table();\n\n // If an active table is an embedded one -\n // store an active table in other to the embedded one from other\n if (current_segment_table == my_embedded_table) {\n other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed);\n } else {\n // Otherwise - store it to the active segment table\n other.my_segment_table.store(current_segment_table, std::memory_order_relaxed);\n }\n\n // If an active table in other segment table is an embedded one -\n // store an active table in other to the embedded one from *this\n if (other_segment_table == other.my_embedded_table) {\n my_segment_table.store(my_embedded_table, std::memory_order_relaxed);\n } else {\n // Otherwise - store it to the active segment table in other\n my_segment_table.store(other_segment_table, std::memory_order_relaxed);\n }\n auto first_block = other.my_first_block.load(std::memory_order_relaxed);\n other.my_first_block.store(my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_first_block.store(first_block, std::memory_order_relaxed);\n\n auto size = other.my_size.load(std::memory_order_relaxed);\n other.my_size.store(my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);\n my_size.store(size, std::memory_order_relaxed);\n }\n\nprotected:\n // A flag indicates that an exception was throws during segment allocations\n const segment_type segment_allocation_failure_tag = reinterpret_cast(1);\n static constexpr size_type embedded_table_size = segment_size(pointers_per_embedded_table);\n\n template \n value_type& internal_subscript( size_type index ) {\n segment_index_type seg_index = segment_index_of(index);\n segment_table_type table = my_segment_table.load(std::memory_order_acquire);\n segment_type segment = nullptr;\n\n if (allow_out_of_range_access) {\n if (derived_type::allow_table_extending) {\n extend_table_if_necessary(table, index, index + 1);\n }\n\n segment = table[seg_index].load(std::memory_order_acquire);\n // If the required segment is disabled - enable it\n if (segment == nullptr) {\n enable_segment(segment, table, seg_index, index);\n }\n // Check if an exception was thrown during segment allocation\n if (segment == segment_allocation_failure_tag) {\n throw_exception(exception_id::bad_alloc);\n }\n } else {\n segment = table[seg_index].load(std::memory_order_acquire);\n }\n __TBB_ASSERT(segment != nullptr, nullptr);\n\n return segment[index];\n }\n\n void assign_first_block_if_necessary(segment_index_type index) {\n size_type zero = 0;\n if (this->my_first_block.load(std::memory_order_relaxed) == zero) {\n this->my_first_block.compare_exchange_strong(zero, index);\n }\n }\n\n void zero_table( segment_table_type table, size_type count ) {\n for (size_type i = 0; i != count; ++i) {\n table[i].store(nullptr, std::memory_order_relaxed);\n }\n }\n\n segment_table_type get_table() const {\n return my_segment_table.load(std::memory_order_acquire);\n }\n\n segment_table_allocator_type my_segment_table_allocator;\n std::atomic my_segment_table;\n atomic_segment my_embedded_table[pointers_per_embedded_table];\n // Number of segments in first block\n std::atomic my_first_block;\n // Number of elements in table\n std::atomic my_size;\n // Flag to indicate failed extend table\n std::atomic my_segment_table_allocation_failed;\n}; // class segment_table\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n// #pragma warning(pop) // warning 4127 is back\n#endif\n\n#endif // __TBB_detail__segment_table_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_small_object_pool.h", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__small_object_pool_H\n#define __TBB__small_object_pool_H\n\n#include \"_config.h\"\n#include \"_assert.h\"\n\n#include \"../profiling.h\"\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass small_object_pool {\nprotected:\n small_object_pool() = default;\n};\nstruct execution_data;\n}\n\nnamespace r1 {\nTBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes,\n const d1::execution_data& ed);\nTBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes);\nTBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes,\n const d1::execution_data& ed);\nTBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes);\n}\n\nnamespace d1 {\nclass small_object_allocator {\npublic:\n template \n Type* new_object(execution_data& ed, Args&&... args) {\n void* allocated_object = r1::allocate(m_pool, sizeof(Type), ed);\n\n auto constructed_object = new(allocated_object) Type(std::forward(args)...);\n return constructed_object;\n }\n\n template \n Type* new_object(Args&&... args) {\n void* allocated_object = r1::allocate(m_pool, sizeof(Type));\n\n auto constructed_object = new(allocated_object) Type(std::forward(args)...);\n return constructed_object;\n }\n\n template \n void delete_object(Type* object, const execution_data& ed) {\n // Copy this since it can be a member of the passed object and\n // unintentionally destroyed when Type destructor is called below\n small_object_allocator alloc = *this;\n object->~Type();\n alloc.deallocate(object, ed);\n }\n\n template \n void delete_object(Type* object) {\n // Copy this since it can be a member of the passed object and\n // unintentionally destroyed when Type destructor is called below\n small_object_allocator alloc = *this;\n object->~Type();\n alloc.deallocate(object);\n }\n\n template \n void deallocate(Type* ptr, const execution_data& ed) {\n call_itt_task_notify(destroy, ptr);\n\n __TBB_ASSERT(m_pool != nullptr, \"Pool must be valid for deallocate call\");\n r1::deallocate(*m_pool, ptr, sizeof(Type), ed);\n }\n\n template \n void deallocate(Type* ptr) {\n call_itt_task_notify(destroy, ptr);\n\n __TBB_ASSERT(m_pool != nullptr, \"Pool must be valid for deallocate call\");\n r1::deallocate(*m_pool, ptr, sizeof(Type));\n }\nprivate:\n small_object_pool* m_pool{};\n};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB__small_object_pool_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_string_resource.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\nTBB_STRING_RESOURCE(ALGORITHM, \"tbb_algorithm\")\nTBB_STRING_RESOURCE(PARALLEL_FOR, \"tbb_parallel_for\")\nTBB_STRING_RESOURCE(PARALLEL_FOR_EACH, \"tbb_parallel_for_each\")\nTBB_STRING_RESOURCE(PARALLEL_INVOKE, \"tbb_parallel_invoke\")\nTBB_STRING_RESOURCE(PARALLEL_REDUCE, \"tbb_parallel_reduce\")\nTBB_STRING_RESOURCE(PARALLEL_SCAN, \"tbb_parallel_scan\")\nTBB_STRING_RESOURCE(PARALLEL_SORT, \"tbb_parallel_sort\")\nTBB_STRING_RESOURCE(PARALLEL_PIPELINE, \"tbb_parallel_pipeline\")\nTBB_STRING_RESOURCE(CUSTOM_CTX, \"tbb_custom\")\n\nTBB_STRING_RESOURCE(FLOW_NULL, \"null\")\nTBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, \"broadcast_node\")\nTBB_STRING_RESOURCE(FLOW_BUFFER_NODE, \"buffer_node\")\nTBB_STRING_RESOURCE(FLOW_CONTINUE_NODE, \"continue_node\")\nTBB_STRING_RESOURCE(FLOW_FUNCTION_NODE, \"function_node\")\nTBB_STRING_RESOURCE(FLOW_JOIN_NODE_QUEUEING, \"join_node (queueing)\")\nTBB_STRING_RESOURCE(FLOW_JOIN_NODE_RESERVING, \"join_node (reserving)\")\nTBB_STRING_RESOURCE(FLOW_JOIN_NODE_TAG_MATCHING, \"join_node (tag_matching)\")\nTBB_STRING_RESOURCE(FLOW_LIMITER_NODE, \"limiter_node\")\nTBB_STRING_RESOURCE(FLOW_MULTIFUNCTION_NODE, \"multifunction_node\")\nTBB_STRING_RESOURCE(FLOW_OVERWRITE_NODE, \"overwrite_node\")\nTBB_STRING_RESOURCE(FLOW_PRIORITY_QUEUE_NODE, \"priority_queue_node\")\nTBB_STRING_RESOURCE(FLOW_QUEUE_NODE, \"queue_node\")\nTBB_STRING_RESOURCE(FLOW_SEQUENCER_NODE, \"sequencer_node\")\nTBB_STRING_RESOURCE(FLOW_INPUT_NODE, \"input_node\")\nTBB_STRING_RESOURCE(FLOW_SPLIT_NODE, \"split_node\")\nTBB_STRING_RESOURCE(FLOW_WRITE_ONCE_NODE, \"write_once_node\")\nTBB_STRING_RESOURCE(FLOW_INDEXER_NODE, \"indexer_node\")\nTBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, \"composite_node\")\nTBB_STRING_RESOURCE(FLOW_ASYNC_NODE, \"async_node\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT, \"input_port\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_0, \"input_port_0\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_1, \"input_port_1\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_2, \"input_port_2\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_3, \"input_port_3\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_4, \"input_port_4\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_5, \"input_port_5\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_6, \"input_port_6\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_7, \"input_port_7\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_8, \"input_port_8\")\nTBB_STRING_RESOURCE(FLOW_INPUT_PORT_9, \"input_port_9\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT, \"output_port\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_0, \"output_port_0\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_1, \"output_port_1\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_2, \"output_port_2\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_3, \"output_port_3\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_4, \"output_port_4\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_5, \"output_port_5\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_6, \"output_port_6\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_7, \"output_port_7\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_8, \"output_port_8\")\nTBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_9, \"output_port_9\")\nTBB_STRING_RESOURCE(FLOW_OBJECT_NAME, \"object_name\")\nTBB_STRING_RESOURCE(FLOW_BODY, \"body\")\nTBB_STRING_RESOURCE(FLOW_GRAPH, \"graph\")\nTBB_STRING_RESOURCE(FLOW_NODE, \"node\")\nTBB_STRING_RESOURCE(FLOW_TASKS, \"tbb_flow_graph\")\nTBB_STRING_RESOURCE(USER_EVENT, \"user_event\")\n\n#if __TBB_FLOW_TRACE_CODEPTR\nTBB_STRING_RESOURCE(CODE_ADDRESS, \"code_address\")\n#endif\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_task.h", "content": "/*\n Copyright (c) 2020-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB__task_H\n#define __TBB__task_H\n\n#include \"_config.h\"\n#include \"_assert.h\"\n#include \"_template_helpers.h\"\n#include \"_small_object_pool.h\"\n\n#include \"../profiling.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nusing slot_id = unsigned short;\nconstexpr slot_id no_slot = slot_id(~0);\nconstexpr slot_id any_slot = slot_id(~1);\n\nclass task;\nclass wait_context;\nclass task_group_context;\nstruct execution_data;\nclass wait_tree_vertex_interface;\nclass task_arena_base;\n}\n\nnamespace d2 {\nclass task_group;\nclass task_group_base;\n}\n\nnamespace r1 {\n//! Task spawn/wait entry points\nTBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx);\nTBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id);\nTBB_EXPORT void __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context&, d1::task_group_context& w_ctx);\nTBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::wait_context&, d1::task_group_context& ctx);\nTBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data*);\nTBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::task_arena_base&);\nTBB_EXPORT d1::task_group_context* __TBB_EXPORTED_FUNC current_context();\nTBB_EXPORT d1::wait_tree_vertex_interface* get_thread_reference_vertex(d1::wait_tree_vertex_interface* wc);\n\n// Do not place under __TBB_RESUMABLE_TASKS. It is a stub for unsupported platforms.\nstruct suspend_point_type;\nusing suspend_callback_type = void(*)(void*, suspend_point_type*);\n//! The resumable tasks entry points\nTBB_EXPORT void __TBB_EXPORTED_FUNC suspend(suspend_callback_type suspend_callback, void* user_callback);\nTBB_EXPORT void __TBB_EXPORTED_FUNC resume(suspend_point_type* tag);\nTBB_EXPORT suspend_point_type* __TBB_EXPORTED_FUNC current_suspend_point();\nTBB_EXPORT void __TBB_EXPORTED_FUNC notify_waiters(std::uintptr_t wait_ctx_addr);\n\nclass thread_data;\nclass task_dispatcher;\nclass external_waiter;\nstruct task_accessor;\nstruct task_arena_impl;\n} // namespace r1\n\nnamespace d1 {\n\nclass task_arena;\nusing suspend_point = r1::suspend_point_type*;\n\n#if __TBB_RESUMABLE_TASKS\ntemplate \nstatic void suspend_callback(void* user_callback, suspend_point sp) {\n // Copy user function to a new stack after the context switch to avoid a race when the previous\n // suspend point is resumed while the user_callback is being called.\n F user_callback_copy = *static_cast(user_callback);\n user_callback_copy(sp);\n}\n\ntemplate \nvoid suspend(F f) {\n r1::suspend(&suspend_callback, &f);\n}\n\ninline void resume(suspend_point tag) {\n r1::resume(tag);\n}\n#endif /* __TBB_RESUMABLE_TASKS */\n\n// TODO align wait_context on cache lane\nclass wait_context {\n static constexpr std::uint64_t overflow_mask = ~((1LLU << 32) - 1);\n\n std::uint64_t m_version_and_traits{1};\n std::atomic m_ref_count{};\n\n void add_reference(std::int64_t delta) {\n call_itt_task_notify(releasing, this);\n std::uint64_t r = m_ref_count.fetch_add(static_cast(delta)) + static_cast(delta);\n\n __TBB_ASSERT_EX((r & overflow_mask) == 0, \"Overflow is detected\");\n\n if (!r) {\n // Some external waiters or coroutine waiters sleep in wait list\n // Should to notify them that work is done\n std::uintptr_t wait_ctx_addr = std::uintptr_t(this);\n r1::notify_waiters(wait_ctx_addr);\n }\n }\n\npublic:\n bool continue_execution() const {\n std::uint64_t r = m_ref_count.load(std::memory_order_acquire);\n __TBB_ASSERT_EX((r & overflow_mask) == 0, \"Overflow is detected\");\n return r > 0;\n }\n \nprivate:\n friend class r1::thread_data;\n friend class r1::task_dispatcher;\n friend class r1::external_waiter;\n friend class wait_context_vertex;\n friend struct r1::task_arena_impl;\n friend struct r1::suspend_point_type;\npublic:\n // Despite the internal reference count is uin64_t we limit the user interface with uint32_t\n // to preserve a part of the internal reference count for special needs.\n wait_context(std::uint32_t ref_count) : m_ref_count{ref_count} { suppress_unused_warning(m_version_and_traits); }\n wait_context(const wait_context&) = delete;\n\n ~wait_context() {\n __TBB_ASSERT(!continue_execution(), nullptr);\n }\n\n void reserve(std::uint32_t delta = 1) {\n add_reference(delta);\n }\n\n void release(std::uint32_t delta = 1) {\n add_reference(-std::int64_t(delta));\n }\n};\n\nclass wait_tree_vertex_interface {\npublic:\n virtual void reserve(std::uint32_t delta = 1) = 0;\n virtual void release(std::uint32_t delta = 1) = 0;\n\nprotected:\n virtual ~wait_tree_vertex_interface() = default;\n};\n\nclass wait_context_vertex : public wait_tree_vertex_interface {\npublic:\n wait_context_vertex(std::uint32_t ref = 0) : m_wait(ref) {}\n\n void reserve(std::uint32_t delta = 1) override {\n m_wait.reserve(delta);\n }\n\n void release(std::uint32_t delta = 1) override {\n m_wait.release(delta);\n }\n\n wait_context& get_context() {\n return m_wait;\n }\nprivate:\n friend class d2::task_group;\n friend class d2::task_group_base;\n\n bool continue_execution() const {\n return m_wait.continue_execution();\n }\n\n wait_context m_wait;\n};\n\nclass reference_vertex : public wait_tree_vertex_interface {\npublic:\n reference_vertex(wait_tree_vertex_interface* parent, std::uint32_t ref_count) : my_parent{parent}, m_ref_count{ref_count}\n {}\n\n void reserve(std::uint32_t delta = 1) override {\n if (m_ref_count.fetch_add(static_cast(delta)) == 0) {\n my_parent->reserve();\n }\n }\n\n void release(std::uint32_t delta = 1) override {\n std::uint64_t ref = m_ref_count.fetch_sub(static_cast(delta)) - static_cast(delta);\n if (ref == 0) {\n my_parent->release();\n }\n }\n\n std::uint32_t get_num_child() {\n return static_cast(m_ref_count.load(std::memory_order_acquire));\n }\nprivate:\n wait_tree_vertex_interface* my_parent;\n std::atomic m_ref_count;\n};\n\nstruct execution_data {\n task_group_context* context{};\n slot_id original_slot{};\n slot_id affinity_slot{};\n};\n\ninline task_group_context* context(const execution_data& ed) {\n return ed.context;\n}\n\ninline slot_id original_slot(const execution_data& ed) {\n return ed.original_slot;\n}\n\ninline slot_id affinity_slot(const execution_data& ed) {\n return ed.affinity_slot;\n}\n\ninline slot_id execution_slot(const execution_data& ed) {\n return r1::execution_slot(&ed);\n}\n\ninline bool is_same_affinity(const execution_data& ed) {\n return affinity_slot(ed) == no_slot || affinity_slot(ed) == execution_slot(ed);\n}\n\ninline bool is_stolen(const execution_data& ed) {\n return original_slot(ed) != execution_slot(ed);\n}\n\ninline void spawn(task& t, task_group_context& ctx) {\n call_itt_task_notify(releasing, &t);\n r1::spawn(t, ctx);\n}\n\ninline void spawn(task& t, task_group_context& ctx, slot_id id) {\n call_itt_task_notify(releasing, &t);\n r1::spawn(t, ctx, id);\n}\n\ninline void execute_and_wait(task& t, task_group_context& t_ctx, wait_context& wait_ctx, task_group_context& w_ctx) {\n r1::execute_and_wait(t, t_ctx, wait_ctx, w_ctx);\n call_itt_task_notify(acquired, &wait_ctx);\n call_itt_task_notify(destroy, &wait_ctx);\n}\n\ninline void wait(wait_context& wait_ctx, task_group_context& ctx) {\n r1::wait(wait_ctx, ctx);\n call_itt_task_notify(acquired, &wait_ctx);\n call_itt_task_notify(destroy, &wait_ctx);\n}\n\nusing r1::current_context;\n\nclass task_traits {\n std::uint64_t m_version_and_traits{};\n friend struct r1::task_accessor;\n};\n\n//! Alignment for a task object\nstatic constexpr std::size_t task_alignment = 64;\n\n//! Base class for user-defined tasks.\n/** @ingroup task_scheduling */\nclass alignas(task_alignment) task : public task_traits {\nprotected:\n virtual ~task() = default;\n\npublic:\n virtual task* execute(execution_data&) = 0;\n virtual task* cancel(execution_data&) = 0;\n\nprivate:\n std::uint64_t m_reserved[6]{};\n friend struct r1::task_accessor;\n};\nstatic_assert(sizeof(task) == task_alignment, \"task size is broken\");\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB__task_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_task_handle.h", "content": "/*\n Copyright (c) 2020-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n\n#ifndef __TBB_task_handle_H\n#define __TBB_task_handle_H\n\n#include \"_config.h\"\n#include \"_task.h\"\n#include \"_small_object_pool.h\"\n#include \"_utils.h\"\n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 { class task_group_context; class wait_context; struct execution_data; }\nnamespace d2 {\n\nclass task_handle;\n\nclass task_handle_task : public d1::task {\n std::uint64_t m_version_and_traits{};\n d1::wait_tree_vertex_interface* m_wait_tree_vertex;\n d1::task_group_context& m_ctx;\n d1::small_object_allocator m_allocator;\npublic:\n void finalize(const d1::execution_data* ed = nullptr) {\n if (ed) {\n m_allocator.delete_object(this, *ed);\n } else {\n m_allocator.delete_object(this);\n }\n }\n\n task_handle_task(d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, d1::small_object_allocator& alloc)\n : m_wait_tree_vertex(vertex)\n , m_ctx(ctx)\n , m_allocator(alloc) {\n suppress_unused_warning(m_version_and_traits);\n m_wait_tree_vertex->reserve();\n }\n\n ~task_handle_task() override {\n m_wait_tree_vertex->release();\n }\n\n d1::task_group_context& ctx() const { return m_ctx; }\n};\n\n\nclass task_handle {\n struct task_handle_task_finalizer_t{\n void operator()(task_handle_task* p){ p->finalize(); }\n };\n using handle_impl_t = std::unique_ptr;\n\n handle_impl_t m_handle = {nullptr};\npublic:\n task_handle() = default;\n task_handle(task_handle&&) = default;\n task_handle& operator=(task_handle&&) = default;\n\n explicit operator bool() const noexcept { return static_cast(m_handle); }\n\n friend bool operator==(task_handle const& th, std::nullptr_t) noexcept;\n friend bool operator==(std::nullptr_t, task_handle const& th) noexcept;\n\n friend bool operator!=(task_handle const& th, std::nullptr_t) noexcept;\n friend bool operator!=(std::nullptr_t, task_handle const& th) noexcept;\n\nprivate:\n friend struct task_handle_accessor;\n\n task_handle(task_handle_task* t) : m_handle {t}{};\n\n d1::task* release() {\n return m_handle.release();\n }\n};\n\nstruct task_handle_accessor {\nstatic task_handle construct(task_handle_task* t) { return {t}; }\nstatic d1::task* release(task_handle& th) { return th.release(); }\nstatic d1::task_group_context& ctx_of(task_handle& th) {\n __TBB_ASSERT(th.m_handle, \"ctx_of does not expect empty task_handle.\");\n return th.m_handle->ctx();\n}\n};\n\ninline bool operator==(task_handle const& th, std::nullptr_t) noexcept {\n return th.m_handle == nullptr;\n}\ninline bool operator==(std::nullptr_t, task_handle const& th) noexcept {\n return th.m_handle == nullptr;\n}\n\ninline bool operator!=(task_handle const& th, std::nullptr_t) noexcept {\n return th.m_handle != nullptr;\n}\n\ninline bool operator!=(std::nullptr_t, task_handle const& th) noexcept {\n return th.m_handle != nullptr;\n}\n\n} // namespace d2\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_task_handle_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_template_helpers.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__template_helpers_H\n#define __TBB_detail__template_helpers_H\n\n#include \"_utils.h\"\n#include \"_config.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n// An internal implementation of void_t, which can be used in SFINAE contexts\ntemplate \nstruct void_impl {\n using type = void;\n}; // struct void_impl\n\ntemplate \nusing void_t = typename void_impl::type;\n\n// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502\ntemplate class... Checks>\nstruct supports_impl {\n using type = std::false_type;\n};\n\ntemplate class... Checks>\nstruct supports_impl...>, Checks...> {\n using type = std::true_type;\n};\n\ntemplate class... Checks>\nusing supports = typename supports_impl::type;\n\n//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size.\ntemplate \nstruct select_size_t_constant {\n // Explicit cast is needed to avoid compiler warnings about possible truncation.\n // The value of the right size, which is selected by ?:, is anyway not truncated or promoted.\n static const std::size_t value = static_cast((sizeof(std::size_t)==sizeof(u)) ? u : ull);\n};\n\n// TODO: do we really need it?\n//! Cast between unrelated pointer types.\n/** This method should be used sparingly as a last resort for dealing with\n situations that inherently break strict ISO C++ aliasing rules. */\n// T is a pointer type because it will be explicitly provided by the programmer as a template argument;\n// U is a referent type to enable the compiler to check that \"ptr\" is a pointer, deducing U in the process.\ntemplate\ninline T punned_cast( U* ptr ) {\n std::uintptr_t x = reinterpret_cast(ptr);\n return reinterpret_cast(x);\n}\n\ntemplate\nstruct padded_base : T {\n char pad[S - R];\n};\ntemplate struct padded_base : T {};\n\n//! Pads type T to fill out to a multiple of cache line size.\ntemplate\nstruct padded : padded_base {};\n\n#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT\n\nusing std::index_sequence;\nusing std::make_index_sequence;\n\n#else\n\ntemplate class index_sequence {};\n\ntemplate\nstruct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {};\n\ntemplate\nstruct make_index_sequence_impl <0, S...> {\n using type = index_sequence;\n};\n\ntemplate\nusing make_index_sequence = typename make_index_sequence_impl::type;\n\n#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */\n\n#if __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT\nusing std::conjunction;\nusing std::disjunction;\n#else // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT\n\ntemplate \nstruct conjunction : std::true_type {};\n\ntemplate \nstruct conjunction\n : std::conditional, First>::type {};\n\ntemplate \nstruct conjunction : T {};\n\ntemplate \nstruct disjunction : std::false_type {};\n\ntemplate \nstruct disjunction\n : std::conditional>::type {};\n\ntemplate \nstruct disjunction : T {};\n\n#endif // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT\n\ntemplate \nusing iterator_value_t = typename std::iterator_traits::value_type;\n\ntemplate \nusing iterator_key_t = typename std::remove_const::first_type>::type;\n\ntemplate \nusing iterator_mapped_t = typename iterator_value_t::second_type;\n\ntemplate \nusing iterator_alloc_pair_t = std::pair>::type,\n iterator_mapped_t>;\n\ntemplate using alloc_value_type = typename A::value_type;\ntemplate using alloc_ptr_t = typename std::allocator_traits::pointer;\ntemplate using has_allocate = decltype(std::declval&>() = std::declval().allocate(0));\ntemplate using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0));\n\n// alloc_value_type should be checked first, because it can be used in other checks\ntemplate \nusing is_allocator = supports;\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate \ninline constexpr bool is_allocator_v = is_allocator::value;\n#endif\n\n// Template class in which the \"type\" determines the type of the element number N in pack Args\ntemplate \nstruct pack_element {\n using type = void;\n};\n\ntemplate \nstruct pack_element {\n using type = typename pack_element::type;\n};\n\ntemplate \nstruct pack_element<0, T, Args...> {\n using type = T;\n};\n\ntemplate \nusing pack_element_t = typename pack_element::type;\n\ntemplate \nclass raii_guard {\npublic:\n static_assert(\n std::is_nothrow_copy_constructible::value &&\n std::is_nothrow_move_constructible::value,\n \"Throwing an exception during the Func copy or move construction cause an unexpected behavior.\"\n );\n\n raii_guard( Func f ) noexcept : my_func(f), is_active(true) {}\n\n raii_guard( raii_guard&& g ) noexcept : my_func(std::move(g.my_func)), is_active(g.is_active) {\n g.is_active = false;\n }\n\n ~raii_guard() {\n if (is_active) {\n my_func();\n }\n }\n\n void dismiss() {\n is_active = false;\n }\nprivate:\n Func my_func;\n bool is_active;\n}; // class raii_guard\n\ntemplate \nraii_guard make_raii_guard( Func f ) {\n return raii_guard(f);\n}\n\ntemplate \nstruct try_call_proxy {\n try_call_proxy( Body b ) : body(b) {}\n\n template \n void on_exception( OnExceptionBody on_exception_body ) {\n auto guard = make_raii_guard(on_exception_body);\n body();\n guard.dismiss();\n }\n\n template \n void on_completion(OnCompletionBody on_completion_body) {\n auto guard = make_raii_guard(on_completion_body);\n body();\n }\n\n Body body;\n}; // struct try_call_proxy\n\n// Template helper function for API\n// try_call(lambda1).on_exception(lambda2)\n// Executes lambda1 and if it throws an exception - executes lambda2\ntemplate \ntry_call_proxy try_call( Body b ) {\n return try_call_proxy(b);\n}\n\n#if __TBB_CPP17_IS_SWAPPABLE_PRESENT\nusing std::is_nothrow_swappable;\nusing std::is_swappable;\n#else // __TBB_CPP17_IS_SWAPPABLE_PRESENT\nnamespace is_swappable_detail {\nusing std::swap;\n\ntemplate \nusing has_swap = decltype(swap(std::declval(), std::declval()));\n\n#if _MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER\n// Workaround for VS2015: it fails to instantiate noexcept(...) inside std::integral_constant.\ntemplate \nstruct noexcept_wrapper {\n static const bool value = noexcept(swap(std::declval(), std::declval()));\n};\ntemplate \nstruct is_nothrow_swappable_impl : std::integral_constant::value> {};\n#else\ntemplate \nstruct is_nothrow_swappable_impl : std::integral_constant(), std::declval()))> {};\n#endif\n}\n\ntemplate \nstruct is_swappable : supports {};\n\ntemplate \nstruct is_nothrow_swappable\n : conjunction, is_swappable_detail::is_nothrow_swappable_impl> {};\n#endif // __TBB_CPP17_IS_SWAPPABLE_PRESENT\n\n//! Allows to store a function parameter pack as a variable and later pass it to another function\ntemplate< typename... Types >\nstruct stored_pack;\n\ntemplate<>\nstruct stored_pack<>\n{\n using pack_type = stored_pack<>;\n stored_pack() {}\n\n // Friend front-end functions\n template< typename F, typename Pack > friend void call(F&& f, Pack&& p);\n template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p);\n\nprotected:\n // Ideally, ref-qualified non-static methods would be used,\n // but that would greatly reduce the set of compilers where it works.\n template< typename Ret, typename F, typename... Preceding >\n static Ret call(F&& f, const pack_type& /*pack*/, Preceding&&... params) {\n return std::forward(f)(std::forward(params)...);\n }\n template< typename Ret, typename F, typename... Preceding >\n static Ret call(F&& f, pack_type&& /*pack*/, Preceding&&... params) {\n return std::forward(f)(std::forward(params)...);\n }\n};\n\ntemplate< typename T, typename... Types >\nstruct stored_pack : stored_pack\n{\n using pack_type = stored_pack;\n using pack_remainder = stored_pack;\n\n // Since lifetime of original values is out of control, copies should be made.\n // Thus references should be stripped away from the deduced type.\n typename std::decay::type leftmost_value;\n\n // Here rvalue references act in the same way as forwarding references,\n // as long as class template parameters were deduced via forwarding references.\n stored_pack(T&& t, Types&&... types)\n : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {}\n\n // Friend front-end functions\n template< typename F, typename Pack > friend void call(F&& f, Pack&& p);\n template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p);\n\nprotected:\n template< typename Ret, typename F, typename... Preceding >\n static Ret call(F&& f, pack_type& pack, Preceding&&... params) {\n return pack_remainder::template call(\n std::forward(f), static_cast(pack),\n std::forward(params)... , pack.leftmost_value\n );\n }\n\n template< typename Ret, typename F, typename... Preceding >\n static Ret call(F&& f, pack_type&& pack, Preceding&&... params) {\n return pack_remainder::template call(\n std::forward(f), static_cast(pack),\n std::forward(params)... , std::move(pack.leftmost_value)\n );\n }\n};\n\n//! Calls the given function with arguments taken from a stored_pack\ntemplate< typename F, typename Pack >\nvoid call(F&& f, Pack&& p) {\n std::decay::type::template call(std::forward(f), std::forward(p));\n}\n\ntemplate< typename Ret, typename F, typename Pack >\nRet call_and_return(F&& f, Pack&& p) {\n return std::decay::type::template call(std::forward(f), std::forward(p));\n}\n\ntemplate< typename... Types >\nstored_pack save_pack(Types&&... types) {\n return stored_pack(std::forward(types)...);\n}\n\n// A structure with the value which is equal to Trait::value\n// but can be used in the immediate context due to parameter T\ntemplate \nstruct dependent_bool : std::integral_constant {};\n\ntemplate \nstruct body_arg_detector;\n\ntemplate \nstruct body_arg_detector {\n using arg_type = Arg;\n};\n\ntemplate \nstruct body_arg_detector {\n using arg_type = Arg;\n};\n\ntemplate \nstruct argument_detector;\n\ntemplate \nstruct argument_detector {\n using type = typename body_arg_detector::arg_type;\n};\n\ntemplate \nstruct argument_detector {\n using type = Arg;\n};\n\n// Detects the argument type of callable, works for callable with one argument.\ntemplate \nusing argument_type_of = typename argument_detector::type>::type;\n\ntemplate \nstruct type_identity {\n using type = T;\n};\n\ntemplate \nusing type_identity_t = typename type_identity::type;\n\n} // inline namespace d0\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__template_helpers_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_utils.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__utils_H\n#define __TBB_detail__utils_H\n\n#include \n#include \n#include \n#include \n\n#include \"_config.h\"\n#include \"_assert.h\"\n#include \"_machine.h\"\n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n\n//! Utility template function to prevent \"unused\" warnings by various compilers.\ntemplate void suppress_unused_warning(T&&...) {}\n\n//! Compile-time constant that is upper bound on cache line/sector size.\n/** It should be used only in situations where having a compile-time upper\n bound is more useful than a run-time exact answer.\n @ingroup memory_allocation */\nconstexpr size_t max_nfs_size = 128;\nconstexpr std::size_t max_nfs_size_exp = 7;\nstatic_assert(1 << max_nfs_size_exp == max_nfs_size, \"max_nfs_size_exp must be a log2(max_nfs_size)\");\n\n//! Class that implements exponential backoff.\nclass atomic_backoff {\n //! Time delay, in units of \"pause\" instructions.\n /** Should be equal to approximately the number of \"pause\" instructions\n that take the same time as an context switch. Must be a power of two.*/\n static constexpr std::int32_t LOOPS_BEFORE_YIELD = 16;\n std::int32_t count;\n\npublic:\n // In many cases, an object of this type is initialized eagerly on hot path,\n // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ }\n // For this reason, the construction cost must be very small!\n atomic_backoff() : count(1) {}\n // This constructor pauses immediately; do not use on hot paths!\n atomic_backoff(bool) : count(1) { pause(); }\n\n //! No Copy\n atomic_backoff(const atomic_backoff&) = delete;\n atomic_backoff& operator=(const atomic_backoff&) = delete;\n\n //! Pause for a while.\n void pause() {\n if (count <= LOOPS_BEFORE_YIELD) {\n machine_pause(count);\n // Pause twice as long the next time.\n count *= 2;\n } else {\n // Pause is so long that we might as well yield CPU to scheduler.\n yield();\n }\n }\n\n //! Pause for a few times and return false if saturated.\n bool bounded_pause() {\n machine_pause(count);\n if (count < LOOPS_BEFORE_YIELD) {\n // Pause twice as long the next time.\n count *= 2;\n return true;\n } else {\n return false;\n }\n }\n\n void reset() {\n count = 1;\n }\n};\n\n//! Spin WHILE the condition is true.\n/** T and U should be comparable types. */\ntemplate \nT spin_wait_while(const std::atomic& location, C comp, std::memory_order order) {\n atomic_backoff backoff;\n T snapshot = location.load(order);\n while (comp(snapshot)) {\n backoff.pause();\n snapshot = location.load(order);\n }\n return snapshot;\n}\n\n//! Spin WHILE the value of the variable is equal to a given value\n/** T and U should be comparable types. */\ntemplate \nT spin_wait_while_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) {\n return spin_wait_while(location, [&value](T t) { return t == value; }, order);\n}\n\n//! Spin UNTIL the value of the variable is equal to a given value\n/** T and U should be comparable types. */\ntemplate\nT spin_wait_until_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) {\n return spin_wait_while(location, [&value](T t) { return t != value; }, order);\n}\n\n//! Spin UNTIL the condition returns true or spinning time is up.\n/** Returns what the passed functor returned last time it was invoked. */\ntemplate \nbool timed_spin_wait_until(Condition condition) {\n // 32 pauses + 32 yields are meausered as balanced spin time before sleep.\n bool finish = condition();\n for (int i = 1; !finish && i < 32; finish = condition(), i *= 2) {\n machine_pause(i);\n }\n for (int i = 32; !finish && i < 64; finish = condition(), ++i) {\n yield();\n }\n return finish;\n}\n\ntemplate \nT clamp(T value, T lower_bound, T upper_bound) {\n __TBB_ASSERT(lower_bound <= upper_bound, \"Incorrect bounds\");\n return value > lower_bound ? (value > upper_bound ? upper_bound : value) : lower_bound;\n}\n\ntemplate \nstd::uintptr_t log2(T in) {\n __TBB_ASSERT(in > 0, \"The logarithm of a non-positive value is undefined.\");\n return machine_log2(in);\n}\n\ntemplate\nT reverse_bits(T src) {\n return machine_reverse_bits(src);\n}\n\ntemplate\nT reverse_n_bits(T src, std::size_t n) {\n __TBB_ASSERT(n != 0, \"Reverse for 0 bits is undefined behavior.\");\n return reverse_bits(src) >> (number_of_bits() - n);\n}\n\n// A function to check if passed integer is a power of two\ntemplate \nconstexpr bool is_power_of_two( IntegerType arg ) {\n static_assert(std::is_integral::value,\n \"An argument for is_power_of_two should be integral type\");\n return arg && (0 == (arg & (arg - 1)));\n}\n\n// A function to determine if passed integer is a power of two\n// at least as big as another power of two, i.e. for strictly positive i and j,\n// with j being a power of two, determines whether i==j<\nconstexpr bool is_power_of_two_at_least(ArgIntegerType arg, DivisorIntegerType divisor) {\n // Divisor should be a power of two\n static_assert(std::is_integral::value,\n \"An argument for is_power_of_two_at_least should be integral type\");\n return 0 == (arg & (arg - divisor));\n}\n\n// A function to compute arg modulo divisor where divisor is a power of 2.\ntemplate\ninline ArgIntegerType modulo_power_of_two(ArgIntegerType arg, DivisorIntegerType divisor) {\n __TBB_ASSERT( is_power_of_two(divisor), \"Divisor should be a power of two\" );\n return arg & (divisor - 1);\n}\n\n//! A function to check if passed in pointer is aligned on a specific border\ntemplate\nconstexpr bool is_aligned(T* pointer, std::uintptr_t alignment) {\n return 0 == (reinterpret_cast(pointer) & (alignment - 1));\n}\n\n#if TBB_USE_ASSERT\nstatic void* const poisoned_ptr = reinterpret_cast(-1);\n\n//! Set p to invalid pointer value.\ntemplate\ninline void poison_pointer( T* &p ) { p = reinterpret_cast(poisoned_ptr); }\n\ntemplate\ninline void poison_pointer(std::atomic& p) { p.store(reinterpret_cast(poisoned_ptr), std::memory_order_relaxed); }\n\n/** Expected to be used in assertions only, thus no empty form is defined. **/\ntemplate\ninline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); }\n\ntemplate\ninline bool is_poisoned(const std::atomic& p) { return is_poisoned(p.load(std::memory_order_relaxed)); }\n#else\ntemplate\ninline void poison_pointer(T&) {/*do nothing*/}\n#endif /* !TBB_USE_ASSERT */\n\ntemplate \nbool assert_pointer_valid(T* p, const char* comment = nullptr) {\n suppress_unused_warning(p, comment);\n __TBB_ASSERT(p != nullptr, comment);\n __TBB_ASSERT(!is_poisoned(p), comment);\n#if !(_MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER)\n __TBB_ASSERT(is_aligned(p, alignment == 0 ? alignof(T) : alignment), comment);\n#endif\n // Returns something to simplify assert_pointers_valid implementation.\n return true;\n}\n\ntemplate \nvoid assert_pointers_valid(Args*... p) {\n // suppress_unused_warning is used as an evaluation context for the variadic pack.\n suppress_unused_warning(assert_pointer_valid(p)...);\n}\n\n//! Base class for types that should not be assigned.\nclass no_assign {\npublic:\n void operator=(const no_assign&) = delete;\n no_assign(const no_assign&) = default;\n no_assign() = default;\n};\n\n//! Base class for types that should not be copied or assigned.\nclass no_copy: no_assign {\npublic:\n no_copy(const no_copy&) = delete;\n no_copy() = default;\n};\n\ntemplate \nvoid swap_atomics_relaxed(std::atomic& lhs, std::atomic& rhs){\n T tmp = lhs.load(std::memory_order_relaxed);\n lhs.store(rhs.load(std::memory_order_relaxed), std::memory_order_relaxed);\n rhs.store(tmp, std::memory_order_relaxed);\n}\n\n//! One-time initialization states\nenum class do_once_state {\n uninitialized = 0, ///< No execution attempts have been undertaken yet\n pending, ///< A thread is executing associated do-once routine\n executed, ///< Do-once routine has been executed\n initialized = executed ///< Convenience alias\n};\n\n//! One-time initialization function\n/** /param initializer Pointer to function without arguments\n The variant that returns bool is used for cases when initialization can fail\n and it is OK to continue execution, but the state should be reset so that\n the initialization attempt was repeated the next time.\n /param state Shared state associated with initializer that specifies its\n initialization state. Must be initially set to #uninitialized value\n (e.g. by means of default static zero initialization). **/\ntemplate \nvoid atomic_do_once( const F& initializer, std::atomic& state ) {\n // The loop in the implementation is necessary to avoid race when thread T2\n // that arrived in the middle of initialization attempt by another thread T1\n // has just made initialization possible.\n // In such a case T2 has to rely on T1 to initialize, but T1 may already be past\n // the point where it can recognize the changed conditions.\n do_once_state expected_state;\n while ( state.load( std::memory_order_acquire ) != do_once_state::executed ) {\n if( state.load( std::memory_order_relaxed ) == do_once_state::uninitialized ) {\n expected_state = do_once_state::uninitialized;\n#if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910\n using enum_type = typename std::underlying_type::type;\n if( ((std::atomic&)state).compare_exchange_strong( (enum_type&)expected_state, (enum_type)do_once_state::pending ) ) {\n#else\n if( state.compare_exchange_strong( expected_state, do_once_state::pending ) ) {\n#endif\n run_initializer( initializer, state );\n break;\n }\n }\n spin_wait_while_eq( state, do_once_state::pending );\n }\n}\n\n// Run the initializer which can not fail\ntemplate\nvoid run_initializer(const Functor& f, std::atomic& state ) {\n f();\n state.store(do_once_state::executed, std::memory_order_release);\n}\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \nconcept boolean_testable_impl = std::convertible_to;\n\ntemplate \nconcept boolean_testable = boolean_testable_impl && requires( T&& t ) {\n { !std::forward(t) } -> boolean_testable_impl;\n };\n\n#if __TBB_CPP20_COMPARISONS_PRESENT\nstruct synthesized_three_way_comparator {\n template \n auto operator()( const T1& lhs, const T2& rhs ) const\n requires requires {\n { lhs < rhs } -> boolean_testable;\n { rhs < lhs } -> boolean_testable;\n }\n {\n if constexpr (std::three_way_comparable_with) {\n return lhs <=> rhs;\n } else {\n if (lhs < rhs) {\n return std::weak_ordering::less;\n }\n if (rhs < lhs) {\n return std::weak_ordering::greater;\n }\n return std::weak_ordering::equivalent;\n }\n }\n}; // struct synthesized_three_way_comparator\n\ntemplate \nusing synthesized_three_way_result = decltype(synthesized_three_way_comparator{}(std::declval(),\n std::declval()));\n\n#endif // __TBB_CPP20_COMPARISONS_PRESENT\n\n// Check if the type T is implicitly OR explicitly convertible to U\ntemplate \nconcept relaxed_convertible_to = std::constructible_from;\n\ntemplate \nconcept adaptive_same_as =\n#if __TBB_STRICT_CONSTRAINTS\n std::same_as;\n#else\n std::convertible_to;\n#endif\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n\ntemplate \nauto invoke(F&& f, Args&&... args)\n#if __TBB_CPP17_INVOKE_PRESENT\n noexcept(std::is_nothrow_invocable_v)\n -> std::invoke_result_t\n{\n return std::invoke(std::forward(f), std::forward(args)...);\n}\n#else // __TBB_CPP17_INVOKE_PRESENT\n noexcept(noexcept(std::forward(f)(std::forward(args)...)))\n -> decltype(std::forward(f)(std::forward(args)...))\n{\n return std::forward(f)(std::forward(args)...);\n}\n#endif // __TBB_CPP17_INVOKE_PRESENT\n\n} // namespace d0\n\nnamespace d1 {\n\nclass delegate_base {\npublic:\n virtual bool operator()() const = 0;\n virtual ~delegate_base() {}\n};\n\ntemplate \nclass delegated_function : public delegate_base {\npublic:\n delegated_function(FuncType& f) : my_func(f) {}\n\n bool operator()() const override {\n return my_func();\n }\n\nprivate:\n FuncType &my_func;\n};\n} // namespace d1\n\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__utils_H\n" }, { "path": "src/tbb/include/oneapi/tbb/detail/_waitable_atomic.h", "content": "/*\n Copyright (c) 2021-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_detail__address_waiters_H\n#define __TBB_detail__address_waiters_H\n\n#include \"_utils.h\"\n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nTBB_EXPORT void __TBB_EXPORTED_FUNC wait_on_address(void* address, d1::delegate_base& wakeup_condition, std::uintptr_t context);\nTBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address(void* address, std::uintptr_t context);\nTBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_one(void* address);\nTBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_all(void* address);\n} // namespace r1\n\nnamespace d1 {\n\ntemplate \nvoid adaptive_wait_on_address(void* address, Predicate wakeup_condition, std::uintptr_t context) {\n if (!timed_spin_wait_until(wakeup_condition)) {\n d1::delegated_function pred(wakeup_condition);\n r1::wait_on_address(address, pred, context);\n }\n}\n\ntemplate \nclass waitable_atomic {\npublic:\n waitable_atomic() = default;\n\n explicit waitable_atomic(T value) : my_atomic(value) {}\n\n waitable_atomic(const waitable_atomic&) = delete;\n waitable_atomic& operator=(const waitable_atomic&) = delete;\n\n T load(std::memory_order order) const noexcept {\n return my_atomic.load(order);\n }\n\n T exchange(T desired) noexcept {\n return my_atomic.exchange(desired);\n }\n\n void wait(T old, std::uintptr_t context, std::memory_order order) {\n auto wakeup_condition = [&] { return my_atomic.load(order) != old; };\n if (!timed_spin_wait_until(wakeup_condition)) {\n // We need to use while here, because notify_all() will wake up all threads\n // But predicate for them might be false\n d1::delegated_function pred(wakeup_condition);\n do {\n r1::wait_on_address(this, pred, context);\n } while (!wakeup_condition());\n }\n }\n\n void notify_one_relaxed() {\n r1::notify_by_address_one(this);\n }\n\n // TODO: consider adding following interfaces:\n // store(desired, memory_order)\n // notify_all_relaxed()\n // wait_until(T, std::uintptr_t, std::memory_order)\n // notify_relaxed(std::uintptr_t context)\n\nprivate:\n std::atomic my_atomic{};\n};\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_detail__address_waiters_H\n" }, { "path": "src/tbb/include/oneapi/tbb/enumerable_thread_specific.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_enumerable_thread_specific_H\n#define __TBB_enumerable_thread_specific_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_aligned_space.h\"\n\n#include \"concurrent_vector.h\"\n#include \"tbb_allocator.h\"\n#include \"cache_aligned_allocator.h\"\n#include \"profiling.h\"\n\n#include \n#include \n#include // memcpy\n#include // std::ptrdiff_t\n\n#include \"task.h\" // for task::suspend_point\n\n#if _WIN32 || _WIN64\n#ifndef NOMINMAX\n#define NOMINMAX\n#define __TBB_DEFINED_NOMINMAX 1\n#endif\n#include \n#if __TBB_DEFINED_NOMINMAX\n#undef NOMINMAX\n#undef __TBB_DEFINED_NOMINMAX\n#endif\n#else\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! enum for selecting between single key and key-per-instance versions\nenum ets_key_usage_type {\n ets_key_per_instance\n , ets_no_key\n#if __TBB_RESUMABLE_TASKS\n , ets_suspend_aware\n#endif\n};\n\n// Forward declaration to use in internal classes\ntemplate \nclass enumerable_thread_specific;\n\ntemplate \nstruct internal_ets_key_selector {\n using key_type = std::thread::id;\n static key_type current_key() {\n return std::this_thread::get_id();\n }\n};\n\n// Intel Compiler on OSX cannot create atomics objects that instantiated from non-fundamental types\n#if __INTEL_COMPILER && __APPLE__\ntemplate<>\nstruct internal_ets_key_selector {\n using key_type = std::size_t;\n static key_type current_key() {\n auto id = std::this_thread::get_id();\n return reinterpret_cast(id);\n }\n};\n#endif\n\ntemplate \nstruct ets_key_selector : internal_ets_key_selector {};\n\n#if __TBB_RESUMABLE_TASKS\ntemplate <>\nstruct ets_key_selector {\n using key_type = suspend_point;\n static key_type current_key() {\n return r1::current_suspend_point();\n }\n};\n#endif\n\ntemplate\nclass ets_base : detail::no_copy {\nprotected:\n using key_type = typename ets_key_selector::key_type;\n\npublic:\n struct slot;\n struct array {\n array* next;\n std::size_t lg_size;\n slot& at( std::size_t k ) {\n return (reinterpret_cast(reinterpret_cast(this+1)))[k];\n }\n std::size_t size() const { return std::size_t(1) << lg_size; }\n std::size_t mask() const { return size() - 1; }\n std::size_t start( std::size_t h ) const {\n return h >> (8 * sizeof(std::size_t) - lg_size);\n }\n };\n struct slot {\n std::atomic key;\n void* ptr;\n bool empty() const { return key.load(std::memory_order_relaxed) == key_type(); }\n bool match( key_type k ) const { return key.load(std::memory_order_relaxed) == k; }\n bool claim( key_type k ) {\n // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size\n key_type expected = key_type();\n return key.compare_exchange_strong(expected, k);\n }\n };\n\nprotected:\n //! Root of linked list of arrays of decreasing size.\n /** nullptr if and only if my_count==0.\n Each array in the list is half the size of its predecessor. */\n std::atomic my_root;\n std::atomic my_count;\n\n virtual void* create_local() = 0;\n virtual void* create_array(std::size_t _size) = 0; // _size in bytes\n virtual void free_array(void* ptr, std::size_t _size) = 0; // _size in bytes\n\n array* allocate( std::size_t lg_size ) {\n std::size_t n = std::size_t(1) << lg_size;\n array* a = static_cast(create_array(sizeof(array) + n * sizeof(slot)));\n a->lg_size = lg_size;\n std::memset( a + 1, 0, n * sizeof(slot) );\n return a;\n }\n void deallocate(array* a) {\n std::size_t n = std::size_t(1) << (a->lg_size);\n free_array( static_cast(a), std::size_t(sizeof(array) + n * sizeof(slot)) );\n }\n\n ets_base() : my_root{nullptr}, my_count{0} {}\n virtual ~ets_base(); // g++ complains if this is not virtual\n\n void* table_lookup( bool& exists );\n void table_clear();\n // The following functions are not used in concurrent context,\n // so we don't need synchronization and ITT annotations there.\n template \n void table_elementwise_copy( const ets_base& other,\n void*(*add_element)(ets_base&, void*) ) {\n __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr);\n __TBB_ASSERT(!my_count.load(std::memory_order_relaxed), nullptr);\n if( !other.my_root.load(std::memory_order_relaxed) ) return;\n array* root = allocate(other.my_root.load(std::memory_order_relaxed)->lg_size);\n my_root.store(root, std::memory_order_relaxed);\n root->next = nullptr;\n my_count.store(other.my_count.load(std::memory_order_relaxed), std::memory_order_relaxed);\n std::size_t mask = root->mask();\n for( array* r = other.my_root.load(std::memory_order_relaxed); r; r = r->next ) {\n for( std::size_t i = 0; i < r->size(); ++i ) {\n slot& s1 = r->at(i);\n if( !s1.empty() ) {\n for( std::size_t j = root->start(std::hash{}(s1.key.load(std::memory_order_relaxed))); ; j = (j+1)&mask ) {\n slot& s2 = root->at(j);\n if( s2.empty() ) {\n s2.ptr = add_element(static_cast&>(*this), s1.ptr);\n s2.key.store(s1.key.load(std::memory_order_relaxed), std::memory_order_relaxed);\n break;\n }\n else if( s2.match(s1.key.load(std::memory_order_relaxed)) )\n break;\n }\n }\n }\n }\n }\n void table_swap( ets_base& other ) {\n __TBB_ASSERT(this!=&other, \"Don't swap an instance with itself\");\n swap_atomics_relaxed(my_root, other.my_root);\n swap_atomics_relaxed(my_count, other.my_count);\n }\n};\n\ntemplate\nets_base::~ets_base() {\n __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr);\n}\n\ntemplate\nvoid ets_base::table_clear() {\n while ( array* r = my_root.load(std::memory_order_relaxed) ) {\n my_root.store(r->next, std::memory_order_relaxed);\n deallocate(r);\n }\n my_count.store(0, std::memory_order_relaxed);\n}\n\ntemplate\nvoid* ets_base::table_lookup( bool& exists ) {\n const key_type k = ets_key_selector::current_key();\n\n __TBB_ASSERT(k != key_type(), nullptr);\n void* found;\n std::size_t h = std::hash{}(k);\n for( array* r = my_root.load(std::memory_order_acquire); r; r = r->next ) {\n call_itt_notify(acquired,r);\n std::size_t mask=r->mask();\n for(std::size_t i = r->start(h); ;i=(i+1)&mask) {\n slot& s = r->at(i);\n if( s.empty() ) break;\n if( s.match(k) ) {\n if( r == my_root.load(std::memory_order_acquire) ) {\n // Success at top level\n exists = true;\n return s.ptr;\n } else {\n // Success at some other level. Need to insert at top level.\n exists = true;\n found = s.ptr;\n goto insert;\n }\n }\n }\n }\n // Key does not yet exist. The density of slots in the table does not exceed 0.5,\n // for if this will occur a new table is allocated with double the current table\n // size, which is swapped in as the new root table. So an empty slot is guaranteed.\n exists = false;\n found = create_local();\n {\n std::size_t c = ++my_count;\n array* r = my_root.load(std::memory_order_acquire);\n call_itt_notify(acquired,r);\n if( !r || c > r->size()/2 ) {\n std::size_t s = r ? r->lg_size : 2;\n while( c > std::size_t(1)<<(s-1) ) ++s;\n array* a = allocate(s);\n for(;;) {\n a->next = r;\n call_itt_notify(releasing,a);\n array* new_r = r;\n if( my_root.compare_exchange_strong(new_r, a) ) break;\n call_itt_notify(acquired, new_r);\n __TBB_ASSERT(new_r != nullptr, nullptr);\n if( new_r->lg_size >= s ) {\n // Another thread inserted an equal or bigger array, so our array is superfluous.\n deallocate(a);\n break;\n }\n r = new_r;\n }\n }\n }\n insert:\n // Whether a slot has been found in an older table, or if it has been inserted at this level,\n // it has already been accounted for in the total. Guaranteed to be room for it, and it is\n // not present, so search for empty slot and use it.\n array* ir = my_root.load(std::memory_order_acquire);\n call_itt_notify(acquired, ir);\n std::size_t mask = ir->mask();\n for(std::size_t i = ir->start(h);; i = (i+1)&mask) {\n slot& s = ir->at(i);\n if( s.empty() ) {\n if( s.claim(k) ) {\n s.ptr = found;\n return found;\n }\n }\n }\n}\n\n//! Specialization that exploits native TLS\ntemplate <>\nclass ets_base: public ets_base {\n using super = ets_base;\n#if _WIN32||_WIN64\n#if __TBB_WIN8UI_SUPPORT\n using tls_key_t = DWORD;\n void create_key() { my_key = FlsAlloc(nullptr); }\n void destroy_key() { FlsFree(my_key); }\n void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); }\n void* get_tls() { return (void *)FlsGetValue(my_key); }\n#else\n using tls_key_t = DWORD;\n void create_key() { my_key = TlsAlloc(); }\n void destroy_key() { TlsFree(my_key); }\n void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); }\n void* get_tls() { return (void *)TlsGetValue(my_key); }\n#endif\n#else\n using tls_key_t = pthread_key_t;\n void create_key() { pthread_key_create(&my_key, nullptr); }\n void destroy_key() { pthread_key_delete(my_key); }\n void set_tls( void * value ) const { pthread_setspecific(my_key, value); }\n void* get_tls() const { return pthread_getspecific(my_key); }\n#endif\n tls_key_t my_key;\n virtual void* create_local() override = 0;\n virtual void* create_array(std::size_t _size) override = 0; // _size in bytes\n virtual void free_array(void* ptr, std::size_t _size) override = 0; // size in bytes\nprotected:\n ets_base() {create_key();}\n ~ets_base() {destroy_key();}\n void* table_lookup( bool& exists ) {\n void* found = get_tls();\n if( found ) {\n exists=true;\n } else {\n found = super::table_lookup(exists);\n set_tls(found);\n }\n return found;\n }\n void table_clear() {\n destroy_key();\n create_key();\n super::table_clear();\n }\n void table_swap( ets_base& other ) {\n using std::swap;\n __TBB_ASSERT(this!=&other, \"Don't swap an instance with itself\");\n swap(my_key, other.my_key);\n super::table_swap(other);\n }\n};\n\n//! Random access iterator for traversing the thread local copies.\ntemplate< typename Container, typename Value >\nclass enumerable_thread_specific_iterator\n{\n //! current position in the concurrent_vector\n\n Container *my_container;\n typename Container::size_type my_index;\n mutable Value *my_value;\n\n template\n friend bool operator==( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j );\n\n template\n friend bool operator<( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j );\n\n template\n friend std::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j );\n\n template\n friend class enumerable_thread_specific_iterator;\n\npublic:\n //! STL support\n using difference_type = std::ptrdiff_t;\n using value_type = Value;\n using pointer = Value*;\n using reference = Value&;\n using iterator_category = std::random_access_iterator_tag;\n\n enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) :\n my_container(&const_cast(container)), my_index(index), my_value(nullptr) {}\n\n //! Default constructor\n enumerable_thread_specific_iterator() : my_container(nullptr), my_index(0), my_value(nullptr) {}\n\n template\n enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) :\n my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {}\n\n enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset ) const {\n return enumerable_thread_specific_iterator(*my_container, my_index + offset);\n }\n\n friend enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset, enumerable_thread_specific_iterator v ) {\n return enumerable_thread_specific_iterator(*v.my_container, v.my_index + offset);\n }\n\n enumerable_thread_specific_iterator &operator+=( std::ptrdiff_t offset ) {\n my_index += offset;\n my_value = nullptr;\n return *this;\n }\n\n enumerable_thread_specific_iterator operator-( std::ptrdiff_t offset ) const {\n return enumerable_thread_specific_iterator( *my_container, my_index-offset );\n }\n\n enumerable_thread_specific_iterator &operator-=( std::ptrdiff_t offset ) {\n my_index -= offset;\n my_value = nullptr;\n return *this;\n }\n\n Value& operator*() const {\n Value* value = my_value;\n if( !value ) {\n value = my_value = (*my_container)[my_index].value();\n }\n __TBB_ASSERT( value==(*my_container)[my_index].value(), \"corrupt cache\" );\n return *value;\n }\n\n Value& operator[]( std::ptrdiff_t k ) const {\n return *(*my_container)[my_index + k].value();\n }\n\n Value* operator->() const {return &operator*();}\n\n enumerable_thread_specific_iterator& operator++() {\n ++my_index;\n my_value = nullptr;\n return *this;\n }\n\n enumerable_thread_specific_iterator& operator--() {\n --my_index;\n my_value = nullptr;\n return *this;\n }\n\n //! Post increment\n enumerable_thread_specific_iterator operator++(int) {\n enumerable_thread_specific_iterator result = *this;\n ++my_index;\n my_value = nullptr;\n return result;\n }\n\n //! Post decrement\n enumerable_thread_specific_iterator operator--(int) {\n enumerable_thread_specific_iterator result = *this;\n --my_index;\n my_value = nullptr;\n return result;\n }\n};\n\ntemplate\nbool operator==( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return i.my_index == j.my_index && i.my_container == j.my_container;\n}\n\ntemplate\nbool operator!=( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return !(i==j);\n}\n\ntemplate\nbool operator<( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return i.my_index\nbool operator>( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return j\nbool operator>=( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return !(i\nbool operator<=( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return !(j\nstd::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i,\n const enumerable_thread_specific_iterator& j ) {\n return i.my_index-j.my_index;\n}\n\ntemplate\nclass segmented_iterator\n{\n template\n friend bool operator==(const segmented_iterator& i, const segmented_iterator& j);\n\n template\n friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j);\n\n template\n friend class segmented_iterator;\n\npublic:\n segmented_iterator() {my_segcont = nullptr;}\n\n segmented_iterator( const SegmentedContainer& _segmented_container ) :\n my_segcont(const_cast(&_segmented_container)),\n outer_iter(my_segcont->end()) { }\n\n ~segmented_iterator() {}\n\n using InnerContainer = typename SegmentedContainer::value_type;\n using inner_iterator = typename InnerContainer::iterator;\n using outer_iterator = typename SegmentedContainer::iterator;\n\n // STL support\n // TODO: inherit all types from segmented container?\n using difference_type = std::ptrdiff_t;\n using value_type = Value;\n using size_type = typename SegmentedContainer::size_type;\n using pointer = Value*;\n using reference = Value&;\n using iterator_category = std::input_iterator_tag;\n\n // Copy Constructor\n template\n segmented_iterator(const segmented_iterator& other) :\n my_segcont(other.my_segcont),\n outer_iter(other.outer_iter),\n // can we assign a default-constructed iterator to inner if we're at the end?\n inner_iter(other.inner_iter)\n {}\n\n // assignment\n template\n segmented_iterator& operator=( const segmented_iterator& other) {\n my_segcont = other.my_segcont;\n outer_iter = other.outer_iter;\n if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter;\n return *this;\n }\n\n // allow assignment of outer iterator to segmented iterator. Once it is\n // assigned, move forward until a non-empty inner container is found or\n // the end of the outer container is reached.\n segmented_iterator& operator=(const outer_iterator& new_outer_iter) {\n __TBB_ASSERT(my_segcont != nullptr, nullptr);\n // check that this iterator points to something inside the segmented container\n for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) {\n if( !outer_iter->empty() ) {\n inner_iter = outer_iter->begin();\n break;\n }\n }\n return *this;\n }\n\n // pre-increment\n segmented_iterator& operator++() {\n advance_me();\n return *this;\n }\n\n // post-increment\n segmented_iterator operator++(int) {\n segmented_iterator tmp = *this;\n operator++();\n return tmp;\n }\n\n bool operator==(const outer_iterator& other_outer) const {\n __TBB_ASSERT(my_segcont != nullptr, nullptr);\n return (outer_iter == other_outer &&\n (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin()));\n }\n\n bool operator!=(const outer_iterator& other_outer) const {\n return !operator==(other_outer);\n\n }\n\n // (i)* RHS\n reference operator*() const {\n __TBB_ASSERT(my_segcont != nullptr, nullptr);\n __TBB_ASSERT(outer_iter != my_segcont->end(), \"Dereferencing a pointer at end of container\");\n __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // should never happen\n return *inner_iter;\n }\n\n // i->\n pointer operator->() const { return &operator*();}\n\nprivate:\n SegmentedContainer* my_segcont;\n outer_iterator outer_iter;\n inner_iterator inner_iter;\n\n void advance_me() {\n __TBB_ASSERT(my_segcont != nullptr, nullptr);\n __TBB_ASSERT(outer_iter != my_segcont->end(), nullptr); // not true if there are no inner containers\n __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // not true if the inner containers are all empty.\n ++inner_iter;\n while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) {\n inner_iter = outer_iter->begin();\n }\n }\n}; // segmented_iterator\n\ntemplate\nbool operator==( const segmented_iterator& i,\n const segmented_iterator& j ) {\n if(i.my_segcont != j.my_segcont) return false;\n if(i.my_segcont == nullptr) return true;\n if(i.outer_iter != j.outer_iter) return false;\n if(i.outer_iter == i.my_segcont->end()) return true;\n return i.inner_iter == j.inner_iter;\n}\n\n// !=\ntemplate\nbool operator!=( const segmented_iterator& i,\n const segmented_iterator& j ) {\n return !(i==j);\n}\n\ntemplate\nstruct construct_by_default: no_assign {\n void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization.\n construct_by_default( int ) {}\n};\n\ntemplate\nstruct construct_by_exemplar: no_assign {\n const T exemplar;\n void construct(void*where) {new(where) T(exemplar);}\n construct_by_exemplar( const T& t ) : exemplar(t) {}\n construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {}\n};\n\ntemplate\nstruct construct_by_finit: no_assign {\n Finit f;\n void construct(void* where) {new(where) T(f());}\n construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {}\n};\n\ntemplate\nstruct construct_by_args: no_assign {\n stored_pack pack;\n void construct(void* where) {\n call( [where](const typename std::decay

::type&... args ){\n new(where) T(args...);\n }, pack );\n }\n construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {}\n};\n\n// storage for initialization function pointer\n// TODO: consider removing the template parameter T here and in callback_leaf\nclass callback_base {\npublic:\n // Clone *this\n virtual callback_base* clone() const = 0;\n // Destruct and free *this\n virtual void destroy() = 0;\n // Need virtual destructor to satisfy GCC compiler warning\n virtual ~callback_base() { }\n // Construct T at where\n virtual void construct(void* where) = 0;\n};\n\ntemplate \nclass callback_leaf: public callback_base, Constructor {\n template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {}\n // TODO: make the construction/destruction consistent (use allocator.construct/destroy)\n using my_allocator_type = typename tbb::tbb_allocator;\n\n callback_base* clone() const override {\n return make(*this);\n }\n\n void destroy() override {\n my_allocator_type alloc;\n tbb::detail::allocator_traits::destroy(alloc, this);\n tbb::detail::allocator_traits::deallocate(alloc, this, 1);\n }\n\n void construct(void* where) override {\n Constructor::construct(where);\n }\n\npublic:\n template\n static callback_base* make( P&& ... params ) {\n void* where = my_allocator_type().allocate(1);\n return new(where) callback_leaf( std::forward

(params)... );\n }\n};\n\n//! Template for recording construction of objects in table\n/** All maintenance of the space will be done explicitly on push_back,\n and all thread local copies must be destroyed before the concurrent\n vector is deleted.\n\n The flag is_built is initialized to false. When the local is\n successfully-constructed, set the flag to true or call value_committed().\n If the constructor throws, the flag will be false.\n*/\ntemplate\nstruct ets_element {\n detail::aligned_space my_space;\n bool is_built;\n ets_element() { is_built = false; } // not currently-built\n U* value() { return my_space.begin(); }\n U* value_committed() { is_built = true; return my_space.begin(); }\n ~ets_element() {\n if(is_built) {\n my_space.begin()->~U();\n is_built = false;\n }\n }\n};\n\n// A predicate that can be used for a compile-time compatibility check of ETS instances\n// Ideally, it should have been declared inside the ETS class, but unfortunately\n// in that case VS2013 does not enable the variadic constructor.\ntemplate struct is_compatible_ets : std::false_type {};\ntemplate\nstruct is_compatible_ets< T, enumerable_thread_specific > : std::is_same {};\n\n// A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression\ntemplate using has_empty_braces_operator = decltype(std::declval()());\ntemplate using is_callable_no_args = supports;\n\n//! The enumerable_thread_specific container\n/** enumerable_thread_specific has the following properties:\n - thread-local copies are lazily created, with default, exemplar or function initialization.\n - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant.\n - the contained objects need not have operator=() defined if combine is not used.\n - enumerable_thread_specific containers may be copy-constructed or assigned.\n - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed.\n - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods\n\n@par Segmented iterator\n When the thread-local objects are containers with input_iterators defined, a segmented iterator may\n be used to iterate over all the elements of all thread-local copies.\n\n@par combine and combine_each\n - Both methods are defined for enumerable_thread_specific.\n - combine() requires the type T have operator=() defined.\n - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.)\n - Both are evaluated in serial context (the methods are assumed to be non-benign.)\n\n@ingroup containers */\ntemplate ,\n ets_key_usage_type ETS_key_type=ets_no_key >\nclass enumerable_thread_specific: ets_base {\n\n template friend class enumerable_thread_specific;\n\n using padded_element = padded>;\n\n //! A generic range, used to create range objects from the iterators\n template\n class generic_range_type: public blocked_range {\n public:\n using value_type = T;\n using reference = T&;\n using const_reference = const T&;\n using iterator = I;\n using difference_type = std::ptrdiff_t;\n\n generic_range_type( I begin_, I end_, std::size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {}\n template\n generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {}\n generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {}\n };\n\n using allocator_traits_type = tbb::detail::allocator_traits;\n\n using padded_allocator_type = typename allocator_traits_type::template rebind_alloc;\n using internal_collection_type = tbb::concurrent_vector< padded_element, padded_allocator_type >;\n\n callback_base *my_construct_callback;\n\n internal_collection_type my_locals;\n\n // TODO: consider unifying the callback mechanism for all create_local* methods below\n // (likely non-compatible and requires interface version increase)\n void* create_local() override {\n padded_element& lref = *my_locals.grow_by(1);\n my_construct_callback->construct(lref.value());\n return lref.value_committed();\n }\n\n static void* create_local_by_copy( ets_base& base, void* p ) {\n enumerable_thread_specific& ets = static_cast(base);\n padded_element& lref = *ets.my_locals.grow_by(1);\n new(lref.value()) T(*static_cast(p));\n return lref.value_committed();\n }\n\n static void* create_local_by_move( ets_base& base, void* p ) {\n enumerable_thread_specific& ets = static_cast(base);\n padded_element& lref = *ets.my_locals.grow_by(1);\n new(lref.value()) T(std::move(*static_cast(p)));\n return lref.value_committed();\n }\n\n using array_allocator_type = typename allocator_traits_type::template rebind_alloc;\n\n // _size is in bytes\n void* create_array(std::size_t _size) override {\n std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t);\n return array_allocator_type().allocate(nelements);\n }\n\n void free_array( void* _ptr, std::size_t _size) override {\n std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t);\n array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements);\n }\n\npublic:\n\n //! Basic types\n using value_type = T;\n using allocator_type = Allocator;\n using size_type = typename internal_collection_type::size_type;\n using difference_type = typename internal_collection_type::difference_type;\n using reference = value_type&;\n using const_reference = const value_type&;\n\n using pointer = typename allocator_traits_type::pointer;\n using const_pointer = typename allocator_traits_type::const_pointer;\n\n // Iterator types\n using iterator = enumerable_thread_specific_iterator;\n using const_iterator = enumerable_thread_specific_iterator;\n\n // Parallel range types\n using range_type = generic_range_type;\n using const_range_type = generic_range_type;\n\n //! Default constructor. Each local instance of T is default constructed.\n enumerable_thread_specific() : my_construct_callback(\n callback_leaf >::make(/*dummy argument*/0)\n ){}\n\n //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()).\n template ::type>::value>::type>\n explicit enumerable_thread_specific( Finit finit ) : my_construct_callback(\n callback_leaf >::make( std::move(finit) )\n ){}\n\n //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar.\n explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback(\n callback_leaf >::make( exemplar )\n ){}\n\n explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback(\n callback_leaf >::make( std::move(exemplar) )\n ){}\n\n //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...)\n template ::type>::value\n && !is_compatible_ets::type>::value\n && !std::is_same::type>::value\n >::type>\n enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback(\n callback_leaf >::make( std::forward(arg1), std::forward

(args)... )\n ){}\n\n //! Destructor\n ~enumerable_thread_specific() {\n if(my_construct_callback) my_construct_callback->destroy();\n // Deallocate the hash table before overridden free_array() becomes inaccessible\n this->ets_base::table_clear();\n }\n\n //! returns reference to local, discarding exists\n reference local() {\n bool exists;\n return local(exists);\n }\n\n //! Returns reference to calling thread's local copy, creating one if necessary\n reference local(bool& exists) {\n void* ptr = this->table_lookup(exists);\n return *(T*)ptr;\n }\n\n //! Get the number of local copies\n size_type size() const { return my_locals.size(); }\n\n //! true if there have been no local copies created\n bool empty() const { return my_locals.empty(); }\n\n //! begin iterator\n iterator begin() { return iterator( my_locals, 0 ); }\n //! end iterator\n iterator end() { return iterator(my_locals, my_locals.size() ); }\n\n //! begin const iterator\n const_iterator begin() const { return const_iterator(my_locals, 0); }\n\n //! end const iterator\n const_iterator end() const { return const_iterator(my_locals, my_locals.size()); }\n\n //! Get range for parallel algorithms\n range_type range( std::size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); }\n\n //! Get const range for parallel algorithms\n const_range_type range( std::size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); }\n\n //! Destroys local copies\n void clear() {\n my_locals.clear();\n this->table_clear();\n // callback is not destroyed\n }\n\nprivate:\n template\n void internal_copy(const enumerable_thread_specific& other) {\n // this tests is_compatible_ets\n static_assert( (is_compatible_ets::type>::value), \"is_compatible_ets fails\" );\n // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception.\n my_construct_callback = other.my_construct_callback->clone();\n __TBB_ASSERT(my_locals.size()==0, nullptr);\n my_locals.reserve(other.size());\n this->table_elementwise_copy( other, create_local_by_copy );\n }\n\n void internal_swap(enumerable_thread_specific& other) {\n using std::swap;\n __TBB_ASSERT( this!=&other, nullptr);\n swap(my_construct_callback, other.my_construct_callback);\n // concurrent_vector::swap() preserves storage space,\n // so addresses to the vector kept in ETS hash table remain valid.\n swap(my_locals, other.my_locals);\n this->ets_base::table_swap(other);\n }\n\n template\n void internal_move(enumerable_thread_specific&& other) {\n static_assert( (is_compatible_ets::type>::value), \"is_compatible_ets fails\" );\n my_construct_callback = other.my_construct_callback;\n other.my_construct_callback = nullptr;\n __TBB_ASSERT(my_locals.size()==0, nullptr);\n my_locals.reserve(other.size());\n this->table_elementwise_copy( other, create_local_by_move );\n }\n\npublic:\n enumerable_thread_specific( const enumerable_thread_specific& other )\n : ets_base() /* prevents GCC warnings with -Wextra */\n {\n internal_copy(other);\n }\n\n template\n enumerable_thread_specific( const enumerable_thread_specific& other )\n {\n internal_copy(other);\n }\n\n enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback()\n {\n // TODO: use internal_move correctly here\n internal_swap(other);\n }\n\n template\n enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback()\n {\n internal_move(std::move(other));\n }\n\n enumerable_thread_specific& operator=( const enumerable_thread_specific& other )\n {\n if( this != &other ) {\n this->clear();\n my_construct_callback->destroy();\n internal_copy( other );\n }\n return *this;\n }\n\n template\n enumerable_thread_specific& operator=( const enumerable_thread_specific& other )\n {\n __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types\n this->clear();\n my_construct_callback->destroy();\n internal_copy(other);\n return *this;\n }\n\n enumerable_thread_specific& operator=( enumerable_thread_specific&& other )\n {\n if( this != &other ) {\n // TODO: use internal_move correctly here\n internal_swap(other);\n }\n return *this;\n }\n\n template\n enumerable_thread_specific& operator=( enumerable_thread_specific&& other )\n {\n __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types\n this->clear();\n my_construct_callback->destroy();\n internal_move(std::move(other));\n return *this;\n }\n\n // CombineFunc has signature T(T,T) or T(const T&, const T&)\n template \n T combine(CombineFunc f_combine) {\n if(begin() == end()) {\n ets_element location;\n my_construct_callback->construct(location.value());\n return *location.value_committed();\n }\n const_iterator ci = begin();\n T my_result = *ci;\n while(++ci != end())\n my_result = f_combine( my_result, *ci );\n return my_result;\n }\n\n // combine_func_t takes T by value or by [const] reference, and returns nothing\n template \n void combine_each(CombineFunc f_combine) {\n for(iterator ci = begin(); ci != end(); ++ci) {\n f_combine( *ci );\n }\n }\n\n}; // enumerable_thread_specific\n\ntemplate< typename Container >\nclass flattened2d {\n // This intermediate typedef is to address issues with VC7.1 compilers\n using conval_type = typename Container::value_type;\n\npublic:\n //! Basic types\n using size_type = typename conval_type::size_type;\n using difference_type = typename conval_type::difference_type;\n using allocator_type = typename conval_type::allocator_type;\n using value_type = typename conval_type::value_type;\n using reference = typename conval_type::reference;\n using const_reference = typename conval_type::const_reference;\n using pointer = typename conval_type::pointer;\n using const_pointer = typename conval_type::const_pointer;\n\n using iterator = segmented_iterator;\n using const_iterator = segmented_iterator;\n\n flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) :\n my_container(const_cast(&c)), my_begin(b), my_end(e) { }\n\n explicit flattened2d( const Container &c ) :\n my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { }\n\n iterator begin() { return iterator(*my_container) = my_begin; }\n iterator end() { return iterator(*my_container) = my_end; }\n const_iterator begin() const { return const_iterator(*my_container) = my_begin; }\n const_iterator end() const { return const_iterator(*my_container) = my_end; }\n\n size_type size() const {\n size_type tot_size = 0;\n for(typename Container::const_iterator i = my_begin; i != my_end; ++i) {\n tot_size += i->size();\n }\n return tot_size;\n }\n\nprivate:\n Container *my_container;\n typename Container::const_iterator my_begin;\n typename Container::const_iterator my_end;\n};\n\ntemplate \nflattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) {\n return flattened2d(c, b, e);\n}\n\ntemplate \nflattened2d flatten2d(const Container &c) {\n return flattened2d(c);\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::enumerable_thread_specific;\nusing detail::d1::flattened2d;\nusing detail::d1::flatten2d;\n// ets enum keys\nusing detail::d1::ets_key_usage_type;\nusing detail::d1::ets_key_per_instance;\nusing detail::d1::ets_no_key;\n#if __TBB_RESUMABLE_TASKS\nusing detail::d1::ets_suspend_aware;\n#endif\n} // inline namespace v1\n\n} // namespace tbb\n\n#endif // __TBB_enumerable_thread_specific_H\n\n" }, { "path": "src/tbb/include/oneapi/tbb/flow_graph.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_flow_graph_H\n#define __TBB_flow_graph_H\n\n#include \n#include \n#include \n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"spin_mutex.h\"\n#include \"null_mutex.h\"\n#include \"spin_rw_mutex.h\"\n#include \"null_rw_mutex.h\"\n#include \"detail/_pipeline_filters.h\"\n#include \"detail/_task.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"cache_aligned_allocator.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_aggregator.h\"\n#include \"detail/_allocator_traits.h\"\n#include \"detail/_utils.h\"\n#include \"profiling.h\"\n#include \"task_arena.h\"\n\n#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ )\n #if __INTEL_COMPILER\n // Disabled warning \"routine is both inline and noinline\"\n // #pragma warning (push)\n // #pragma warning( disable: 2196 )\n #endif\n #define __TBB_NOINLINE_SYM __attribute__((noinline))\n#else\n #define __TBB_NOINLINE_SYM\n#endif\n\n#include \n#include \n#include \n#include \n#if __TBB_CPP20_CONCEPTS_PRESENT\n#include \n#endif\n\n/** @file\n \\brief The graph related classes and functions\n\n There are some applications that best express dependencies as messages\n passed between nodes in a graph. These messages may contain data or\n simply act as signals that a predecessors has completed. The graph\n class and its associated node classes can be used to express such\n applications.\n*/\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d2 {\n\n//! An enumeration the provides the two most common concurrency levels: unlimited and serial\nenum concurrency { unlimited = 0, serial = 1 };\n\n//! A generic null type\nstruct null_type {};\n\n//! An empty class used for messages that mean \"I'm done\"\nclass continue_msg {};\n\n} // namespace d2\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d0 {\n\ntemplate \nconcept node_body_return_type = std::same_as ||\n std::convertible_to;\n\n// TODO: consider using std::invocable here\ntemplate \nconcept continue_node_body = std::copy_constructible &&\n requires( Body& body, const tbb::detail::d2::continue_msg& v ) {\n { body(v) } -> node_body_return_type;\n };\n\ntemplate \nconcept function_node_body = std::copy_constructible &&\n std::invocable &&\n node_body_return_type, Output>;\n\ntemplate \nconcept join_node_function_object = std::copy_constructible &&\n std::invocable &&\n std::convertible_to, Key>;\n\ntemplate \nconcept input_node_body = std::copy_constructible &&\n requires( Body& body, tbb::detail::d1::flow_control& fc ) {\n { body(fc) } -> adaptive_same_as;\n };\n\ntemplate \nconcept multifunction_node_body = std::copy_constructible &&\n std::invocable;\n\ntemplate \nconcept sequencer = std::copy_constructible &&\n std::invocable &&\n std::convertible_to, std::size_t>;\n\ntemplate \nconcept async_node_body = std::copy_constructible &&\n std::invocable;\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n\nnamespace d2 {\n\n//! Forward declaration section\ntemplate< typename T > class sender;\ntemplate< typename T > class receiver;\nclass continue_receiver;\n\ntemplate< typename T, typename U > class limiter_node; // needed for resetting decrementer\n\ntemplate class successor_cache;\ntemplate class broadcast_cache;\ntemplate class round_robin_cache;\ntemplate class predecessor_cache;\ntemplate class reservable_predecessor_cache;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\nnamespace order {\nstruct following;\nstruct preceding;\n}\ntemplate struct node_set;\n#endif\n\n\n} // namespace d2\n} // namespace detail\n} // namespace tbb\n\n//! The graph class\n#include \"detail/_flow_graph_impl.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\nstatic inline std::pair order_tasks(graph_task* first, graph_task* second) {\n if (second->priority > first->priority)\n return std::make_pair(second, first);\n return std::make_pair(first, second);\n}\n\n// submit task if necessary. Returns the non-enqueued task if there is one.\nstatic inline graph_task* combine_tasks(graph& g, graph_task* left, graph_task* right) {\n // if no RHS task, don't change left.\n if (right == nullptr) return left;\n // right != nullptr\n if (left == nullptr) return right;\n if (left == SUCCESSFULLY_ENQUEUED) return right;\n // left contains a task\n if (right != SUCCESSFULLY_ENQUEUED) {\n // both are valid tasks\n auto tasks_pair = order_tasks(left, right);\n spawn_in_graph_arena(g, *tasks_pair.first);\n return tasks_pair.second;\n }\n return left;\n}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\nclass message_metainfo {\npublic:\n using waiters_type = std::forward_list;\n\n message_metainfo() = default;\n\n message_metainfo(const waiters_type& waiters) : my_waiters(waiters) {}\n message_metainfo(waiters_type&& waiters) : my_waiters(std::move(waiters)) {}\n\n const waiters_type& waiters() const & { return my_waiters; }\n waiters_type&& waiters() && { return std::move(my_waiters); }\n\n bool empty() const { return my_waiters.empty(); }\n\n void merge(const message_metainfo& other) {\n // TODO: should we avoid duplications on merging\n my_waiters.insert_after(my_waiters.before_begin(),\n other.waiters().begin(),\n other.waiters().end());\n }\nprivate:\n waiters_type my_waiters;\n}; // class message_metainfo\n\n#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) , metainfo\n\n#else\n#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)\n#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n\n//! Pure virtual template class that defines a sender of messages of type T\ntemplate< typename T >\nclass sender {\npublic:\n virtual ~sender() {}\n\n //! Request an item from the sender\n virtual bool try_get( T & ) { return false; }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual bool try_get( T &, message_metainfo& ) { return false; }\n#endif\n\n //! Reserves an item in the sender\n virtual bool try_reserve( T & ) { return false; }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual bool try_reserve( T &, message_metainfo& ) { return false; }\n#endif\n\n //! Releases the reserved item\n virtual bool try_release( ) { return false; }\n\n //! Consumes the reserved item\n virtual bool try_consume( ) { return false; }\n\nprotected:\n //! The output type of this sender\n typedef T output_type;\n\n //! The successor type for this node\n typedef receiver successor_type;\n\n //! Add a new successor to this node\n virtual bool register_successor( successor_type &r ) = 0;\n\n //! Removes a successor from this node\n virtual bool remove_successor( successor_type &r ) = 0;\n\n template\n friend bool register_successor(sender& s, receiver& r);\n\n template\n friend bool remove_successor (sender& s, receiver& r);\n}; // class sender\n\ntemplate\nbool register_successor(sender& s, receiver& r) {\n return s.register_successor(r);\n}\n\ntemplate\nbool remove_successor(sender& s, receiver& r) {\n return s.remove_successor(r);\n}\n\n//! Pure virtual template class that defines a receiver of messages of type T\ntemplate< typename T >\nclass receiver {\nprivate:\n template \n bool internal_try_put(const T& t, TryPutTaskArgs&&... args) {\n graph_task* res = try_put_task(t, std::forward(args)...);\n if (!res) return false;\n if (res != SUCCESSFULLY_ENQUEUED) spawn_in_graph_arena(graph_reference(), *res);\n return true;\n }\n\npublic:\n //! Destructor\n virtual ~receiver() {}\n\n //! Put an item to the receiver\n bool try_put( const T& t ) {\n return internal_try_put(t);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n //! Put an item to the receiver and wait for completion\n bool try_put_and_wait( const T& t ) {\n // Since try_put_and_wait is a blocking call, it is safe to create wait_context on stack\n d1::wait_context_vertex msg_wait_vertex{};\n\n bool res = internal_try_put(t, message_metainfo{message_metainfo::waiters_type{&msg_wait_vertex}});\n if (res) {\n __TBB_ASSERT(graph_reference().my_context != nullptr, \"No wait_context associated with the Flow Graph\");\n wait(msg_wait_vertex.get_context(), *graph_reference().my_context);\n }\n return res;\n }\n#endif\n\n //! put item to successor; return task to run the successor if possible.\nprotected:\n //! The input type of this receiver\n typedef T input_type;\n\n //! The predecessor type for this node\n typedef sender predecessor_type;\n\n template< typename R, typename B > friend class run_and_put_task;\n template< typename X, typename Y > friend class broadcast_cache;\n template< typename X, typename Y > friend class round_robin_cache;\n virtual graph_task *try_put_task(const T& t) = 0;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual graph_task *try_put_task(const T& t, const message_metainfo&) = 0;\n#endif\n virtual graph& graph_reference() const = 0;\n\n template friend class successor_cache;\n virtual bool is_continue_receiver() { return false; }\n\n // TODO revamp: reconsider the inheritance and move node priority out of receiver\n virtual node_priority_t priority() const { return no_priority; }\n\n //! Add a predecessor to the node\n virtual bool register_predecessor( predecessor_type & ) { return false; }\n\n //! Remove a predecessor from the node\n virtual bool remove_predecessor( predecessor_type & ) { return false; }\n\n template \n friend bool register_predecessor(receiver& r, sender& s);\n template \n friend bool remove_predecessor (receiver& r, sender& s);\n}; // class receiver\n\ntemplate \nbool register_predecessor(receiver& r, sender& s) {\n return r.register_predecessor(s);\n}\n\ntemplate \nbool remove_predecessor(receiver& r, sender& s) {\n return r.remove_predecessor(s);\n}\n\n//! Base class for receivers of completion messages\n/** These receivers automatically reset, but cannot be explicitly waited on */\nclass continue_receiver : public receiver< continue_msg > {\nprotected:\n\n //! Constructor\n explicit continue_receiver( int number_of_predecessors, node_priority_t a_priority ) {\n my_predecessor_count = my_initial_predecessor_count = number_of_predecessors;\n my_current_count = 0;\n my_priority = a_priority;\n }\n\n //! Copy constructor\n continue_receiver( const continue_receiver& src ) : receiver() {\n my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count;\n my_current_count = 0;\n my_priority = src.my_priority;\n }\n\n //! Increments the trigger threshold\n bool register_predecessor( predecessor_type & ) override {\n spin_mutex::scoped_lock l(my_mutex);\n ++my_predecessor_count;\n return true;\n }\n\n //! Decrements the trigger threshold\n /** Does not check to see if the removal of the predecessor now makes the current count\n exceed the new threshold. So removing a predecessor while the graph is active can cause\n unexpected results. */\n bool remove_predecessor( predecessor_type & ) override {\n spin_mutex::scoped_lock l(my_mutex);\n --my_predecessor_count;\n return true;\n }\n\n //! The input type\n typedef continue_msg input_type;\n\n //! The predecessor type for this node\n typedef receiver::predecessor_type predecessor_type;\n\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n\nprivate:\n // execute body is supposed to be too small to create a task for.\n graph_task* try_put_task_impl( const input_type& __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo predecessor_metainfo;\n#endif\n {\n spin_mutex::scoped_lock l(my_mutex);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n // Prolong the wait and store the metainfo until receiving signals from all the predecessors\n for (auto waiter : metainfo.waiters()) {\n waiter->reserve(1);\n }\n my_current_metainfo.merge(metainfo);\n#endif\n if ( ++my_current_count < my_predecessor_count )\n return SUCCESSFULLY_ENQUEUED;\n else {\n my_current_count = 0;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n predecessor_metainfo = my_current_metainfo;\n my_current_metainfo = message_metainfo{};\n#endif\n }\n }\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* res = execute(predecessor_metainfo);\n for (auto waiter : predecessor_metainfo.waiters()) {\n waiter->release(1);\n }\n#else\n graph_task* res = execute();\n#endif\n return res? res : SUCCESSFULLY_ENQUEUED;\n }\n\nprotected:\n graph_task* try_put_task( const input_type& input ) override {\n return try_put_task_impl(input __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task( const input_type& input, const message_metainfo& metainfo ) override {\n return try_put_task_impl(input, metainfo);\n }\n#endif\n\n spin_mutex my_mutex;\n int my_predecessor_count;\n int my_current_count;\n int my_initial_predecessor_count;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo my_current_metainfo;\n#endif\n node_priority_t my_priority;\n // the friend declaration in the base class did not eliminate the \"protected class\"\n // error in gcc 4.1.2\n template friend class limiter_node;\n\n virtual void reset_receiver( reset_flags f ) {\n my_current_count = 0;\n if (f & rf_clear_edges) {\n my_predecessor_count = my_initial_predecessor_count;\n }\n }\n\n //! Does whatever should happen when the threshold is reached\n /** This should be very fast or else spawn a task. This is\n called while the sender is blocked in the try_put(). */\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n virtual graph_task* execute(const message_metainfo& metainfo) = 0;\n#else\n virtual graph_task* execute() = 0;\n#endif\n template friend class successor_cache;\n bool is_continue_receiver() override { return true; }\n\n node_priority_t priority() const override { return my_priority; }\n}; // class continue_receiver\n\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n template \n K key_from_message( const T &t ) {\n return t.key();\n }\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n\n} // d1\n} // detail\n} // tbb\n\n#include \"detail/_flow_graph_trace_impl.h\"\n#include \"detail/_hash_compare.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\n#include \"detail/_flow_graph_body_impl.h\"\n#include \"detail/_flow_graph_cache_impl.h\"\n#include \"detail/_flow_graph_types_impl.h\"\n\nusing namespace graph_policy_namespace;\n\ntemplate \ngraph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(nullptr)\n{\n if (begin) current_node = my_graph->my_nodes;\n //else it is an end iterator by default\n}\n\ntemplate \ntypename graph_iterator::reference graph_iterator::operator*() const {\n __TBB_ASSERT(current_node, \"graph_iterator at end\");\n return *operator->();\n}\n\ntemplate \ntypename graph_iterator::pointer graph_iterator::operator->() const {\n return current_node;\n}\n\ntemplate \nvoid graph_iterator::internal_forward() {\n if (current_node) current_node = current_node->next;\n}\n\n//! Constructs a graph with isolated task_group_context\ninline graph::graph() : my_wait_context_vertex(0), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) {\n prepare_task_arena();\n own_context = true;\n cancelled = false;\n caught_exception = false;\n my_context = new (r1::cache_aligned_allocate(sizeof(task_group_context))) task_group_context(FLOW_TASKS);\n fgt_graph(this);\n my_is_active = true;\n}\n\ninline graph::graph(task_group_context& use_this_context) :\n my_wait_context_vertex(0), my_context(&use_this_context), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) {\n prepare_task_arena();\n own_context = false;\n cancelled = false;\n caught_exception = false;\n fgt_graph(this);\n my_is_active = true;\n}\n\ninline graph::~graph() {\n wait_for_all();\n if (own_context) {\n my_context->~task_group_context();\n r1::cache_aligned_deallocate(my_context);\n }\n delete my_task_arena;\n}\n\ninline void graph::reserve_wait() {\n my_wait_context_vertex.reserve();\n fgt_reserve_wait(this);\n}\n\ninline void graph::release_wait() {\n fgt_release_wait(this);\n my_wait_context_vertex.release();\n}\n\ninline void graph::register_node(graph_node *n) {\n n->next = nullptr;\n {\n spin_mutex::scoped_lock lock(nodelist_mutex);\n n->prev = my_nodes_last;\n if (my_nodes_last) my_nodes_last->next = n;\n my_nodes_last = n;\n if (!my_nodes) my_nodes = n;\n }\n}\n\ninline void graph::remove_node(graph_node *n) {\n {\n spin_mutex::scoped_lock lock(nodelist_mutex);\n __TBB_ASSERT(my_nodes && my_nodes_last, \"graph::remove_node: Error: no registered nodes\");\n if (n->prev) n->prev->next = n->next;\n if (n->next) n->next->prev = n->prev;\n if (my_nodes_last == n) my_nodes_last = n->prev;\n if (my_nodes == n) my_nodes = n->next;\n }\n n->prev = n->next = nullptr;\n}\n\ninline void graph::reset( reset_flags f ) {\n // reset context\n deactivate_graph(*this);\n\n my_context->reset();\n cancelled = false;\n caught_exception = false;\n // reset all the nodes comprising the graph\n for(iterator ii = begin(); ii != end(); ++ii) {\n graph_node *my_p = &(*ii);\n my_p->reset_node(f);\n }\n // Reattach the arena. Might be useful to run the graph in a particular task_arena\n // while not limiting graph lifetime to a single task_arena::execute() call.\n prepare_task_arena( /*reinit=*/true );\n activate_graph(*this);\n}\n\ninline void graph::cancel() {\n my_context->cancel_group_execution();\n}\n\ninline graph::iterator graph::begin() { return iterator(this, true); }\n\ninline graph::iterator graph::end() { return iterator(this, false); }\n\ninline graph::const_iterator graph::begin() const { return const_iterator(this, true); }\n\ninline graph::const_iterator graph::end() const { return const_iterator(this, false); }\n\ninline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); }\n\ninline graph::const_iterator graph::cend() const { return const_iterator(this, false); }\n\ninline graph_node::graph_node(graph& g) : my_graph(g) {\n my_graph.register_node(this);\n}\n\ninline graph_node::~graph_node() {\n my_graph.remove_node(this);\n}\n\n#include \"detail/_flow_graph_node_impl.h\"\n\n\n//! An executable node that acts as a source, i.e. it has no predecessors\n\ntemplate < typename Output >\n __TBB_requires(std::copyable)\nclass input_node : public graph_node, public sender< Output > {\npublic:\n //! The type of the output message, which is complete\n typedef Output output_type;\n\n //! The type of successors of this node\n typedef typename sender::successor_type successor_type;\n\n // Input node has no input type\n typedef null_type input_type;\n\n //! Constructor for a node with a successor\n template< typename Body >\n __TBB_requires(input_node_body)\n __TBB_NOINLINE_SYM input_node( graph &g, Body body )\n : graph_node(g), my_active(false)\n , my_body( new input_body_leaf< output_type, Body>(body) )\n , my_init_body( new input_body_leaf< output_type, Body>(body) )\n , my_successors(this), my_reserved(false), my_has_cached_item(false)\n {\n fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph,\n static_cast *>(this), this->my_body);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(input_node_body)\n input_node( const node_set& successors, Body body )\n : input_node(successors.graph_reference(), body)\n {\n make_edges(*this, successors);\n }\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM input_node( const input_node& src )\n : graph_node(src.my_graph), sender()\n , my_active(false)\n , my_body(src.my_init_body->clone()), my_init_body(src.my_init_body->clone())\n , my_successors(this), my_reserved(false), my_has_cached_item(false)\n {\n fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph,\n static_cast *>(this), this->my_body);\n }\n\n //! The destructor\n ~input_node() { delete my_body; delete my_init_body; }\n\n //! Add a new successor to this node\n bool register_successor( successor_type &r ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n my_successors.register_successor(r);\n if ( my_active )\n spawn_put();\n return true;\n }\n\n //! Removes a successor from this node\n bool remove_successor( successor_type &r ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n my_successors.remove_successor(r);\n return true;\n }\n\n //! Request an item from the node\n bool try_get( output_type &v ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( my_reserved )\n return false;\n\n if ( my_has_cached_item ) {\n v = my_cached_item;\n my_has_cached_item = false;\n return true;\n }\n // we've been asked to provide an item, but we have none. enqueue a task to\n // provide one.\n if ( my_active )\n spawn_put();\n return false;\n }\n\n //! Reserves an item.\n bool try_reserve( output_type &v ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( my_reserved ) {\n return false;\n }\n\n if ( my_has_cached_item ) {\n v = my_cached_item;\n my_reserved = true;\n return true;\n } else {\n return false;\n }\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\nprivate:\n bool try_reserve( output_type& v, message_metainfo& ) override {\n return try_reserve(v);\n }\n\n bool try_get( output_type& v, message_metainfo& ) override {\n return try_get(v);\n }\npublic:\n#endif\n\n //! Release a reserved item.\n /** true = item has been released and so remains in sender, dest must request or reserve future items */\n bool try_release( ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n __TBB_ASSERT( my_reserved && my_has_cached_item, \"releasing non-existent reservation\" );\n my_reserved = false;\n if(!my_successors.empty())\n spawn_put();\n return true;\n }\n\n //! Consumes a reserved item\n bool try_consume( ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n __TBB_ASSERT( my_reserved && my_has_cached_item, \"consuming non-existent reservation\" );\n my_reserved = false;\n my_has_cached_item = false;\n if ( !my_successors.empty() ) {\n spawn_put();\n }\n return true;\n }\n\n //! Activates a node that was created in the inactive state\n void activate() {\n spin_mutex::scoped_lock lock(my_mutex);\n my_active = true;\n if (!my_successors.empty())\n spawn_put();\n }\n\n template\n Body copy_function_object() {\n input_body &body_ref = *this->my_body;\n return dynamic_cast< input_body_leaf & >(body_ref).get_body();\n }\n\nprotected:\n\n //! resets the input_node to its initial state\n void reset_node( reset_flags f) override {\n my_active = false;\n my_reserved = false;\n my_has_cached_item = false;\n\n if(f & rf_clear_edges) my_successors.clear();\n if(f & rf_reset_bodies) {\n input_body *tmp = my_init_body->clone();\n delete my_body;\n my_body = tmp;\n }\n }\n\nprivate:\n spin_mutex my_mutex;\n bool my_active;\n input_body *my_body;\n input_body *my_init_body;\n broadcast_cache< output_type > my_successors;\n bool my_reserved;\n bool my_has_cached_item;\n output_type my_cached_item;\n\n // used by apply_body_bypass, can invoke body of node.\n bool try_reserve_apply_body(output_type &v) {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( my_reserved ) {\n return false;\n }\n if ( !my_has_cached_item ) {\n d1::flow_control control;\n\n fgt_begin_body( my_body );\n\n my_cached_item = (*my_body)(control);\n my_has_cached_item = !control.is_pipeline_stopped;\n\n fgt_end_body( my_body );\n }\n if ( my_has_cached_item ) {\n v = my_cached_item;\n my_reserved = true;\n return true;\n } else {\n return false;\n }\n }\n\n graph_task* create_put_task() {\n d1::small_object_allocator allocator{};\n typedef input_node_task_bypass< input_node > task_type;\n graph_task* t = allocator.new_object(my_graph, allocator, *this);\n return t;\n }\n\n //! Spawns a task that applies the body\n void spawn_put( ) {\n if(is_graph_active(this->my_graph)) {\n spawn_in_graph_arena(this->my_graph, *create_put_task());\n }\n }\n\n friend class input_node_task_bypass< input_node >;\n //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it.\n graph_task* apply_body_bypass( ) {\n output_type v;\n if ( !try_reserve_apply_body(v) )\n return nullptr;\n\n graph_task *last_task = my_successors.try_put_task(v);\n if ( last_task )\n try_consume();\n else\n try_release();\n return last_task;\n }\n}; // class input_node\n\n//! Implements a function node that supports Input -> Output\ntemplate\n __TBB_requires(std::default_initializable &&\n std::copy_constructible &&\n std::copy_constructible)\nclass function_node\n : public graph_node\n , public function_input< Input, Output, Policy, cache_aligned_allocator >\n , public function_output\n{\n typedef cache_aligned_allocator internals_allocator;\n\npublic:\n typedef Input input_type;\n typedef Output output_type;\n typedef function_input input_impl_type;\n typedef function_input_queue input_queue_type;\n typedef function_output fOutput_type;\n typedef typename input_impl_type::predecessor_type predecessor_type;\n typedef typename fOutput_type::successor_type successor_type;\n\n using input_impl_type::my_predecessors;\n\n //! Constructor\n // input_queue_type is allocated here, but destroyed in the function_input_base.\n // TODO: pass the graph_buffer_policy to the function_input_base so it can all\n // be done in one place. This would be an interface-breaking change.\n template< typename Body >\n __TBB_requires(function_node_body)\n __TBB_NOINLINE_SYM function_node( graph &g, size_t concurrency,\n Body body, Policy = Policy(), node_priority_t a_priority = no_priority )\n : graph_node(g), input_impl_type(g, concurrency, body, a_priority),\n fOutput_type(g) {\n fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(this), this->my_body );\n }\n\n template \n __TBB_requires(function_node_body)\n function_node( graph& g, size_t concurrency, Body body, node_priority_t a_priority )\n : function_node(g, concurrency, body, Policy(), a_priority) {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(function_node_body)\n function_node( const node_set& nodes, size_t concurrency, Body body,\n Policy p = Policy(), node_priority_t a_priority = no_priority )\n : function_node(nodes.graph_reference(), concurrency, body, p, a_priority) {\n make_edges_in_order(nodes, *this);\n }\n\n template \n __TBB_requires(function_node_body)\n function_node( const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority )\n : function_node(nodes, concurrency, body, Policy(), a_priority) {}\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n //! Copy constructor\n __TBB_NOINLINE_SYM function_node( const function_node& src ) :\n graph_node(src.my_graph),\n input_impl_type(src),\n fOutput_type(src.my_graph) {\n fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(this), this->my_body );\n }\n\nprotected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n using input_impl_type::try_put_task;\n\n broadcast_cache &successors () override { return fOutput_type::my_successors; }\n\n void reset_node(reset_flags f) override {\n input_impl_type::reset_function_input(f);\n // TODO: use clear() instead.\n if(f & rf_clear_edges) {\n successors().clear();\n my_predecessors.clear();\n }\n __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), \"function_node successors not empty\");\n __TBB_ASSERT(this->my_predecessors.empty(), \"function_node predecessors not empty\");\n }\n\n}; // class function_node\n\n//! implements a function node that supports Input -> (set of outputs)\n// Output is a tuple of output types.\ntemplate\n __TBB_requires(std::default_initializable &&\n std::copy_constructible)\nclass multifunction_node :\n public graph_node,\n public multifunction_input\n <\n Input,\n typename wrap_tuple_elements<\n std::tuple_size::value, // #elements in tuple\n multifunction_output, // wrap this around each element\n Output // the tuple providing the types\n >::type,\n Policy,\n cache_aligned_allocator\n >\n{\n typedef cache_aligned_allocator internals_allocator;\n\nprotected:\n static const int N = std::tuple_size::value;\npublic:\n typedef Input input_type;\n typedef null_type output_type;\n typedef typename wrap_tuple_elements::type output_ports_type;\n typedef multifunction_input<\n input_type, output_ports_type, Policy, internals_allocator> input_impl_type;\n typedef function_input_queue input_queue_type;\nprivate:\n using input_impl_type::my_predecessors;\npublic:\n template\n __TBB_requires(multifunction_node_body)\n __TBB_NOINLINE_SYM multifunction_node(\n graph &g, size_t concurrency,\n Body body, Policy = Policy(), node_priority_t a_priority = no_priority\n ) : graph_node(g), input_impl_type(g, concurrency, body, a_priority) {\n fgt_multioutput_node_with_body(\n CODEPTR(), FLOW_MULTIFUNCTION_NODE,\n &this->my_graph, static_cast *>(this),\n this->output_ports(), this->my_body\n );\n }\n\n template \n __TBB_requires(multifunction_node_body)\n __TBB_NOINLINE_SYM multifunction_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority)\n : multifunction_node(g, concurrency, body, Policy(), a_priority) {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(multifunction_node_body)\n __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body,\n Policy p = Policy(), node_priority_t a_priority = no_priority)\n : multifunction_node(nodes.graph_reference(), concurrency, body, p, a_priority) {\n make_edges_in_order(nodes, *this);\n }\n\n template \n __TBB_requires(multifunction_node_body)\n __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority)\n : multifunction_node(nodes, concurrency, body, Policy(), a_priority) {}\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n __TBB_NOINLINE_SYM multifunction_node( const multifunction_node &other) :\n graph_node(other.my_graph), input_impl_type(other) {\n fgt_multioutput_node_with_body( CODEPTR(), FLOW_MULTIFUNCTION_NODE,\n &this->my_graph, static_cast *>(this),\n this->output_ports(), this->my_body );\n }\n\n // all the guts are in multifunction_input...\nprotected:\n void reset_node(reset_flags f) override { input_impl_type::reset(f); }\n}; // multifunction_node\n\n//! split_node: accepts a tuple as input, forwards each element of the tuple to its\n// successors. The node has unlimited concurrency, so it does not reject inputs.\ntemplate\nclass split_node : public graph_node, public receiver {\n static const int N = std::tuple_size::value;\n typedef receiver base_type;\npublic:\n typedef TupleType input_type;\n typedef typename wrap_tuple_elements<\n N, // #elements in tuple\n multifunction_output, // wrap this around each element\n TupleType // the tuple providing the types\n >::type output_ports_type;\n\n __TBB_NOINLINE_SYM explicit split_node(graph &g)\n : graph_node(g),\n my_output_ports(init_output_ports::call(g, my_output_ports))\n {\n fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph,\n static_cast *>(this), this->output_ports());\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_NOINLINE_SYM split_node(const node_set& nodes) : split_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n __TBB_NOINLINE_SYM split_node(const split_node& other)\n : graph_node(other.my_graph), base_type(other),\n my_output_ports(init_output_ports::call(other.my_graph, my_output_ports))\n {\n fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph,\n static_cast *>(this), this->output_ports());\n }\n\n output_ports_type &output_ports() { return my_output_ports; }\n\nprotected:\n graph_task *try_put_task(const TupleType& t) override {\n // Sending split messages in parallel is not justified, as overheads would prevail.\n // Also, we do not have successors here. So we just tell the task returned here is successful.\n return emit_element::emit_this(this->my_graph, t, output_ports());\n }\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const TupleType& t, const message_metainfo& metainfo) override {\n // Sending split messages in parallel is not justified, as overheads would prevail.\n // Also, we do not have successors here. So we just tell the task returned here is successful.\n return emit_element::emit_this(this->my_graph, t, output_ports(), metainfo);\n }\n#endif\n\n void reset_node(reset_flags f) override {\n if (f & rf_clear_edges)\n clear_element::clear_this(my_output_ports);\n\n __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), \"split_node reset failed\");\n }\n graph& graph_reference() const override {\n return my_graph;\n }\n\nprivate:\n output_ports_type my_output_ports;\n};\n\n//! Implements an executable node that supports continue_msg -> Output\ntemplate >\n __TBB_requires(std::copy_constructible)\nclass continue_node : public graph_node, public continue_input,\n public function_output {\npublic:\n typedef continue_msg input_type;\n typedef Output output_type;\n typedef continue_input input_impl_type;\n typedef function_output fOutput_type;\n typedef typename input_impl_type::predecessor_type predecessor_type;\n typedef typename fOutput_type::successor_type successor_type;\n\n //! Constructor for executable node with continue_msg -> Output\n template \n __TBB_requires(continue_node_body)\n __TBB_NOINLINE_SYM continue_node(\n graph &g,\n Body body, Policy = Policy(), node_priority_t a_priority = no_priority\n ) : graph_node(g), input_impl_type( g, body, a_priority ),\n fOutput_type(g) {\n fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph,\n\n static_cast *>(this),\n static_cast *>(this), this->my_body );\n }\n\n template \n __TBB_requires(continue_node_body)\n continue_node( graph& g, Body body, node_priority_t a_priority )\n : continue_node(g, body, Policy(), a_priority) {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(continue_node_body)\n continue_node( const node_set& nodes, Body body,\n Policy p = Policy(), node_priority_t a_priority = no_priority )\n : continue_node(nodes.graph_reference(), body, p, a_priority ) {\n make_edges_in_order(nodes, *this);\n }\n template \n __TBB_requires(continue_node_body)\n continue_node( const node_set& nodes, Body body, node_priority_t a_priority)\n : continue_node(nodes, body, Policy(), a_priority) {}\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n //! Constructor for executable node with continue_msg -> Output\n template \n __TBB_requires(continue_node_body)\n __TBB_NOINLINE_SYM continue_node(\n graph &g, int number_of_predecessors,\n Body body, Policy = Policy(), node_priority_t a_priority = no_priority\n ) : graph_node(g)\n , input_impl_type(g, number_of_predecessors, body, a_priority),\n fOutput_type(g) {\n fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph,\n static_cast *>(this),\n static_cast *>(this), this->my_body );\n }\n\n template \n __TBB_requires(continue_node_body)\n continue_node( graph& g, int number_of_predecessors, Body body, node_priority_t a_priority)\n : continue_node(g, number_of_predecessors, body, Policy(), a_priority) {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(continue_node_body)\n continue_node( const node_set& nodes, int number_of_predecessors,\n Body body, Policy p = Policy(), node_priority_t a_priority = no_priority )\n : continue_node(nodes.graph_reference(), number_of_predecessors, body, p, a_priority) {\n make_edges_in_order(nodes, *this);\n }\n\n template \n __TBB_requires(continue_node_body)\n continue_node( const node_set& nodes, int number_of_predecessors,\n Body body, node_priority_t a_priority )\n : continue_node(nodes, number_of_predecessors, body, Policy(), a_priority) {}\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM continue_node( const continue_node& src ) :\n graph_node(src.my_graph), input_impl_type(src),\n function_output(src.my_graph) {\n fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph,\n static_cast *>(this),\n static_cast *>(this), this->my_body );\n }\n\nprotected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n using input_impl_type::try_put_task;\n broadcast_cache &successors () override { return fOutput_type::my_successors; }\n\n void reset_node(reset_flags f) override {\n input_impl_type::reset_receiver(f);\n if(f & rf_clear_edges)successors().clear();\n __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), \"continue_node not reset\");\n }\n}; // continue_node\n\n//! Forwards messages of type T to all successors\ntemplate \nclass broadcast_node : public graph_node, public receiver, public sender {\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\nprivate:\n broadcast_cache my_successors;\npublic:\n\n __TBB_NOINLINE_SYM explicit broadcast_node(graph& g) : graph_node(g), my_successors(this) {\n fgt_node( CODEPTR(), FLOW_BROADCAST_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n broadcast_node(const node_set& nodes) : broadcast_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM broadcast_node( const broadcast_node& src ) : broadcast_node(src.my_graph) {}\n\n //! Adds a successor\n bool register_successor( successor_type &r ) override {\n my_successors.register_successor( r );\n return true;\n }\n\n //! Removes s as a successor\n bool remove_successor( successor_type &r ) override {\n my_successors.remove_successor( r );\n return true;\n }\n\nprivate:\n graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n graph_task* new_task = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n if (!new_task) new_task = SUCCESSFULLY_ENQUEUED;\n return new_task;\n }\n\nprotected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n //! build a task to run the successor if possible. Default is old behavior.\n graph_task* try_put_task(const T& t) override {\n return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override {\n return try_put_task_impl(t, metainfo);\n }\n#endif\n\n graph& graph_reference() const override {\n return my_graph;\n }\n\n void reset_node(reset_flags f) override {\n if (f&rf_clear_edges) {\n my_successors.clear();\n }\n __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), \"Error resetting broadcast_node\");\n }\n}; // broadcast_node\n\n//! Forwards messages in arbitrary order\ntemplate \nclass buffer_node\n : public graph_node\n , public reservable_item_buffer< T, cache_aligned_allocator >\n , public receiver, public sender\n{\n typedef cache_aligned_allocator internals_allocator;\n\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n typedef buffer_node class_type;\n\nprotected:\n typedef size_t size_type;\n round_robin_cache< T, null_rw_mutex > my_successors;\n\n friend class forward_task_bypass< class_type >;\n\n enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task\n };\n\n // implements the aggregator_operation concept\n class buffer_operation : public d1::aggregated_operation< buffer_operation > {\n public:\n char type;\n T* elem;\n graph_task* ltask;\n successor_type *r;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo* metainfo{ nullptr };\n#endif\n\n buffer_operation(const T& e, op_type t) : type(char(t))\n , elem(const_cast(&e)) , ltask(nullptr)\n , r(nullptr)\n {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n buffer_operation(const T& e, op_type t, const message_metainfo& info)\n : type(char(t)), elem(const_cast(&e)), ltask(nullptr), r(nullptr)\n , metainfo(const_cast(&info))\n {}\n\n buffer_operation(op_type t, message_metainfo& info)\n : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr), metainfo(&info) {}\n#endif\n buffer_operation(op_type t) : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr) {}\n };\n\n bool forwarder_busy;\n typedef d1::aggregating_functor handler_type;\n friend class d1::aggregating_functor;\n d1::aggregator< handler_type, buffer_operation> my_aggregator;\n\n virtual void handle_operations(buffer_operation *op_list) {\n handle_operations_impl(op_list, this);\n }\n\n template\n void handle_operations_impl(buffer_operation *op_list, derived_type* derived) {\n __TBB_ASSERT(static_cast(derived) == this, \"'this' is not a base class for derived\");\n\n buffer_operation *tmp = nullptr;\n bool try_forwarding = false;\n while (op_list) {\n tmp = op_list;\n op_list = op_list->next;\n switch (tmp->type) {\n case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break;\n case rem_succ: internal_rem_succ(tmp); break;\n case req_item: internal_pop(tmp); break;\n case res_item: internal_reserve(tmp); break;\n case rel_res: internal_release(tmp); try_forwarding = true; break;\n case con_res: internal_consume(tmp); try_forwarding = true; break;\n case put_item: try_forwarding = internal_push(tmp); break;\n case try_fwd_task: internal_forward_task(tmp); break;\n }\n }\n\n derived->order();\n\n if (try_forwarding && !forwarder_busy) {\n if(is_graph_active(this->my_graph)) {\n forwarder_busy = true;\n typedef forward_task_bypass task_type;\n d1::small_object_allocator allocator{};\n graph_task* new_task = allocator.new_object(graph_reference(), allocator, *this);\n // tmp should point to the last item handled by the aggregator. This is the operation\n // the handling thread enqueued. So modifying that record will be okay.\n // TODO revamp: check that the issue is still present\n // workaround for icc bug (at least 12.0 and 13.0)\n // error: function \"tbb::flow::interfaceX::combine_tasks\" cannot be called with the given argument list\n // argument types are: (graph, graph_task *, graph_task *)\n graph_task *z = tmp->ltask;\n graph &g = this->my_graph;\n tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task\n }\n }\n } // handle_operations\n\n inline graph_task *grab_forwarding_task( buffer_operation &op_data) {\n return op_data.ltask;\n }\n\n inline bool enqueue_forwarding_task(buffer_operation &op_data) {\n graph_task *ft = grab_forwarding_task(op_data);\n if(ft) {\n spawn_in_graph_arena(graph_reference(), *ft);\n return true;\n }\n return false;\n }\n\n //! This is executed by an enqueued task, the \"forwarder\"\n virtual graph_task *forward_task() {\n buffer_operation op_data(try_fwd_task);\n graph_task *last_task = nullptr;\n do {\n op_data.status = WAIT;\n op_data.ltask = nullptr;\n my_aggregator.execute(&op_data);\n\n // workaround for icc bug\n graph_task *xtask = op_data.ltask;\n graph& g = this->my_graph;\n last_task = combine_tasks(g, last_task, xtask);\n } while (op_data.status ==SUCCEEDED);\n return last_task;\n }\n\n //! Register successor\n virtual void internal_reg_succ(buffer_operation *op) {\n __TBB_ASSERT(op->r, nullptr);\n my_successors.register_successor(*(op->r));\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n\n //! Remove successor\n virtual void internal_rem_succ(buffer_operation *op) {\n __TBB_ASSERT(op->r, nullptr);\n my_successors.remove_successor(*(op->r));\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n\nprivate:\n void order() {}\n\n bool is_item_valid() {\n return this->my_item_valid(this->my_tail - 1);\n }\n\n void try_put_and_add_task(graph_task*& last_task) {\n graph_task* new_task = my_successors.try_put_task(this->back()\n __TBB_FLOW_GRAPH_METAINFO_ARG(this->back_metainfo()));\n if (new_task) {\n // workaround for icc bug\n graph& g = this->my_graph;\n last_task = combine_tasks(g, last_task, new_task);\n this->destroy_back();\n }\n }\n\nprotected:\n //! Tries to forward valid items to successors\n virtual void internal_forward_task(buffer_operation *op) {\n internal_forward_task_impl(op, this);\n }\n\n template\n void internal_forward_task_impl(buffer_operation *op, derived_type* derived) {\n __TBB_ASSERT(static_cast(derived) == this, \"'this' is not a base class for derived\");\n\n if (this->my_reserved || !derived->is_item_valid()) {\n op->status.store(FAILED, std::memory_order_release);\n this->forwarder_busy = false;\n return;\n }\n // Try forwarding, giving each successor a chance\n graph_task* last_task = nullptr;\n size_type counter = my_successors.size();\n for (; counter > 0 && derived->is_item_valid(); --counter)\n derived->try_put_and_add_task(last_task);\n\n op->ltask = last_task; // return task\n if (last_task && !counter) {\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n else {\n op->status.store(FAILED, std::memory_order_release);\n forwarder_busy = false;\n }\n }\n\n virtual bool internal_push(buffer_operation *op) {\n __TBB_ASSERT(op->elem, nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(op->metainfo, nullptr);\n this->push_back(*(op->elem), (*op->metainfo));\n#else\n this->push_back(*(op->elem));\n#endif\n op->status.store(SUCCEEDED, std::memory_order_release);\n return true;\n }\n\n virtual void internal_pop(buffer_operation *op) {\n __TBB_ASSERT(op->elem, nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool pop_result = op->metainfo ? this->pop_back(*(op->elem), *(op->metainfo))\n : this->pop_back(*(op->elem));\n#else\n bool pop_result = this->pop_back(*(op->elem));\n#endif\n if (pop_result) {\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n else {\n op->status.store(FAILED, std::memory_order_release);\n }\n }\n\n virtual void internal_reserve(buffer_operation *op) {\n __TBB_ASSERT(op->elem, nullptr);\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool reserve_result = op->metainfo ? this->reserve_front(*(op->elem), *(op->metainfo))\n : this->reserve_front(*(op->elem));\n#else\n bool reserve_result = this->reserve_front(*(op->elem));\n#endif\n if (reserve_result) {\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n else {\n op->status.store(FAILED, std::memory_order_release);\n }\n }\n\n virtual void internal_consume(buffer_operation *op) {\n this->consume_front();\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n\n virtual void internal_release(buffer_operation *op) {\n this->release_front();\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n\npublic:\n //! Constructor\n __TBB_NOINLINE_SYM explicit buffer_node( graph &g )\n : graph_node(g), reservable_item_buffer(), receiver(),\n sender(), my_successors(this), forwarder_busy(false)\n {\n my_aggregator.initialize_handler(handler_type(this));\n fgt_node( CODEPTR(), FLOW_BUFFER_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n buffer_node(const node_set& nodes) : buffer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM buffer_node( const buffer_node& src ) : buffer_node(src.my_graph) {}\n\n //\n // message sender implementation\n //\n\n //! Adds a new successor.\n /** Adds successor r to the list of successors; may forward tasks. */\n bool register_successor( successor_type &r ) override {\n buffer_operation op_data(reg_succ);\n op_data.r = &r;\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return true;\n }\n\n //! Removes a successor.\n /** Removes successor r from the list of successors.\n It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */\n bool remove_successor( successor_type &r ) override {\n // TODO revamp: investigate why full qualification is necessary here\n tbb::detail::d2::remove_predecessor(r, *this);\n buffer_operation op_data(rem_succ);\n op_data.r = &r;\n my_aggregator.execute(&op_data);\n // even though this operation does not cause a forward, if we are the handler, and\n // a forward is scheduled, we may be the first to reach this point after the aggregator,\n // and so should check for the task.\n (void)enqueue_forwarding_task(op_data);\n return true;\n }\n\n //! Request an item from the buffer_node\n /** true = v contains the returned item
\n false = no item has been returned */\n bool try_get( T &v ) override {\n buffer_operation op_data(req_item);\n op_data.elem = &v;\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return (op_data.status==SUCCEEDED);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_get( T &v, message_metainfo& metainfo ) override {\n buffer_operation op_data(req_item, metainfo);\n op_data.elem = &v;\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return (op_data.status==SUCCEEDED);\n }\n#endif\n\n //! Reserves an item.\n /** false = no item can be reserved
\n true = an item is reserved */\n bool try_reserve( T &v ) override {\n buffer_operation op_data(res_item);\n op_data.elem = &v;\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return (op_data.status==SUCCEEDED);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_reserve( output_type& v, message_metainfo& metainfo ) override {\n buffer_operation op_data(res_item, metainfo);\n op_data.elem = &v;\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return op_data.status==SUCCEEDED;\n }\n#endif\n\n //! Release a reserved item.\n /** true = item has been released and so remains in sender */\n bool try_release() override {\n buffer_operation op_data(rel_res);\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return true;\n }\n\n //! Consumes a reserved item.\n /** true = item is removed from sender and reservation removed */\n bool try_consume() override {\n buffer_operation op_data(con_res);\n my_aggregator.execute(&op_data);\n (void)enqueue_forwarding_task(op_data);\n return true;\n }\n\nprivate:\n graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n buffer_operation op_data(t, put_item __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n my_aggregator.execute(&op_data);\n graph_task *ft = grab_forwarding_task(op_data);\n // sequencer_nodes can return failure (if an item has been previously inserted)\n // We have to spawn the returned task if our own operation fails.\n\n if(ft && op_data.status ==FAILED) {\n // we haven't succeeded queueing the item, but for some reason the\n // call returned a task (if another request resulted in a successful\n // forward this could happen.) Queue the task and reset the pointer.\n spawn_in_graph_arena(graph_reference(), *ft); ft = nullptr;\n }\n else if(!ft && op_data.status ==SUCCEEDED) {\n ft = SUCCESSFULLY_ENQUEUED;\n }\n return ft;\n }\n\nprotected:\n\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n //! receive an item, return a task *if possible\n graph_task *try_put_task(const T &t) override {\n return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override {\n return try_put_task_impl(t, metainfo);\n }\n#endif\n\n graph& graph_reference() const override {\n return my_graph;\n }\n\nprotected:\n void reset_node( reset_flags f) override {\n reservable_item_buffer::reset();\n // TODO: just clear structures\n if (f&rf_clear_edges) {\n my_successors.clear();\n }\n forwarder_busy = false;\n }\n}; // buffer_node\n\n//! Forwards messages in FIFO order\ntemplate \nclass queue_node : public buffer_node {\nprotected:\n typedef buffer_node base_type;\n typedef typename base_type::size_type size_type;\n typedef typename base_type::buffer_operation queue_operation;\n typedef queue_node class_type;\n\nprivate:\n template friend class buffer_node;\n\n bool is_item_valid() {\n return this->my_item_valid(this->my_head);\n }\n\n void try_put_and_add_task(graph_task*& last_task) {\n graph_task* new_task = this->my_successors.try_put_task(this->front()\n __TBB_FLOW_GRAPH_METAINFO_ARG(this->front_metainfo()));\n\n if (new_task) {\n // workaround for icc bug\n graph& graph_ref = this->graph_reference();\n last_task = combine_tasks(graph_ref, last_task, new_task);\n this->destroy_front();\n }\n }\n\nprotected:\n void internal_forward_task(queue_operation *op) override {\n this->internal_forward_task_impl(op, this);\n }\n\n void internal_pop(queue_operation *op) override {\n if ( this->my_reserved || !this->my_item_valid(this->my_head)){\n op->status.store(FAILED, std::memory_order_release);\n }\n else {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (op->metainfo) {\n this->pop_front(*(op->elem), *(op->metainfo));\n } else\n#endif\n {\n this->pop_front(*(op->elem));\n }\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n }\n void internal_reserve(queue_operation *op) override {\n if (this->my_reserved || !this->my_item_valid(this->my_head)) {\n op->status.store(FAILED, std::memory_order_release);\n }\n else {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (op->metainfo) {\n this->reserve_front(*(op->elem), *(op->metainfo));\n }\n else\n#endif\n {\n this->reserve_front(*(op->elem));\n }\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n }\n void internal_consume(queue_operation *op) override {\n this->consume_front();\n op->status.store(SUCCEEDED, std::memory_order_release);\n }\n\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n\n //! Constructor\n __TBB_NOINLINE_SYM explicit queue_node( graph &g ) : base_type(g) {\n fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n queue_node( const node_set& nodes) : queue_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM queue_node( const queue_node& src) : base_type(src) {\n fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n\nprotected:\n void reset_node( reset_flags f) override {\n base_type::reset_node(f);\n }\n}; // queue_node\n\n//! Forwards messages in sequence order\ntemplate \n __TBB_requires(std::copyable)\nclass sequencer_node : public queue_node {\n function_body< T, size_t > *my_sequencer;\n // my_sequencer should be a benign function and must be callable\n // from a parallel context. Does this mean it needn't be reset?\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n\n //! Constructor\n template< typename Sequencer >\n __TBB_requires(sequencer)\n __TBB_NOINLINE_SYM sequencer_node( graph &g, const Sequencer& s ) : queue_node(g),\n my_sequencer(new function_body_leaf< T, size_t, Sequencer>(s) ) {\n fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(sequencer)\n sequencer_node( const node_set& nodes, const Sequencer& s)\n : sequencer_node(nodes.graph_reference(), s) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM sequencer_node( const sequencer_node& src ) : queue_node(src),\n my_sequencer( src.my_sequencer->clone() ) {\n fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n //! Destructor\n ~sequencer_node() { delete my_sequencer; }\n\nprotected:\n typedef typename buffer_node::size_type size_type;\n typedef typename buffer_node::buffer_operation sequencer_operation;\n\nprivate:\n bool internal_push(sequencer_operation *op) override {\n size_type tag = (*my_sequencer)(*(op->elem));\n#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES\n if (tag < this->my_head) {\n // have already emitted a message with this tag\n op->status.store(FAILED, std::memory_order_release);\n return false;\n }\n#endif\n // cannot modify this->my_tail now; the buffer would be inconsistent.\n size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail;\n\n if (this->size(new_tail) > this->capacity()) {\n this->grow_my_array(this->size(new_tail));\n }\n this->my_tail = new_tail;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(op->metainfo, nullptr);\n bool place_item_result = this->place_item(tag, *(op->elem), *(op->metainfo));\n const op_stat res = place_item_result ? SUCCEEDED : FAILED;\n#else\n const op_stat res = this->place_item(tag, *(op->elem)) ? SUCCEEDED : FAILED;\n#endif\n op->status.store(res, std::memory_order_release);\n return res ==SUCCEEDED;\n }\n}; // sequencer_node\n\n//! Forwards messages in priority order\ntemplate>\nclass priority_queue_node : public buffer_node {\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef buffer_node base_type;\n typedef priority_queue_node class_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n\n //! Constructor\n __TBB_NOINLINE_SYM explicit priority_queue_node( graph &g, const Compare& comp = Compare() )\n : buffer_node(g), compare(comp), mark(0) {\n fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n priority_queue_node(const node_set& nodes, const Compare& comp = Compare())\n : priority_queue_node(nodes.graph_reference(), comp) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor\n __TBB_NOINLINE_SYM priority_queue_node( const priority_queue_node &src )\n : buffer_node(src), mark(0)\n {\n fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\nprotected:\n\n void reset_node( reset_flags f) override {\n mark = 0;\n base_type::reset_node(f);\n }\n\n typedef typename buffer_node::size_type size_type;\n typedef typename buffer_node::item_type item_type;\n typedef typename buffer_node::buffer_operation prio_operation;\n\n //! Tries to forward valid items to successors\n void internal_forward_task(prio_operation *op) override {\n this->internal_forward_task_impl(op, this);\n }\n\n void handle_operations(prio_operation *op_list) override {\n this->handle_operations_impl(op_list, this);\n }\n\n bool internal_push(prio_operation *op) override {\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n __TBB_ASSERT(op->metainfo, nullptr);\n prio_push(*(op->elem), *(op->metainfo));\n#else\n prio_push(*(op->elem));\n#endif\n op->status.store(SUCCEEDED, std::memory_order_release);\n return true;\n }\n\n void internal_pop(prio_operation *op) override {\n // if empty or already reserved, don't pop\n if ( this->my_reserved == true || this->my_tail == 0 ) {\n op->status.store(FAILED, std::memory_order_release);\n return;\n }\n\n *(op->elem) = prio();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (op->metainfo) {\n *(op->metainfo) = std::move(prio_metainfo());\n }\n#endif\n op->status.store(SUCCEEDED, std::memory_order_release);\n prio_pop();\n\n }\n\n // pops the highest-priority item, saves copy\n void internal_reserve(prio_operation *op) override {\n if (this->my_reserved == true || this->my_tail == 0) {\n op->status.store(FAILED, std::memory_order_release);\n return;\n }\n this->my_reserved = true;\n *(op->elem) = prio();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n if (op->metainfo) {\n *(op->metainfo) = std::move(prio_metainfo());\n reserved_metainfo = *(op->metainfo);\n }\n#endif\n reserved_item = *(op->elem);\n op->status.store(SUCCEEDED, std::memory_order_release);\n prio_pop();\n }\n\n void internal_consume(prio_operation *op) override {\n op->status.store(SUCCEEDED, std::memory_order_release);\n this->my_reserved = false;\n reserved_item = input_type();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (auto waiter : reserved_metainfo.waiters()) {\n waiter->release(1);\n }\n\n reserved_metainfo = message_metainfo{};\n#endif\n }\n\n void internal_release(prio_operation *op) override {\n op->status.store(SUCCEEDED, std::memory_order_release);\n prio_push(reserved_item __TBB_FLOW_GRAPH_METAINFO_ARG(reserved_metainfo));\n this->my_reserved = false;\n reserved_item = input_type();\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (auto waiter : reserved_metainfo.waiters()) {\n waiter->release(1);\n }\n\n reserved_metainfo = message_metainfo{};\n#endif\n }\n\nprivate:\n template friend class buffer_node;\n\n void order() {\n if (mark < this->my_tail) heapify();\n __TBB_ASSERT(mark == this->my_tail, \"mark unequal after heapify\");\n }\n\n bool is_item_valid() {\n return this->my_tail > 0;\n }\n\n void try_put_and_add_task(graph_task*& last_task) {\n graph_task* new_task = this->my_successors.try_put_task(this->prio()\n __TBB_FLOW_GRAPH_METAINFO_ARG(this->prio_metainfo()));\n if (new_task) {\n // workaround for icc bug\n graph& graph_ref = this->graph_reference();\n last_task = combine_tasks(graph_ref, last_task, new_task);\n prio_pop();\n }\n }\n\nprivate:\n Compare compare;\n size_type mark;\n\n input_type reserved_item;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo reserved_metainfo;\n#endif\n\n // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item\n bool prio_use_tail() {\n __TBB_ASSERT(mark <= this->my_tail, \"mark outside bounds before test\");\n return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1));\n }\n\n // prio_push: checks that the item will fit, expand array if necessary, put at end\n void prio_push(const T &src __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n if ( this->my_tail >= this->my_array_size )\n this->grow_my_array( this->my_tail + 1 );\n (void) this->place_item(this->my_tail, src __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n ++(this->my_tail);\n __TBB_ASSERT(mark < this->my_tail, \"mark outside bounds after push\");\n }\n\n // prio_pop: deletes highest priority item from the array, and if it is item\n // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail\n // and mark. Assumes the array has already been tested for emptiness; no failure.\n void prio_pop() {\n if (prio_use_tail()) {\n // there are newly pushed elements; last one higher than top\n // copy the data\n this->destroy_item(this->my_tail-1);\n --(this->my_tail);\n __TBB_ASSERT(mark <= this->my_tail, \"mark outside bounds after pop\");\n return;\n }\n this->destroy_item(0);\n if(this->my_tail > 1) {\n // push the last element down heap\n __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), nullptr);\n this->move_item(0,this->my_tail - 1);\n }\n --(this->my_tail);\n if(mark > this->my_tail) --mark;\n if (this->my_tail > 1) // don't reheap for heap of size 1\n reheap();\n __TBB_ASSERT(mark <= this->my_tail, \"mark outside bounds after pop\");\n }\n\n const T& prio() {\n return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo& prio_metainfo() {\n return this->get_my_metainfo(prio_use_tail() ? this->my_tail-1 : 0);\n }\n#endif\n\n // turn array into heap\n void heapify() {\n if(this->my_tail == 0) {\n mark = 0;\n return;\n }\n if (!mark) mark = 1;\n for (; markmy_tail; ++mark) { // for each unheaped element\n size_type cur_pos = mark;\n input_type to_place;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n this->fetch_item(mark, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n do { // push to_place up the heap\n size_type parent = (cur_pos-1)>>1;\n if (!compare(this->get_my_item(parent), to_place))\n break;\n this->move_item(cur_pos, parent);\n cur_pos = parent;\n } while( cur_pos );\n this->place_item(cur_pos, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo)));\n }\n }\n\n // otherwise heapified array with new root element; rearrange to heap\n void reheap() {\n size_type cur_pos=0, child=1;\n while (child < mark) {\n size_type target = child;\n if (child+1get_my_item(child),\n this->get_my_item(child+1)))\n ++target;\n // target now has the higher priority child\n if (compare(this->get_my_item(target),\n this->get_my_item(cur_pos)))\n break;\n // swap\n this->swap_items(cur_pos, target);\n cur_pos = target;\n child = (cur_pos<<1)+1;\n }\n }\n}; // priority_queue_node\n\n//! Forwards messages only if the threshold has not been reached\n/** This node forwards items until its threshold is reached.\n It contains no buffering. If the downstream node rejects, the\n message is dropped. */\ntemplate< typename T, typename DecrementType=continue_msg >\nclass limiter_node : public graph_node, public receiver< T >, public sender< T > {\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n //TODO: There is a lack of predefined types for its controlling \"decrementer\" port. It should be fixed later.\n\nprivate:\n size_t my_threshold;\n size_t my_count; // number of successful puts\n size_t my_tries; // number of active put attempts\n size_t my_future_decrement; // number of active decrement\n reservable_predecessor_cache< T, spin_mutex > my_predecessors;\n spin_mutex my_mutex;\n broadcast_cache< T > my_successors;\n\n //! The internal receiver< DecrementType > that adjusts the count\n threshold_regulator< limiter_node, DecrementType > decrement;\n\n graph_task* decrement_counter( long long delta ) {\n if ( delta > 0 && size_t(delta) > my_threshold ) {\n delta = my_threshold;\n }\n\n {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( delta > 0 && size_t(delta) > my_count ) {\n if( my_tries > 0 ) {\n my_future_decrement += (size_t(delta) - my_count);\n }\n my_count = 0;\n }\n else if ( delta < 0 && size_t(-delta) > my_threshold - my_count ) {\n my_count = my_threshold;\n }\n else {\n my_count -= size_t(delta); // absolute value of delta is sufficiently small\n }\n __TBB_ASSERT(my_count <= my_threshold, \"counter values are truncated to be inside the [0, threshold] interval\");\n }\n return forward_task();\n }\n\n // Let threshold_regulator call decrement_counter()\n friend class threshold_regulator< limiter_node, DecrementType >;\n\n friend class forward_task_bypass< limiter_node >;\n\n bool check_conditions() { // always called under lock\n return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() );\n }\n\n // only returns a valid task pointer or nullptr, never SUCCESSFULLY_ENQUEUED\n graph_task* forward_task() {\n input_type v;\n graph_task* rval = nullptr;\n bool reserved = false;\n\n {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( check_conditions() )\n ++my_tries;\n else\n return nullptr;\n }\n\n //SUCCESS\n // if we can reserve and can put, we consume the reservation\n // we increment the count and decrement the tries\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo metainfo;\n#endif\n if ( (my_predecessors.try_reserve(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) == true ) {\n reserved = true;\n if ( (rval = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) != nullptr ) {\n {\n spin_mutex::scoped_lock lock(my_mutex);\n ++my_count;\n if ( my_future_decrement ) {\n if ( my_count > my_future_decrement ) {\n my_count -= my_future_decrement;\n my_future_decrement = 0;\n }\n else {\n my_future_decrement -= my_count;\n my_count = 0;\n }\n }\n --my_tries;\n my_predecessors.try_consume();\n if ( check_conditions() ) {\n if ( is_graph_active(this->my_graph) ) {\n typedef forward_task_bypass> task_type;\n d1::small_object_allocator allocator{};\n graph_task* rtask = allocator.new_object( my_graph, allocator, *this );\n spawn_in_graph_arena(graph_reference(), *rtask);\n }\n }\n }\n return rval;\n }\n }\n //FAILURE\n //if we can't reserve, we decrement the tries\n //if we can reserve but can't put, we decrement the tries and release the reservation\n {\n spin_mutex::scoped_lock lock(my_mutex);\n --my_tries;\n if (reserved) my_predecessors.try_release();\n if ( check_conditions() ) {\n if ( is_graph_active(this->my_graph) ) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass> task_type;\n graph_task* t = allocator.new_object(my_graph, allocator, *this);\n __TBB_ASSERT(!rval, \"Have two tasks to handle\");\n return t;\n }\n }\n return rval;\n }\n }\n\n void initialize() {\n fgt_node(\n CODEPTR(), FLOW_LIMITER_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(&decrement),\n static_cast *>(this)\n );\n }\n\npublic:\n //! Constructor\n limiter_node(graph &g, size_t threshold)\n : graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), my_future_decrement(0),\n my_predecessors(this), my_successors(this), decrement(this)\n {\n initialize();\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n limiter_node(const node_set& nodes, size_t threshold)\n : limiter_node(nodes.graph_reference(), threshold) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor\n limiter_node( const limiter_node& src ) : limiter_node(src.my_graph, src.my_threshold) {}\n\n //! The interface for accessing internal receiver< DecrementType > that adjusts the count\n receiver& decrementer() { return decrement; }\n\n //! Replace the current successor with this new successor\n bool register_successor( successor_type &r ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n bool was_empty = my_successors.empty();\n my_successors.register_successor(r);\n //spawn a forward task if this is the only successor\n if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) {\n if ( is_graph_active(this->my_graph) ) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass> task_type;\n graph_task* t = allocator.new_object(my_graph, allocator, *this);\n spawn_in_graph_arena(graph_reference(), *t);\n }\n }\n return true;\n }\n\n //! Removes a successor from this node\n /** r.remove_predecessor(*this) is also called. */\n bool remove_successor( successor_type &r ) override {\n // TODO revamp: investigate why qualification is needed for remove_predecessor() call\n tbb::detail::d2::remove_predecessor(r, *this);\n my_successors.remove_successor(r);\n return true;\n }\n\n //! Adds src to the list of cached predecessors.\n bool register_predecessor( predecessor_type &src ) override {\n spin_mutex::scoped_lock lock(my_mutex);\n my_predecessors.add( src );\n if ( my_count + my_tries < my_threshold && !my_successors.empty() && is_graph_active(this->my_graph) ) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass> task_type;\n graph_task* t = allocator.new_object(my_graph, allocator, *this);\n spawn_in_graph_arena(graph_reference(), *t);\n }\n return true;\n }\n\n //! Removes src from the list of cached predecessors.\n bool remove_predecessor( predecessor_type &src ) override {\n my_predecessors.remove( src );\n return true;\n }\n\nprotected:\n\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n\nprivate:\n //! Puts an item to this receiver\n graph_task* try_put_task_impl( const T &t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) {\n {\n spin_mutex::scoped_lock lock(my_mutex);\n if ( my_count + my_tries >= my_threshold )\n return nullptr;\n else\n ++my_tries;\n }\n\n graph_task* rtask = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo));\n if ( !rtask ) { // try_put_task failed.\n spin_mutex::scoped_lock lock(my_mutex);\n --my_tries;\n if (check_conditions() && is_graph_active(this->my_graph)) {\n d1::small_object_allocator allocator{};\n typedef forward_task_bypass> task_type;\n rtask = allocator.new_object(my_graph, allocator, *this);\n }\n }\n else {\n spin_mutex::scoped_lock lock(my_mutex);\n ++my_count;\n if ( my_future_decrement ) {\n if ( my_count > my_future_decrement ) {\n my_count -= my_future_decrement;\n my_future_decrement = 0;\n }\n else {\n my_future_decrement -= my_count;\n my_count = 0;\n }\n }\n --my_tries;\n }\n return rtask;\n }\n\nprotected:\n graph_task* try_put_task(const T& t) override {\n return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override {\n return try_put_task_impl(t, metainfo);\n }\n#endif\n\n graph& graph_reference() const override { return my_graph; }\n\n void reset_node( reset_flags f ) override {\n my_count = 0;\n if ( f & rf_clear_edges ) {\n my_predecessors.clear();\n my_successors.clear();\n }\n else {\n my_predecessors.reset();\n }\n decrement.reset_receiver(f);\n }\n}; // limiter_node\n\n#include \"detail/_flow_graph_join_impl.h\"\n\ntemplate class join_node;\n\ntemplate\nclass join_node: public unfolded_join_node::value, reserving_port, OutputTuple, reserving> {\nprivate:\n static const int N = std::tuple_size::value;\n typedef unfolded_join_node unfolded_type;\npublic:\n typedef OutputTuple output_type;\n typedef typename unfolded_type::input_ports_type input_ports_type;\n __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_NOINLINE_SYM join_node(const node_set& nodes, reserving = reserving()) : join_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\ntemplate\nclass join_node: public unfolded_join_node::value, queueing_port, OutputTuple, queueing> {\nprivate:\n static const int N = std::tuple_size::value;\n typedef unfolded_join_node unfolded_type;\npublic:\n typedef OutputTuple output_type;\n typedef typename unfolded_type::input_ports_type input_ports_type;\n __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_NOINLINE_SYM join_node(const node_set& nodes, queueing = queueing()) : join_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\n// Helper function which is well-formed only if all of the elements in OutputTuple\n// satisfies join_node_function_object\ntemplate \nvoid join_node_function_objects_helper( std::index_sequence )\n requires (std::tuple_size_v == sizeof...(Functions)) &&\n (... && join_node_function_object, K>);\n\ntemplate \nconcept join_node_functions = requires {\n join_node_function_objects_helper(std::make_index_sequence{});\n};\n\n#endif\n\n// template for key_matching join_node\n// tag_matching join_node is a specialization of key_matching, and is source-compatible.\ntemplate\nclass join_node > : public unfolded_join_node::value,\n key_matching_port, OutputTuple, key_matching > {\nprivate:\n static const int N = std::tuple_size::value;\n typedef unfolded_join_node > unfolded_type;\npublic:\n typedef OutputTuple output_type;\n typedef typename unfolded_type::input_ports_type input_ports_type;\n\n#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING\n join_node(graph &g) : unfolded_type(g) {}\n#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */\n\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1) : unfolded_type(g, b0, b1) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2) : unfolded_type(g, b0, b1, b2) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3) : unfolded_type(g, b0, b1, b2, b3) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4) :\n unfolded_type(g, b0, b1, b2, b3, b4) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#if __TBB_VARIADIC_MAX >= 6\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5) :\n unfolded_type(g, b0, b1, b2, b3, b4, b5) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#endif\n#if __TBB_VARIADIC_MAX >= 7\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6) :\n unfolded_type(g, b0, b1, b2, b3, b4, b5, b6) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#endif\n#if __TBB_VARIADIC_MAX >= 8\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6,\n __TBB_B7 b7) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#endif\n#if __TBB_VARIADIC_MAX >= 9\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6,\n __TBB_B7 b7, __TBB_B8 b8) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#endif\n#if __TBB_VARIADIC_MAX >= 10\n template\n __TBB_requires(join_node_functions)\n __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6,\n __TBB_B7 b7, __TBB_B8 b8, __TBB_B9 b9) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n#endif\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template <\n#if (__clang_major__ == 3 && __clang_minor__ == 4)\n // clang 3.4 misdeduces 'Args...' for 'node_set' while it can cope with template template parameter.\n template class node_set,\n#endif\n typename... Args, typename... Bodies\n >\n __TBB_requires((sizeof...(Bodies) == 0) || join_node_functions)\n __TBB_NOINLINE_SYM join_node(const node_set& nodes, Bodies... bodies)\n : join_node(nodes.graph_reference(), bodies...) {\n make_edges_in_order(nodes, *this);\n }\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\n// indexer node\n#include \"detail/_flow_graph_indexer_impl.h\"\n\n// TODO: Implement interface with variadic template or tuple\ntemplate class indexer_node;\n\n//indexer node specializations\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 1;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n};\n\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 2;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 3;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 4;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 5;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n\n#if __TBB_VARIADIC_MAX >= 6\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 6;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n#endif //variadic max 6\n\n#if __TBB_VARIADIC_MAX >= 7\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 7;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n#endif //variadic max 7\n\n#if __TBB_VARIADIC_MAX >= 8\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 8;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n#endif //variadic max 8\n\n#if __TBB_VARIADIC_MAX >= 9\ntemplate\nclass indexer_node : public unfolded_indexer_node > {\nprivate:\n static const int N = 9;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n#endif //variadic max 9\n\n#if __TBB_VARIADIC_MAX >= 10\ntemplate\nclass indexer_node/*default*/ : public unfolded_indexer_node > {\nprivate:\n static const int N = 10;\npublic:\n typedef std::tuple InputTuple;\n typedef tagged_msg output_type;\n typedef unfolded_indexer_node unfolded_type;\n __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n // Copy constructor\n __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) {\n fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph,\n this->input_ports(), static_cast< sender< output_type > *>(this) );\n }\n\n};\n#endif //variadic max 10\n\ntemplate< typename T >\ninline void internal_make_edge( sender &p, receiver &s ) {\n register_successor(p, s);\n fgt_make_edge( &p, &s );\n}\n\n//! Makes an edge between a single predecessor and a single successor\ntemplate< typename T >\ninline void make_edge( sender &p, receiver &s ) {\n internal_make_edge( p, s );\n}\n\n//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor.\ntemplate< typename T, typename V,\n typename = typename T::output_ports_type, typename = typename V::input_ports_type >\ninline void make_edge( T& output, V& input) {\n make_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports()));\n}\n\n//Makes an edge from port 0 of a multi-output predecessor to a receiver.\ntemplate< typename T, typename R,\n typename = typename T::output_ports_type >\ninline void make_edge( T& output, receiver& input) {\n make_edge(std::get<0>(output.output_ports()), input);\n}\n\n//Makes an edge from a sender to port 0 of a multi-input successor.\ntemplate< typename S, typename V,\n typename = typename V::input_ports_type >\ninline void make_edge( sender& output, V& input) {\n make_edge(output, std::get<0>(input.input_ports()));\n}\n\ntemplate< typename T >\ninline void internal_remove_edge( sender &p, receiver &s ) {\n remove_successor( p, s );\n fgt_remove_edge( &p, &s );\n}\n\n//! Removes an edge between a single predecessor and a single successor\ntemplate< typename T >\ninline void remove_edge( sender &p, receiver &s ) {\n internal_remove_edge( p, s );\n}\n\n//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor.\ntemplate< typename T, typename V,\n typename = typename T::output_ports_type, typename = typename V::input_ports_type >\ninline void remove_edge( T& output, V& input) {\n remove_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports()));\n}\n\n//Removes an edge between port 0 of a multi-output predecessor and a receiver.\ntemplate< typename T, typename R,\n typename = typename T::output_ports_type >\ninline void remove_edge( T& output, receiver& input) {\n remove_edge(std::get<0>(output.output_ports()), input);\n}\n//Removes an edge between a sender and port 0 of a multi-input successor.\ntemplate< typename S, typename V,\n typename = typename V::input_ports_type >\ninline void remove_edge( sender& output, V& input) {\n remove_edge(output, std::get<0>(input.input_ports()));\n}\n\n//! Returns a copy of the body from a function or continue node\ntemplate< typename Body, typename Node >\nBody copy_body( Node &n ) {\n return n.template copy_function_object();\n}\n\n//composite_node\ntemplate< typename InputTuple, typename OutputTuple > class composite_node;\n\ntemplate< typename... InputTypes, typename... OutputTypes>\nclass composite_node , std::tuple > : public graph_node {\n\npublic:\n typedef std::tuple< receiver&... > input_ports_type;\n typedef std::tuple< sender&... > output_ports_type;\n\nprivate:\n std::unique_ptr my_input_ports;\n std::unique_ptr my_output_ports;\n\n static const size_t NUM_INPUTS = sizeof...(InputTypes);\n static const size_t NUM_OUTPUTS = sizeof...(OutputTypes);\n\nprotected:\n void reset_node(reset_flags) override {}\n\npublic:\n composite_node( graph &g ) : graph_node(g) {\n fgt_multiinput_multioutput_node( CODEPTR(), FLOW_COMPOSITE_NODE, this, &this->my_graph );\n }\n\n template\n void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) {\n static_assert(NUM_INPUTS == std::tuple_size::value, \"number of arguments does not match number of input ports\");\n static_assert(NUM_OUTPUTS == std::tuple_size::value, \"number of arguments does not match number of output ports\");\n\n fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple);\n fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple);\n\n my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) );\n my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) );\n }\n\n template< typename... NodeTypes >\n void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); }\n\n template< typename... NodeTypes >\n void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); }\n\n\n input_ports_type& input_ports() {\n __TBB_ASSERT(my_input_ports, \"input ports not set, call set_external_ports to set input ports\");\n return *my_input_ports;\n }\n\n output_ports_type& output_ports() {\n __TBB_ASSERT(my_output_ports, \"output ports not set, call set_external_ports to set output ports\");\n return *my_output_ports;\n }\n}; // class composite_node\n\n//composite_node with only input ports\ntemplate< typename... InputTypes>\nclass composite_node , std::tuple<> > : public graph_node {\npublic:\n typedef std::tuple< receiver&... > input_ports_type;\n\nprivate:\n std::unique_ptr my_input_ports;\n static const size_t NUM_INPUTS = sizeof...(InputTypes);\n\nprotected:\n void reset_node(reset_flags) override {}\n\npublic:\n composite_node( graph &g ) : graph_node(g) {\n fgt_composite( CODEPTR(), this, &g );\n }\n\n template\n void set_external_ports(T&& input_ports_tuple) {\n static_assert(NUM_INPUTS == std::tuple_size::value, \"number of arguments does not match number of input ports\");\n\n fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple);\n\n my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) );\n }\n\n template< typename... NodeTypes >\n void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); }\n\n template< typename... NodeTypes >\n void add_nodes( const NodeTypes&... n) { add_nodes_impl(this, false, n...); }\n\n\n input_ports_type& input_ports() {\n __TBB_ASSERT(my_input_ports, \"input ports not set, call set_external_ports to set input ports\");\n return *my_input_ports;\n }\n\n}; // class composite_node\n\n//composite_nodes with only output_ports\ntemplate\nclass composite_node , std::tuple > : public graph_node {\npublic:\n typedef std::tuple< sender&... > output_ports_type;\n\nprivate:\n std::unique_ptr my_output_ports;\n static const size_t NUM_OUTPUTS = sizeof...(OutputTypes);\n\nprotected:\n void reset_node(reset_flags) override {}\n\npublic:\n __TBB_NOINLINE_SYM composite_node( graph &g ) : graph_node(g) {\n fgt_composite( CODEPTR(), this, &g );\n }\n\n template\n void set_external_ports(T&& output_ports_tuple) {\n static_assert(NUM_OUTPUTS == std::tuple_size::value, \"number of arguments does not match number of output ports\");\n\n fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple);\n\n my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) );\n }\n\n template\n void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); }\n\n template\n void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); }\n\n\n output_ports_type& output_ports() {\n __TBB_ASSERT(my_output_ports, \"output ports not set, call set_external_ports to set output ports\");\n return *my_output_ports;\n }\n\n}; // class composite_node\n\ntemplate\nclass async_body_base: no_assign {\npublic:\n typedef Gateway gateway_type;\n\n async_body_base(gateway_type *gateway): my_gateway(gateway) { }\n void set_gateway(gateway_type *gateway) {\n my_gateway = gateway;\n }\n\nprotected:\n gateway_type *my_gateway;\n};\n\ntemplate\nclass async_body: public async_body_base {\nprivate:\n Body my_body;\n\npublic:\n typedef async_body_base base_type;\n typedef Gateway gateway_type;\n\n async_body(const Body &body, gateway_type *gateway)\n : base_type(gateway), my_body(body) { }\n\n void operator()( const Input &v, Ports & ) noexcept(noexcept(tbb::detail::invoke(my_body, v, std::declval()))) {\n tbb::detail::invoke(my_body, v, *this->my_gateway);\n }\n\n Body get_body() { return my_body; }\n};\n\n//! Implements async node\ntemplate < typename Input, typename Output,\n typename Policy = queueing_lightweight >\n __TBB_requires(std::default_initializable && std::copy_constructible)\nclass async_node\n : public multifunction_node< Input, std::tuple< Output >, Policy >, public sender< Output >\n{\n typedef multifunction_node< Input, std::tuple< Output >, Policy > base_type;\n typedef multifunction_input<\n Input, typename base_type::output_ports_type, Policy, cache_aligned_allocator> mfn_input_type;\n\npublic:\n typedef Input input_type;\n typedef Output output_type;\n typedef receiver receiver_type;\n typedef receiver successor_type;\n typedef sender predecessor_type;\n typedef receiver_gateway gateway_type;\n typedef async_body_base async_body_base_type;\n typedef typename base_type::output_ports_type output_ports_type;\n\nprivate:\n class receiver_gateway_impl: public receiver_gateway {\n public:\n receiver_gateway_impl(async_node* node): my_node(node) {}\n void reserve_wait() override {\n fgt_async_reserve(static_cast(my_node), &my_node->my_graph);\n my_node->my_graph.reserve_wait();\n }\n\n void release_wait() override {\n async_node* n = my_node;\n graph* g = &n->my_graph;\n g->release_wait();\n fgt_async_commit(static_cast(n), g);\n }\n\n //! Implements gateway_type::try_put for an external activity to submit a message to FG\n bool try_put(const Output &i) override {\n return my_node->try_put_impl(i);\n }\n\n private:\n async_node* my_node;\n } my_gateway;\n\n //The substitute of 'this' for member construction, to prevent compiler warnings\n async_node* self() { return this; }\n\n //! Implements gateway_type::try_put for an external activity to submit a message to FG\n bool try_put_impl(const Output &i) {\n multifunction_output &port_0 = output_port<0>(*this);\n broadcast_cache& port_successors = port_0.successors();\n fgt_async_try_put_begin(this, &port_0);\n // TODO revamp: change to std::list\n graph_task_list tasks;\n bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks);\n __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(),\n \"Return status is inconsistent with the method operation.\" );\n\n while( !tasks.empty() ) {\n enqueue_in_graph_arena(this->my_graph, tasks.pop_front());\n }\n fgt_async_try_put_end(this, &port_0);\n return is_at_least_one_put_successful;\n }\n\npublic:\n template\n __TBB_requires(async_node_body)\n __TBB_NOINLINE_SYM async_node(\n graph &g, size_t concurrency,\n Body body, Policy = Policy(), node_priority_t a_priority = no_priority\n ) : base_type(\n g, concurrency,\n async_body\n (body, &my_gateway), a_priority ), my_gateway(self()) {\n fgt_multioutput_node_with_body<1>(\n CODEPTR(), FLOW_ASYNC_NODE,\n &this->my_graph, static_cast *>(this),\n this->output_ports(), this->my_body\n );\n }\n\n template \n __TBB_requires(async_node_body)\n __TBB_NOINLINE_SYM async_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority)\n : async_node(g, concurrency, body, Policy(), a_priority) {}\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n __TBB_requires(async_node_body)\n __TBB_NOINLINE_SYM async_node(\n const node_set& nodes, size_t concurrency, Body body,\n Policy = Policy(), node_priority_t a_priority = no_priority )\n : async_node(nodes.graph_reference(), concurrency, body, a_priority) {\n make_edges_in_order(nodes, *this);\n }\n\n template \n __TBB_requires(async_node_body)\n __TBB_NOINLINE_SYM async_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority)\n : async_node(nodes, concurrency, body, Policy(), a_priority) {}\n#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n\n __TBB_NOINLINE_SYM async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) {\n static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway);\n static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway);\n\n fgt_multioutput_node_with_body<1>( CODEPTR(), FLOW_ASYNC_NODE,\n &this->my_graph, static_cast *>(this),\n this->output_ports(), this->my_body );\n }\n\n gateway_type& gateway() {\n return my_gateway;\n }\n\n // Define sender< Output >\n\n //! Add a new successor to this node\n bool register_successor(successor_type&) override {\n __TBB_ASSERT(false, \"Successors must be registered only via ports\");\n return false;\n }\n\n //! Removes a successor from this node\n bool remove_successor(successor_type&) override {\n __TBB_ASSERT(false, \"Successors must be removed only via ports\");\n return false;\n }\n\n template\n Body copy_function_object() {\n typedef multifunction_body mfn_body_type;\n typedef async_body async_body_type;\n mfn_body_type &body_ref = *this->my_body;\n async_body_type ab = *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr());\n return ab.get_body();\n }\n\nprotected:\n\n void reset_node( reset_flags f) override {\n base_type::reset_node(f);\n }\n};\n\n#include \"detail/_flow_graph_node_set_impl.h\"\n\ntemplate< typename T >\nclass overwrite_node : public graph_node, public receiver, public sender {\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n\n __TBB_NOINLINE_SYM explicit overwrite_node(graph &g)\n : graph_node(g), my_successors(this), my_buffer_is_valid(false)\n {\n fgt_node( CODEPTR(), FLOW_OVERWRITE_NODE, &this->my_graph,\n static_cast *>(this), static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n overwrite_node(const node_set& nodes) : overwrite_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor; doesn't take anything from src; default won't work\n __TBB_NOINLINE_SYM overwrite_node( const overwrite_node& src ) : overwrite_node(src.my_graph) {}\n\n ~overwrite_node() {}\n\n bool register_successor( successor_type &s ) override {\n spin_mutex::scoped_lock l( my_mutex );\n if (my_buffer_is_valid && is_graph_active( my_graph )) {\n // We have a valid value that must be forwarded immediately.\n bool ret = s.try_put( my_buffer );\n if ( ret ) {\n // We add the successor that accepted our put\n my_successors.register_successor( s );\n } else {\n // In case of reservation a race between the moment of reservation and register_successor can appear,\n // because failed reserve does not mean that register_successor is not ready to put a message immediately.\n // We have some sort of infinite loop: reserving node tries to set pull state for the edge,\n // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation.\n d1::small_object_allocator allocator{};\n typedef register_predecessor_task task_type;\n graph_task* t = allocator.new_object(graph_reference(), allocator, *this, s);\n spawn_in_graph_arena( my_graph, *t );\n }\n } else {\n // No valid value yet, just add as successor\n my_successors.register_successor( s );\n }\n return true;\n }\n\n bool remove_successor( successor_type &s ) override {\n spin_mutex::scoped_lock l( my_mutex );\n my_successors.remove_successor(s);\n return true;\n }\n\n bool try_get( input_type &v ) override {\n spin_mutex::scoped_lock l( my_mutex );\n if ( my_buffer_is_valid ) {\n v = my_buffer;\n return true;\n }\n return false;\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n bool try_get( input_type &v, message_metainfo& metainfo ) override {\n spin_mutex::scoped_lock l( my_mutex );\n if (my_buffer_is_valid) {\n v = my_buffer;\n metainfo = my_buffered_metainfo;\n\n // Since the successor of the node will use move semantics while wrapping the metainfo\n // that is designed to transfer the ownership of the value from single-push buffer to the task\n // It is required to reserve one more reference here because the value keeps in the buffer\n // and the ownership is not transferred\n for (auto msg_waiter : metainfo.waiters()) {\n msg_waiter->reserve(1);\n }\n return true;\n }\n return false;\n }\n#endif\n\n //! Reserves an item\n bool try_reserve( T &v ) override {\n return try_get(v);\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\nprivate:\n bool try_reserve(T& v, message_metainfo& metainfo) override {\n spin_mutex::scoped_lock l( my_mutex );\n if (my_buffer_is_valid) {\n v = my_buffer;\n metainfo = my_buffered_metainfo;\n return true;\n }\n return false;\n }\npublic:\n#endif\n\n //! Releases the reserved item\n bool try_release() override { return true; }\n\n //! Consumes the reserved item\n bool try_consume() override { return true; }\n\n bool is_valid() {\n spin_mutex::scoped_lock l( my_mutex );\n return my_buffer_is_valid;\n }\n\n void clear() {\n spin_mutex::scoped_lock l( my_mutex );\n my_buffer_is_valid = false;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n for (auto msg_waiter : my_buffered_metainfo.waiters()) {\n msg_waiter->release(1);\n }\n my_buffered_metainfo = message_metainfo{};\n#endif\n }\n\nprotected:\n\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n graph_task* try_put_task( const input_type &v ) override {\n spin_mutex::scoped_lock l( my_mutex );\n return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override {\n spin_mutex::scoped_lock l( my_mutex );\n return try_put_task_impl(v, metainfo);\n }\n#endif\n\n graph_task * try_put_task_impl(const input_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) {\n my_buffer = v;\n my_buffer_is_valid = true;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n // Since the new item is pushed to the buffer - reserving the waiters\n for (auto msg_waiter : metainfo.waiters()) {\n msg_waiter->reserve(1);\n }\n\n // Since the item is taken out from the buffer - releasing the stored waiters\n for (auto msg_waiter : my_buffered_metainfo.waiters()) {\n msg_waiter->release(1);\n }\n\n my_buffered_metainfo = metainfo;\n#endif\n graph_task* rtask = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(my_buffered_metainfo) );\n if (!rtask) rtask = SUCCESSFULLY_ENQUEUED;\n return rtask;\n }\n\n graph& graph_reference() const override {\n return my_graph;\n }\n\n //! Breaks an infinite loop between the node reservation and register_successor call\n struct register_predecessor_task : public graph_task {\n register_predecessor_task(\n graph& g, d1::small_object_allocator& allocator, predecessor_type& owner, successor_type& succ)\n : graph_task(g, allocator), o(owner), s(succ) {};\n\n d1::task* execute(d1::execution_data& ed) override {\n // TODO revamp: investigate why qualification is needed for register_successor() call\n using tbb::detail::d2::register_predecessor;\n using tbb::detail::d2::register_successor;\n if ( !register_predecessor(s, o) ) {\n register_successor(o, s);\n }\n finalize(ed);\n return nullptr;\n }\n\n d1::task* cancel(d1::execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n\n predecessor_type& o;\n successor_type& s;\n };\n\n spin_mutex my_mutex;\n broadcast_cache< input_type, null_rw_mutex > my_successors;\n input_type my_buffer;\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n message_metainfo my_buffered_metainfo;\n#endif\n bool my_buffer_is_valid;\n\n void reset_node( reset_flags f) override {\n my_buffer_is_valid = false;\n if (f&rf_clear_edges) {\n my_successors.clear();\n }\n }\n}; // overwrite_node\n\ntemplate< typename T >\nclass write_once_node : public overwrite_node {\npublic:\n typedef T input_type;\n typedef T output_type;\n typedef overwrite_node base_type;\n typedef typename receiver::predecessor_type predecessor_type;\n typedef typename sender::successor_type successor_type;\n\n //! Constructor\n __TBB_NOINLINE_SYM explicit write_once_node(graph& g) : base_type(g) {\n fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n template \n write_once_node(const node_set& nodes) : write_once_node(nodes.graph_reference()) {\n make_edges_in_order(nodes, *this);\n }\n#endif\n\n //! Copy constructor: call base class copy constructor\n __TBB_NOINLINE_SYM write_once_node( const write_once_node& src ) : base_type(src) {\n fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph),\n static_cast *>(this),\n static_cast *>(this) );\n }\n\nprotected:\n template< typename R, typename B > friend class run_and_put_task;\n template friend class broadcast_cache;\n template friend class round_robin_cache;\n graph_task *try_put_task( const T &v ) override {\n spin_mutex::scoped_lock l( this->my_mutex );\n return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{}));\n }\n\n#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT\n graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override {\n spin_mutex::scoped_lock l( this->my_mutex );\n return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v, metainfo);\n }\n#endif\n}; // write_once_node\n\ninline void set_name(const graph& g, const char *name) {\n fgt_graph_desc(&g, name);\n}\n\ntemplate \ninline void set_name(const input_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const function_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const continue_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const broadcast_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const buffer_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const queue_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const sequencer_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const priority_queue_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const limiter_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const join_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const indexer_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const overwrite_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate \ninline void set_name(const write_once_node& node, const char *name) {\n fgt_node_desc(&node, name);\n}\n\ntemplate\ninline void set_name(const multifunction_node& node, const char *name) {\n fgt_multioutput_node_desc(&node, name);\n}\n\ntemplate\ninline void set_name(const split_node& node, const char *name) {\n fgt_multioutput_node_desc(&node, name);\n}\n\ntemplate< typename InputTuple, typename OutputTuple >\ninline void set_name(const composite_node& node, const char *name) {\n fgt_multiinput_multioutput_node_desc(&node, name);\n}\n\ntemplate\ninline void set_name(const async_node& node, const char *name)\n{\n fgt_multioutput_node_desc(&node, name);\n}\n} // d2\n} // detail\n} // tbb\n\n\n// Include deduction guides for node classes\n#include \"detail/_flow_graph_nodes_deduction.h\"\n\nnamespace tbb {\nnamespace flow {\ninline namespace v1 {\n using detail::d2::receiver;\n using detail::d2::sender;\n\n using detail::d2::serial;\n using detail::d2::unlimited;\n\n using detail::d2::reset_flags;\n using detail::d2::rf_reset_protocol;\n using detail::d2::rf_reset_bodies;\n using detail::d2::rf_clear_edges;\n\n using detail::d2::graph;\n using detail::d2::graph_node;\n using detail::d2::continue_msg;\n\n using detail::d2::input_node;\n using detail::d2::function_node;\n using detail::d2::multifunction_node;\n using detail::d2::split_node;\n using detail::d2::output_port;\n using detail::d2::indexer_node;\n using detail::d2::tagged_msg;\n using detail::d2::cast_to;\n using detail::d2::is_a;\n using detail::d2::continue_node;\n using detail::d2::overwrite_node;\n using detail::d2::write_once_node;\n using detail::d2::broadcast_node;\n using detail::d2::buffer_node;\n using detail::d2::queue_node;\n using detail::d2::sequencer_node;\n using detail::d2::priority_queue_node;\n using detail::d2::limiter_node;\n using namespace detail::d2::graph_policy_namespace;\n using detail::d2::join_node;\n using detail::d2::input_port;\n using detail::d2::copy_body;\n using detail::d2::make_edge;\n using detail::d2::remove_edge;\n using detail::d2::tag_value;\n using detail::d2::composite_node;\n using detail::d2::async_node;\n using detail::d2::node_priority_t;\n using detail::d2::no_priority;\n\n#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET\n using detail::d2::follows;\n using detail::d2::precedes;\n using detail::d2::make_node_set;\n using detail::d2::make_edges;\n#endif\n\n} // v1\n} // flow\n\n using detail::d1::flow_control;\n\nnamespace profiling {\n using detail::d2::set_name;\n} // profiling\n\n} // tbb\n\n\n#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ )\n // We don't do pragma pop here, since it still gives warning on the USER side\n #undef __TBB_NOINLINE_SYM\n#endif\n\n#endif // __TBB_flow_graph_H\n" }, { "path": "src/tbb/include/oneapi/tbb/flow_graph_abstractions.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_flow_graph_abstractions_H\n#define __TBB_flow_graph_abstractions_H\n\nnamespace tbb {\nnamespace detail {\nnamespace d2 {\n\n//! Pure virtual template classes that define interfaces for async communication\nclass graph_proxy {\npublic:\n //! Inform a graph that messages may come from outside, to prevent premature graph completion\n virtual void reserve_wait() = 0;\n\n //! Inform a graph that a previous call to reserve_wait is no longer in effect\n virtual void release_wait() = 0;\n\n virtual ~graph_proxy() {}\n};\n\ntemplate \nclass receiver_gateway : public graph_proxy {\npublic:\n //! Type of inputing data into FG.\n typedef Input input_type;\n\n //! Submit signal from an asynchronous activity to FG.\n virtual bool try_put(const input_type&) = 0;\n};\n\n} // d2\n\n\n} // detail\n} // tbb\n#endif\n" }, { "path": "src/tbb/include/oneapi/tbb/global_control.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_global_control_H\n#define __TBB_global_control_H\n\n#include \"detail/_config.h\"\n\n#include \"detail/_assert.h\"\n#include \"detail/_attach.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_template_helpers.h\"\n\n#include \n#include // std::nothrow_t\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass global_control;\nclass task_scheduler_handle;\n}\n\nnamespace r1 {\nTBB_EXPORT void __TBB_EXPORTED_FUNC create(d1::global_control&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::global_control&);\nTBB_EXPORT std::size_t __TBB_EXPORTED_FUNC global_control_active_value(int);\nstruct global_control_impl;\nstruct control_storage_comparator;\nvoid release_impl(d1::task_scheduler_handle& handle);\nbool finalize_impl(d1::task_scheduler_handle& handle);\nTBB_EXPORT void __TBB_EXPORTED_FUNC get(d1::task_scheduler_handle&);\nTBB_EXPORT bool __TBB_EXPORTED_FUNC finalize(d1::task_scheduler_handle&, std::intptr_t mode);\n}\n\nnamespace d1 {\n\nclass global_control {\npublic:\n enum parameter {\n max_allowed_parallelism,\n thread_stack_size,\n terminate_on_exception,\n scheduler_handle, // not a public parameter\n parameter_max // insert new parameters above this point\n };\n\n global_control(parameter p, std::size_t value) :\n my_value(value), my_reserved(), my_param(p) {\n suppress_unused_warning(my_reserved);\n __TBB_ASSERT(my_param < parameter_max, \"Invalid parameter\");\n#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\n // For Windows 8 Store* apps it's impossible to set stack size\n if (p==thread_stack_size)\n return;\n#elif __TBB_x86_64 && (_WIN32 || _WIN64)\n if (p==thread_stack_size)\n __TBB_ASSERT_RELEASE((unsigned)value == value, \"Stack size is limited to unsigned int range\");\n#endif\n if (my_param==max_allowed_parallelism)\n __TBB_ASSERT_RELEASE(my_value>0, \"max_allowed_parallelism cannot be 0.\");\n r1::create(*this);\n }\n\n ~global_control() {\n __TBB_ASSERT(my_param < parameter_max, \"Invalid parameter\");\n#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\n // For Windows 8 Store* apps it's impossible to set stack size\n if (my_param==thread_stack_size)\n return;\n#endif\n r1::destroy(*this);\n }\n\n static std::size_t active_value(parameter p) {\n __TBB_ASSERT(p < parameter_max, \"Invalid parameter\");\n return r1::global_control_active_value((int)p);\n }\n\nprivate:\n std::size_t my_value;\n std::intptr_t my_reserved; // TODO: substitution of global_control* not to break backward compatibility\n parameter my_param;\n\n friend struct r1::global_control_impl;\n friend struct r1::control_storage_comparator;\n};\n\n//! Finalization options.\n//! Outside of the class to avoid extensive friendship.\nstatic constexpr std::intptr_t release_nothrowing = 0;\nstatic constexpr std::intptr_t finalize_nothrowing = 1;\nstatic constexpr std::intptr_t finalize_throwing = 2;\n\n//! User side wrapper for a task scheduler lifetime control object\nclass task_scheduler_handle {\npublic:\n //! Creates an empty task_scheduler_handle\n task_scheduler_handle() = default;\n\n //! Creates an attached instance of task_scheduler_handle\n task_scheduler_handle(attach) {\n r1::get(*this);\n }\n\n //! Release a reference if any\n ~task_scheduler_handle() {\n release();\n }\n\n //! No copy\n task_scheduler_handle(const task_scheduler_handle& other) = delete;\n task_scheduler_handle& operator=(const task_scheduler_handle& other) = delete;\n\n //! Move only\n task_scheduler_handle(task_scheduler_handle&& other) noexcept {\n std::swap(m_ctl, other.m_ctl);\n }\n task_scheduler_handle& operator=(task_scheduler_handle&& other) noexcept {\n std::swap(m_ctl, other.m_ctl);\n return *this;\n };\n\n //! Checks if the task_scheduler_handle is empty\n explicit operator bool() const noexcept {\n return m_ctl != nullptr;\n }\n\n //! Release the reference and deactivate handle\n void release() {\n if (m_ctl != nullptr) {\n r1::finalize(*this, release_nothrowing);\n m_ctl = nullptr;\n }\n }\n\nprivate:\n friend void r1::release_impl(task_scheduler_handle& handle);\n friend bool r1::finalize_impl(task_scheduler_handle& handle);\n friend void __TBB_EXPORTED_FUNC r1::get(task_scheduler_handle&);\n\n friend void finalize(task_scheduler_handle&);\n friend bool finalize(task_scheduler_handle&, const std::nothrow_t&) noexcept;\n\n global_control* m_ctl{nullptr};\n};\n\n#if TBB_USE_EXCEPTIONS\n//! Waits for worker threads termination. Throws exception on error.\ninline void finalize(task_scheduler_handle& handle) {\n try_call([&] {\n if (handle.m_ctl != nullptr) {\n bool finalized = r1::finalize(handle, finalize_throwing);\n __TBB_ASSERT_EX(finalized, \"r1::finalize did not respect finalize_throwing ?\");\n \n }\n }).on_completion([&] {\n __TBB_ASSERT(!handle, \"The handle should be empty after finalize\");\n });\n}\n#endif\n//! Waits for worker threads termination. Returns false on error.\ninline bool finalize(task_scheduler_handle& handle, const std::nothrow_t&) noexcept {\n bool finalized = true;\n if (handle.m_ctl != nullptr) {\n finalized = r1::finalize(handle, finalize_nothrowing);\n }\n __TBB_ASSERT(!handle, \"The handle should be empty after finalize\");\n return finalized;\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::global_control;\nusing detail::d1::attach;\nusing detail::d1::finalize;\nusing detail::d1::task_scheduler_handle;\nusing detail::r1::unsafe_wait;\n} // namespace v1\n\n} // namespace tbb\n\n#endif // __TBB_global_control_H\n" }, { "path": "src/tbb/include/oneapi/tbb/info.h", "content": "/*\n Copyright (c) 2019-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_info_H\n#define __TBB_info_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n\n#if __TBB_ARENA_BINDING\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1{\n\nusing numa_node_id = int;\nusing core_type_id = int;\n\n// TODO: consider version approach to resolve backward compatibility potential issues.\nstruct constraints {\n#if !__TBB_CPP20_PRESENT\n constraints(numa_node_id id = -1, int maximal_concurrency = -1)\n : numa_id(id)\n , max_concurrency(maximal_concurrency)\n {}\n#endif /*!__TBB_CPP20_PRESENT*/\n\n constraints& set_numa_id(numa_node_id id) {\n numa_id = id;\n return *this;\n }\n constraints& set_max_concurrency(int maximal_concurrency) {\n max_concurrency = maximal_concurrency;\n return *this;\n }\n constraints& set_core_type(core_type_id id) {\n core_type = id;\n return *this;\n }\n constraints& set_max_threads_per_core(int threads_number) {\n max_threads_per_core = threads_number;\n return *this;\n }\n\n numa_node_id numa_id = -1;\n int max_concurrency = -1;\n core_type_id core_type = -1;\n int max_threads_per_core = -1;\n};\n\n} // namespace d1\n\nnamespace r1 {\nTBB_EXPORT unsigned __TBB_EXPORTED_FUNC numa_node_count();\nTBB_EXPORT void __TBB_EXPORTED_FUNC fill_numa_indices(int* index_array);\nTBB_EXPORT int __TBB_EXPORTED_FUNC numa_default_concurrency(int numa_id);\n\n// Reserved fields are required to save binary backward compatibility in case of future changes.\n// They must be defined to 0 at this moment.\nTBB_EXPORT unsigned __TBB_EXPORTED_FUNC core_type_count(intptr_t reserved = 0);\nTBB_EXPORT void __TBB_EXPORTED_FUNC fill_core_type_indices(int* index_array, intptr_t reserved = 0);\n\nTBB_EXPORT int __TBB_EXPORTED_FUNC constraints_default_concurrency(const d1::constraints& c, intptr_t reserved = 0);\nTBB_EXPORT int __TBB_EXPORTED_FUNC constraints_threads_per_core(const d1::constraints& c, intptr_t reserved = 0);\n} // namespace r1\n\nnamespace d1 {\n\ninline std::vector numa_nodes() {\n std::vector node_indices(r1::numa_node_count());\n r1::fill_numa_indices(node_indices.data());\n return node_indices;\n}\n\ninline int default_concurrency(numa_node_id id = -1) {\n return r1::numa_default_concurrency(id);\n}\n\ninline std::vector core_types() {\n std::vector core_type_indexes(r1::core_type_count());\n r1::fill_core_type_indices(core_type_indexes.data());\n return core_type_indexes;\n}\n\ninline int default_concurrency(constraints c) {\n if (c.max_concurrency > 0) { return c.max_concurrency; }\n return r1::constraints_default_concurrency(c);\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::numa_node_id;\nusing detail::d1::core_type_id;\n\nnamespace info {\nusing detail::d1::numa_nodes;\nusing detail::d1::core_types;\n\nusing detail::d1::default_concurrency;\n} // namespace info\n} // namespace v1\n\n} // namespace tbb\n\n#endif /*__TBB_ARENA_BINDING*/\n\n#endif /*__TBB_info_H*/\n" }, { "path": "src/tbb/include/oneapi/tbb/memory_pool.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_memory_pool_H\n#define __TBB_memory_pool_H\n\n#if !TBB_PREVIEW_MEMORY_POOL\n#error Set TBB_PREVIEW_MEMORY_POOL to include memory_pool.h\n#endif\n/** @file */\n\n#include \"scalable_allocator.h\"\n\n#include // std::bad_alloc\n#include // std::runtime_error, std::invalid_argument\n#include // std::forward\n\n\n#if __TBB_EXTRA_DEBUG\n#define __TBBMALLOC_ASSERT ASSERT\n#else\n#define __TBBMALLOC_ASSERT(a,b) ((void)0)\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! Base of thread-safe pool allocator for variable-size requests\nclass pool_base : no_copy {\n // Pool interface is separate from standard allocator classes because it has\n // to maintain internal state, no copy or assignment. Move and swap are possible.\npublic:\n //! Reset pool to reuse its memory (free all objects at once)\n void recycle() { rml::pool_reset(my_pool); }\n\n //! The \"malloc\" analogue to allocate block of memory of size bytes\n void *malloc(size_t size) { return rml::pool_malloc(my_pool, size); }\n\n //! The \"free\" analogue to discard a previously allocated piece of memory.\n void free(void* ptr) { rml::pool_free(my_pool, ptr); }\n\n //! The \"realloc\" analogue complementing pool_malloc.\n // Enables some low-level optimization possibilities\n void *realloc(void* ptr, size_t size) {\n return rml::pool_realloc(my_pool, ptr, size);\n }\n\nprotected:\n //! destroy pool - must be called in a child class\n void destroy() { rml::pool_destroy(my_pool); }\n\n rml::MemoryPool *my_pool;\n};\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Workaround for erroneous \"unreferenced parameter\" warning in method destroy.\n // #pragma warning (push)\n // #pragma warning (disable: 4100)\n#endif\n\n//! Meets \"allocator\" requirements of ISO C++ Standard, Section 20.1.5\n/** @ingroup memory_allocation */\ntemplate\nclass memory_pool_allocator {\nprotected:\n typedef P pool_type;\n pool_type *my_pool;\n template\n friend class memory_pool_allocator;\n template\n friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b);\n template\n friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b);\npublic:\n typedef T value_type;\n typedef value_type* pointer;\n typedef const value_type* const_pointer;\n typedef value_type& reference;\n typedef const value_type& const_reference;\n typedef size_t size_type;\n typedef ptrdiff_t difference_type;\n template struct rebind {\n typedef memory_pool_allocator other;\n };\n\n explicit memory_pool_allocator(pool_type &pool) noexcept : my_pool(&pool) {}\n memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {}\n template\n memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {}\n\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n\n //! Allocate space for n objects.\n pointer allocate( size_type n, const void* /*hint*/ = nullptr) {\n pointer p = static_cast( my_pool->malloc( n*sizeof(value_type) ) );\n if (!p)\n throw_exception(std::bad_alloc());\n return p;\n }\n //! Free previously allocated block of memory.\n void deallocate( pointer p, size_type ) {\n my_pool->free(p);\n }\n //! Largest value for which method allocate might succeed.\n size_type max_size() const noexcept {\n size_type max = static_cast(-1) / sizeof (value_type);\n return (max > 0 ? max : 1);\n }\n //! Copy-construct value at location pointed to by p.\n\n template\n void construct(U *p, Args&&... args)\n { ::new((void *)p) U(std::forward(args)...); }\n\n //! Destroy value at location pointed to by p.\n void destroy( pointer p ) { p->~value_type(); }\n\n};\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning (pop)\n#endif // warning 4100 is back\n\n//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1\n/** @ingroup memory_allocation */\ntemplate\nclass memory_pool_allocator {\npublic:\n typedef P pool_type;\n typedef void* pointer;\n typedef const void* const_pointer;\n typedef void value_type;\n template struct rebind {\n typedef memory_pool_allocator other;\n };\n\n explicit memory_pool_allocator( pool_type &pool) noexcept : my_pool(&pool) {}\n memory_pool_allocator( const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {}\n template\n memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {}\n\nprotected:\n pool_type *my_pool;\n template\n friend class memory_pool_allocator;\n template\n friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b);\n template\n friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b);\n};\n\ntemplate\ninline bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool==b.my_pool;}\n\ntemplate\ninline bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool!=b.my_pool;}\n\n//! Thread-safe growable pool allocator for variable-size requests\ntemplate \nclass memory_pool : public pool_base {\n Alloc my_alloc; // TODO: base-class optimization\n static void *allocate_request(intptr_t pool_id, size_t & bytes);\n static int deallocate_request(intptr_t pool_id, void*, size_t raw_bytes);\n\npublic:\n //! construct pool with underlying allocator\n explicit memory_pool(const Alloc &src = Alloc());\n\n //! destroy pool\n ~memory_pool() { destroy(); } // call the callbacks first and destroy my_alloc latter\n};\n\nclass fixed_pool : public pool_base {\n void *my_buffer;\n size_t my_size;\n inline static void *allocate_request(intptr_t pool_id, size_t & bytes);\n\npublic:\n //! construct pool with underlying allocator\n inline fixed_pool(void *buf, size_t size);\n //! destroy pool\n ~fixed_pool() { destroy(); }\n};\n\n//////////////// Implementation ///////////////\n\ntemplate \nmemory_pool::memory_pool(const Alloc &src) : my_alloc(src) {\n rml::MemPoolPolicy args(allocate_request, deallocate_request,\n sizeof(typename Alloc::value_type));\n rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool);\n if (res!=rml::POOL_OK)\n throw_exception(std::runtime_error(\"Can't create pool\"));\n}\ntemplate \nvoid *memory_pool::allocate_request(intptr_t pool_id, size_t & bytes) {\n memory_pool &self = *reinterpret_cast*>(pool_id);\n const size_t unit_size = sizeof(typename Alloc::value_type);\n __TBBMALLOC_ASSERT( 0 == bytes%unit_size, nullptr);\n void *ptr;\n#if TBB_USE_EXCEPTIONS\n try {\n#endif\n ptr = self.my_alloc.allocate( bytes/unit_size );\n#if TBB_USE_EXCEPTIONS\n } catch(...) {\n return nullptr;\n }\n#endif\n return ptr;\n}\n#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED\n // Workaround for erroneous \"unreachable code\" warning in the template below.\n // Specific for VC++ 17-18 compiler\n // #pragma warning (push)\n // #pragma warning (disable: 4702)\n#endif\ntemplate \nint memory_pool::deallocate_request(intptr_t pool_id, void* raw_ptr, size_t raw_bytes) {\n memory_pool &self = *reinterpret_cast*>(pool_id);\n const size_t unit_size = sizeof(typename Alloc::value_type);\n __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, nullptr);\n self.my_alloc.deallocate( static_cast(raw_ptr), raw_bytes/unit_size );\n return 0;\n}\n#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED\n // #pragma warning (pop)\n#endif\ninline fixed_pool::fixed_pool(void *buf, size_t size) : my_buffer(buf), my_size(size) {\n if (!buf || !size)\n // TODO: improve support for mode with exceptions disabled\n throw_exception(std::invalid_argument(\"Zero in parameter is invalid\"));\n rml::MemPoolPolicy args(allocate_request, nullptr, size, /*fixedPool=*/true);\n rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool);\n if (res!=rml::POOL_OK)\n throw_exception(std::runtime_error(\"Can't create pool\"));\n}\ninline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) {\n fixed_pool &self = *reinterpret_cast(pool_id);\n __TBBMALLOC_ASSERT(0 != self.my_size, \"The buffer must not be used twice.\");\n bytes = self.my_size;\n self.my_size = 0; // remember that buffer has been used\n return self.my_buffer;\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::memory_pool_allocator;\nusing detail::d1::memory_pool;\nusing detail::d1::fixed_pool;\n} // inline namepspace v1\n} // namespace tbb\n\n#undef __TBBMALLOC_ASSERT\n#endif// __TBB_memory_pool_H\n" }, { "path": "src/tbb/include/oneapi/tbb/mutex.h", "content": "/*\n Copyright (c) 2021-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_mutex_H\n#define __TBB_mutex_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_scoped_lock.h\"\n#include \"detail/_waitable_atomic.h\"\n#include \"detail/_mutex_common.h\"\n#include \"profiling.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\nclass mutex {\npublic:\n //! Constructors\n mutex() {\n create_itt_sync(this, \"tbb::mutex\", \"\");\n };\n\n //! Destructor\n ~mutex() = default;\n\n //! No Copy\n mutex(const mutex&) = delete;\n mutex& operator=(const mutex&) = delete;\n\n using scoped_lock = unique_scoped_lock;\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = false;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\n\n //! Acquire lock\n /** Spin if the lock is taken */\n void lock() {\n call_itt_notify(prepare, this);\n while (!try_lock()) {\n my_flag.wait(true, /* context = */ 0, std::memory_order_relaxed);\n }\n }\n\n //! Try acquiring lock (non-blocking)\n /** Return true if lock acquired; false otherwise. */\n bool try_lock() {\n bool result = !my_flag.load(std::memory_order_relaxed) && !my_flag.exchange(true);\n if (result) {\n call_itt_notify(acquired, this);\n }\n return result;\n }\n\n //! Release lock\n void unlock() {\n call_itt_notify(releasing, this);\n // We need Write Read memory barrier before notify that reads the waiter list.\n // In C++ only full fence covers this type of barrier.\n my_flag.exchange(false);\n my_flag.notify_one_relaxed();\n }\n\nprivate:\n waitable_atomic my_flag{0};\n}; // class mutex\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::mutex;\n} // namespace v1\n\n} // namespace tbb\n\n#endif // __TBB_mutex_H\n" }, { "path": "src/tbb/include/oneapi/tbb/null_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_null_mutex_H\n#define __TBB_null_mutex_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_mutex_common.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! A mutex which does nothing\n/** A null_mutex does no operation and simulates success.\n @ingroup synchronization */\nclass null_mutex {\npublic:\n //! Constructors\n constexpr null_mutex() noexcept = default;\n\n //! Destructor\n ~null_mutex() = default;\n\n //! No Copy\n null_mutex(const null_mutex&) = delete;\n null_mutex& operator=(const null_mutex&) = delete;\n\n //! Represents acquisition of a mutex.\n class scoped_lock {\n public:\n //! Constructors\n constexpr scoped_lock() noexcept = default;\n scoped_lock(null_mutex&) {}\n\n //! Destructor\n ~scoped_lock() = default;\n\n //! No Copy\n scoped_lock(const scoped_lock&) = delete;\n scoped_lock& operator=(const scoped_lock&) = delete;\n\n void acquire(null_mutex&) {}\n bool try_acquire(null_mutex&) { return true; }\n void release() {}\n };\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = false;\n static constexpr bool is_recursive_mutex = true;\n static constexpr bool is_fair_mutex = true;\n\n void lock() {}\n bool try_lock() { return true; }\n void unlock() {}\n}; // class null_mutex\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::null_mutex;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_null_mutex_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/null_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_null_rw_mutex_H\n#define __TBB_null_rw_mutex_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_mutex_common.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! A rw mutex which does nothing\n/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation.\n @ingroup synchronization */\nclass null_rw_mutex {\npublic:\n //! Constructors\n constexpr null_rw_mutex() noexcept = default;\n\n //! Destructor\n ~null_rw_mutex() = default;\n\n //! No Copy\n null_rw_mutex(const null_rw_mutex&) = delete;\n null_rw_mutex& operator=(const null_rw_mutex&) = delete;\n\n //! Represents acquisition of a mutex.\n class scoped_lock {\n public:\n //! Constructors\n constexpr scoped_lock() noexcept = default;\n scoped_lock(null_rw_mutex&, bool = true) {}\n\n //! Destructor\n ~scoped_lock() = default;\n\n //! No Copy\n scoped_lock(const scoped_lock&) = delete;\n scoped_lock& operator=(const scoped_lock&) = delete;\n\n void acquire(null_rw_mutex&, bool = true) {}\n bool try_acquire(null_rw_mutex&, bool = true) { return true; }\n void release() {}\n bool upgrade_to_writer() { return true; }\n bool downgrade_to_reader() { return true; }\n\n bool is_writer() const { return true; }\n };\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = true;\n static constexpr bool is_recursive_mutex = true;\n static constexpr bool is_fair_mutex = true;\n\n void lock() {}\n bool try_lock() { return true; }\n void unlock() {}\n void lock_shared() {}\n bool try_lock_shared() { return true; }\n void unlock_shared() {}\n}; // class null_rw_mutex\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::null_rw_mutex;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_null_rw_mutex_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_for.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_for_H\n#define __TBB_parallel_for_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_task.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"profiling.h\"\n\n#include \"partitioner.h\"\n#include \"blocked_range.h\"\n#include \"task_group.h\"\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n#if __TBB_CPP20_CONCEPTS_PRESENT\ninline namespace d0 {\n\ntemplate \nconcept parallel_for_body = std::copy_constructible && std::invocable&, Range&>;\n\ntemplate \nconcept parallel_for_index = std::constructible_from &&\n std::copyable &&\n requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) {\n { lhs < rhs } -> adaptive_same_as;\n { lhs - rhs } -> std::convertible_to;\n { lhs + (rhs - lhs) } -> std::convertible_to;\n };\n\ntemplate \nconcept parallel_for_function = std::invocable&, Index>;\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d1 {\n\n//! Task type used in parallel_for\n/** @ingroup algorithms */\ntemplate\nstruct start_for : public task {\n Range my_range;\n const Body my_body;\n node* my_parent;\n\n typename Partitioner::task_partition_type my_partition;\n small_object_allocator my_allocator;\n\n task* execute(execution_data&) override;\n task* cancel(execution_data&) override;\n void finalize(const execution_data&);\n\n //! Constructor for root task.\n start_for( const Range& range, const Body& body, Partitioner& partitioner, small_object_allocator& alloc ) :\n my_range(range),\n my_body(body),\n my_parent(nullptr),\n my_partition(partitioner),\n my_allocator(alloc) {}\n //! Splitting constructor used to generate children.\n /** parent_ becomes left child. Newly constructed object is right child. */\n start_for( start_for& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) :\n my_range(parent_.my_range, get_range_split_object(split_obj)),\n my_body(parent_.my_body),\n my_parent(nullptr),\n my_partition(parent_.my_partition, split_obj),\n my_allocator(alloc) {}\n //! Construct right child from the given range as response to the demand.\n /** parent_ remains left child. Newly constructed object is right child. */\n start_for( start_for& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) :\n my_range(r),\n my_body(parent_.my_body),\n my_parent(nullptr),\n my_partition(parent_.my_partition, split()),\n my_allocator(alloc)\n {\n my_partition.align_depth( d );\n }\n static void run(const Range& range, const Body& body, Partitioner& partitioner) {\n task_group_context context(PARALLEL_FOR);\n run(range, body, partitioner, context);\n }\n\n static void run(const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context) {\n if ( !range.empty() ) {\n small_object_allocator alloc{};\n start_for& for_task = *alloc.new_object(range, body, partitioner, alloc);\n\n // defer creation of the wait node until task allocation succeeds\n wait_node wn;\n for_task.my_parent = &wn;\n execute_and_wait(for_task, context, wn.m_wait, context);\n }\n }\n //! Run body for range, serves as callback for partitioner\n void run_body( Range &r ) {\n tbb::detail::invoke(my_body, r);\n }\n\n //! spawn right task, serves as callback for partitioner\n void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) {\n offer_work_impl(ed, *this, split_obj);\n }\n\n //! spawn right task, serves as callback for partitioner\n void offer_work(const Range& r, depth_t d, execution_data& ed) {\n offer_work_impl(ed, *this, r, d);\n }\n\nprivate:\n template \n void offer_work_impl(execution_data& ed, Args&&... constructor_args) {\n // New right child\n small_object_allocator alloc{};\n start_for& right_child = *alloc.new_object(ed, std::forward(constructor_args)..., alloc);\n\n // New root node as a continuation and ref count. Left and right child attach to the new parent.\n right_child.my_parent = my_parent = alloc.new_object(ed, my_parent, 2, alloc);\n // Spawn the right sibling\n right_child.spawn_self(ed);\n }\n\n void spawn_self(execution_data& ed) {\n my_partition.spawn_task(*this, *context(ed));\n }\n};\n\n//! fold the tree and deallocate the task\ntemplate\nvoid start_for::finalize(const execution_data& ed) {\n // Get the current parent and allocator an object destruction\n node* parent = my_parent;\n auto allocator = my_allocator;\n // Task execution finished - destroy it\n this->~start_for();\n // Unwind the tree decrementing the parent`s reference count\n\n fold_tree(parent, ed);\n allocator.deallocate(this, ed);\n\n}\n\n//! execute task for parallel_for\ntemplate\ntask* start_for::execute(execution_data& ed) {\n if (!is_same_affinity(ed)) {\n my_partition.note_affinity(execution_slot(ed));\n }\n my_partition.check_being_stolen(*this, ed);\n my_partition.execute(*this, my_range, ed);\n finalize(ed);\n return nullptr;\n}\n\n//! cancel task for parallel_for\ntemplate\ntask* start_for::cancel(execution_data& ed) {\n finalize(ed);\n return nullptr;\n}\n\n//! Calls the function with values from range [begin, end) with a step provided\ntemplate\nclass parallel_for_body_wrapper : detail::no_assign {\n const Function &my_func;\n const Index my_begin;\n const Index my_step;\npublic:\n parallel_for_body_wrapper( const Function& _func, Index& _begin, Index& _step )\n : my_func(_func), my_begin(_begin), my_step(_step) {}\n\n void operator()( const blocked_range& r ) const {\n // A set of local variables to help the compiler with vectorization of the following loop.\n Index b = r.begin();\n Index e = r.end();\n Index ms = my_step;\n Index k = my_begin + b*ms;\n\n#if __INTEL_COMPILER\n#pragma ivdep\n#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE\n#pragma vector always assert\n#endif\n#endif\n for ( Index i = b; i < e; ++i, k += ms ) {\n tbb::detail::invoke(my_func, k);\n }\n }\n};\n\n// Requirements on Range concept are documented in blocked_range.h\n\n/** \\page parallel_for_body_req Requirements on parallel_for body\n Class \\c Body implementing the concept of parallel_for body must define:\n - \\code Body::Body( const Body& ); \\endcode Copy constructor\n - \\code Body::~Body(); \\endcode Destructor\n - \\code void Body::operator()( Range& r ) const; \\endcode Function call operator applying the body to range \\c r.\n**/\n\n/** \\name parallel_for\n See also requirements on \\ref range_req \"Range\" and \\ref parallel_for_body_req \"parallel_for Body\". **/\n//@{\n\n//! Parallel iteration over range with default partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body ) {\n start_for::run(range,body,__TBB_DEFAULT_PARTITIONER());\n}\n\n//! Parallel iteration over range with simple partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) {\n start_for::run(range,body,partitioner);\n}\n\n//! Parallel iteration over range with auto_partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) {\n start_for::run(range,body,partitioner);\n}\n\n//! Parallel iteration over range with static_partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) {\n start_for::run(range,body,partitioner);\n}\n\n//! Parallel iteration over range with affinity_partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) {\n start_for::run(range,body,partitioner);\n}\n\n//! Parallel iteration over range with default partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, task_group_context& context ) {\n start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context);\n}\n\n//! Parallel iteration over range with simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) {\n start_for::run(range, body, partitioner, context);\n}\n\n//! Parallel iteration over range with auto_partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) {\n start_for::run(range, body, partitioner, context);\n}\n\n//! Parallel iteration over range with static_partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) {\n start_for::run(range, body, partitioner, context);\n}\n\n//! Parallel iteration over range with affinity_partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_for_body)\nvoid parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) {\n start_for::run(range,body,partitioner, context);\n}\n\n//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner\ntemplate \nvoid parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) {\n if (step <= 0 )\n throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument\n else if (first < last) {\n // Above \"else\" avoids \"potential divide by zero\" warning on some platforms\n Index end = Index(last - first - 1ul) / step + Index(1);\n blocked_range range(static_cast(0), end);\n parallel_for_body_wrapper body(f, first, step);\n parallel_for(range, body, partitioner);\n }\n}\n\n//! Parallel iteration over a range of integers with a step provided and default partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f) {\n parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER());\n}\n//! Parallel iteration over a range of integers with a step provided and simple partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) {\n parallel_for_impl(first, last, step, f, partitioner);\n}\n//! Parallel iteration over a range of integers with a step provided and auto partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) {\n parallel_for_impl(first, last, step, f, partitioner);\n}\n//! Parallel iteration over a range of integers with a step provided and static partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) {\n parallel_for_impl(first, last, step, f, partitioner);\n}\n//! Parallel iteration over a range of integers with a step provided and affinity partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) {\n parallel_for_impl(first, last, step, f, partitioner);\n}\n\n//! Parallel iteration over a range of integers with a default step value and default partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f) {\n parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER());\n}\n//! Parallel iteration over a range of integers with a default step value and simple partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner);\n}\n//! Parallel iteration over a range of integers with a default step value and auto partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner);\n}\n//! Parallel iteration over a range of integers with a default step value and static partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner);\n}\n//! Parallel iteration over a range of integers with a default step value and affinity partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner);\n}\n\n//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner\ntemplate \nvoid parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, task_group_context &context) {\n if (step <= 0 )\n throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument\n else if (first < last) {\n // Above \"else\" avoids \"potential divide by zero\" warning on some platforms\n Index end = (last - first - Index(1)) / step + Index(1);\n blocked_range range(static_cast(0), end);\n parallel_for_body_wrapper body(f, first, step);\n parallel_for(range, body, partitioner, context);\n }\n}\n\n//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, task_group_context &context) {\n parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER(), context);\n}\n//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, step, f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, step, f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, step, f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, step, f, partitioner, context);\n}\n\n//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, task_group_context &context) {\n parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER(), context);\n}\n//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner, context);\n}\n//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner\ntemplate \n __TBB_requires(parallel_for_index && parallel_for_function)\nvoid parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, task_group_context &context) {\n parallel_for_impl(first, last, static_cast(1), f, partitioner, context);\n}\n// @}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::parallel_for;\n// Split types\nusing detail::split;\nusing detail::proportional_split;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_parallel_for_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_for_each.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_for_each_H\n#define __TBB_parallel_for_each_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_task.h\"\n#include \"detail/_aligned_space.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"detail/_utils.h\"\n\n#include \"parallel_for.h\"\n#include \"task_group.h\" // task_group_context\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n#if __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d1 {\ntemplate \nclass feeder;\n\n} // namespace d1\ninline namespace d0 {\n\ntemplate \nconcept parallel_for_each_body = std::invocable&, ItemType&&> ||\n std::invocable&, ItemType&&, tbb::detail::d1::feeder&>;\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d2 {\ntemplate class feeder_impl;\n} // namespace d2\n\nnamespace d1 {\n//! Class the user supplied algorithm body uses to add new tasks\ntemplate\nclass feeder {\n feeder() {}\n feeder(const feeder&) = delete;\n void operator=( const feeder&) = delete;\n\n virtual ~feeder () {}\n virtual void internal_add_copy(const Item& item) = 0;\n virtual void internal_add_move(Item&& item) = 0;\n\n template friend class d2::feeder_impl;\npublic:\n //! Add a work item to a running parallel_for_each.\n void add(const Item& item) {internal_add_copy(item);}\n void add(Item&& item) {internal_add_move(std::move(item));}\n};\n\n} // namespace d1\n\nnamespace d2 {\nusing namespace tbb::detail::d1;\n/** Selects one of the two possible forms of function call member operator.\n @ingroup algorithms **/\ntemplate\nstruct parallel_for_each_operator_selector {\npublic:\n template\n static auto call(const Body& body, ItemArg&& item, FeederArg*)\n -> decltype(tbb::detail::invoke(body, std::forward(item)), void()) {\n #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Suppression of Microsoft non-standard extension warnings\n // #pragma warning (push)\n // #pragma warning (disable: 4239)\n #endif\n\n tbb::detail::invoke(body, std::forward(item));\n\n #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // #pragma warning (pop)\n #endif\n }\n\n template\n static auto call(const Body& body, ItemArg&& item, FeederArg* feeder)\n -> decltype(tbb::detail::invoke(body, std::forward(item), *feeder), void()) {\n #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Suppression of Microsoft non-standard extension warnings\n // #pragma warning (push)\n // #pragma warning (disable: 4239)\n #endif\n __TBB_ASSERT(feeder, \"Feeder was not created but should be\");\n\n tbb::detail::invoke(body, std::forward(item), *feeder);\n\n #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // #pragma warning (pop)\n #endif\n }\n};\n\ntemplate\nstruct feeder_item_task: public task {\n using feeder_type = feeder_impl;\n\n template \n feeder_item_task(ItemType&& input_item, feeder_type& feeder, small_object_allocator& alloc, wait_tree_vertex_interface& wait_vertex) :\n item(std::forward(input_item)),\n my_feeder(feeder),\n my_allocator(alloc),\n m_wait_tree_vertex(r1::get_thread_reference_vertex(&wait_vertex))\n {\n m_wait_tree_vertex->reserve();\n }\n\n void finalize(const execution_data& ed) {\n m_wait_tree_vertex->release();\n my_allocator.delete_object(this, ed);\n }\n\n //! Hack for resolve ambiguity between calls to the body with and without moving the stored copy\n //! Executing body with moving the copy should have higher priority\n using first_priority = int;\n using second_priority = double;\n\n template \n static auto call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, first_priority)\n -> decltype(parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder), void())\n {\n parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder);\n }\n\n template \n static void call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, second_priority) {\n parallel_for_each_operator_selector::call(call_body, call_item, &call_feeder);\n }\n\n task* execute(execution_data& ed) override {\n call(my_feeder.my_body, item, my_feeder, first_priority{});\n finalize(ed);\n return nullptr;\n }\n\n task* cancel(execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n\n Item item;\n feeder_type& my_feeder;\n small_object_allocator my_allocator;\n wait_tree_vertex_interface* m_wait_tree_vertex;\n}; // class feeder_item_task\n\n/** Implements new task adding procedure.\n @ingroup algorithms **/\ntemplate\nclass feeder_impl : public feeder {\n // Avoiding use of copy constructor in a virtual method if the type does not support it\n void internal_add_copy_impl(std::true_type, const Item& item) {\n using feeder_task = feeder_item_task;\n small_object_allocator alloc;\n auto task = alloc.new_object(item, *this, alloc, my_wait_context);\n\n spawn(*task, my_execution_context);\n }\n\n void internal_add_copy_impl(std::false_type, const Item&) {\n __TBB_ASSERT(false, \"Overloading for r-value reference doesn't work or it's not movable and not copyable object\");\n }\n\n void internal_add_copy(const Item& item) override {\n internal_add_copy_impl(typename std::is_copy_constructible::type(), item);\n }\n\n void internal_add_move(Item&& item) override {\n using feeder_task = feeder_item_task;\n small_object_allocator alloc{};\n auto task = alloc.new_object(std::move(item), *this, alloc, my_wait_context);\n\n spawn(*task, my_execution_context);\n }\npublic:\n feeder_impl(const Body& body, wait_context_vertex& w_context, task_group_context &context)\n : my_body(body),\n my_wait_context(w_context)\n , my_execution_context(context)\n {}\n\n const Body& my_body;\n wait_context_vertex& my_wait_context;\n task_group_context& my_execution_context;\n}; // class feeder_impl\n\n/** Execute computation under one element of the range\n @ingroup algorithms **/\ntemplate\nstruct for_each_iteration_task: public task {\n using feeder_type = feeder_impl;\n\n for_each_iteration_task(Iterator input_item_ptr, const Body& body, feeder_impl* feeder_ptr, wait_context& wait_context) :\n item_ptr(input_item_ptr), my_body(body), my_feeder_ptr(feeder_ptr), parent_wait_context(wait_context)\n {}\n\n void finalize() {\n parent_wait_context.release();\n }\n\n task* execute(execution_data&) override {\n parallel_for_each_operator_selector::call(my_body, *item_ptr, my_feeder_ptr);\n finalize();\n return nullptr;\n }\n\n task* cancel(execution_data&) override {\n finalize();\n return nullptr;\n }\n\n Iterator item_ptr;\n const Body& my_body;\n feeder_impl* my_feeder_ptr;\n wait_context& parent_wait_context;\n}; // class for_each_iteration_task\n\n// Helper to get the type of the iterator to the internal sequence of copies\n// If the element can be passed to the body as an rvalue - this iterator should be move_iterator\ntemplate \nstruct input_iteration_task_iterator_helper {\n // For input iterators we pass const lvalue reference to the body\n // It is prohibited to take non-constant lvalue references for input iterators\n using type = const Item*;\n};\n\ntemplate \nstruct input_iteration_task_iterator_helper::call(std::declval(),\n std::declval(),\n std::declval*>()))>>\n{\n using type = std::move_iterator;\n};\n\n/** Split one block task to several(max_block_size) iteration tasks for input iterators\n @ingroup algorithms **/\ntemplate \nstruct input_block_handling_task : public task {\n static constexpr size_t max_block_size = 4;\n\n using feeder_type = feeder_impl;\n using iteration_task_iterator_type = typename input_iteration_task_iterator_helper::type;\n using iteration_task = for_each_iteration_task;\n\n input_block_handling_task(wait_context_vertex& root_wait_context, task_group_context& e_context,\n const Body& body, feeder_impl* feeder_ptr, small_object_allocator& alloc)\n :my_size(0), my_wait_context(0), my_root_wait_context(root_wait_context),\n my_execution_context(e_context), my_allocator(alloc)\n {\n auto item_it = block_iteration_space.begin();\n for (auto* it = task_pool.begin(); it != task_pool.end(); ++it) {\n new (it) iteration_task(iteration_task_iterator_type(item_it++), body, feeder_ptr, my_wait_context);\n }\n }\n\n void finalize(const execution_data& ed) {\n my_root_wait_context.release();\n my_allocator.delete_object(this, ed);\n }\n\n task* execute(execution_data& ed) override {\n __TBB_ASSERT( my_size > 0, \"Negative size was passed to task\");\n for (std::size_t counter = 1; counter < my_size; ++counter) {\n my_wait_context.reserve();\n spawn(*(task_pool.begin() + counter), my_execution_context);\n }\n my_wait_context.reserve();\n execute_and_wait(*task_pool.begin(), my_execution_context,\n my_wait_context, my_execution_context);\n\n // deallocate current task after children execution\n finalize(ed);\n return nullptr;\n }\n\n task* cancel(execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n\n ~input_block_handling_task() {\n for(std::size_t counter = 0; counter < max_block_size; ++counter) {\n (task_pool.begin() + counter)->~iteration_task();\n if (counter < my_size) {\n (block_iteration_space.begin() + counter)->~Item();\n }\n }\n }\n\n aligned_space block_iteration_space;\n aligned_space task_pool;\n std::size_t my_size;\n wait_context my_wait_context;\n wait_context_vertex& my_root_wait_context;\n task_group_context& my_execution_context;\n small_object_allocator my_allocator;\n}; // class input_block_handling_task\n\n/** Split one block task to several(max_block_size) iteration tasks for forward iterators\n @ingroup algorithms **/\ntemplate \nstruct forward_block_handling_task : public task {\n static constexpr size_t max_block_size = 4;\n\n using iteration_task = for_each_iteration_task;\n\n forward_block_handling_task(Iterator first, std::size_t size,\n wait_context_vertex& w_context, task_group_context& e_context,\n const Body& body, feeder_impl* feeder_ptr,\n small_object_allocator& alloc)\n : my_size(size), my_wait_context(0), my_root_wait_context(w_context),\n my_execution_context(e_context), my_allocator(alloc)\n {\n auto* task_it = task_pool.begin();\n for (std::size_t i = 0; i < size; i++) {\n new (task_it++) iteration_task(first, body, feeder_ptr, my_wait_context);\n ++first;\n }\n }\n\n void finalize(const execution_data& ed) {\n my_root_wait_context.release();\n my_allocator.delete_object(this, ed);\n }\n\n task* execute(execution_data& ed) override {\n __TBB_ASSERT( my_size > 0, \"Negative size was passed to task\");\n for(std::size_t counter = 1; counter < my_size; ++counter) {\n my_wait_context.reserve();\n spawn(*(task_pool.begin() + counter), my_execution_context);\n }\n my_wait_context.reserve();\n execute_and_wait(*task_pool.begin(), my_execution_context,\n my_wait_context, my_execution_context);\n\n // deallocate current task after children execution\n finalize(ed);\n return nullptr;\n }\n\n task* cancel(execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n\n ~forward_block_handling_task() {\n for(std::size_t counter = 0; counter < my_size; ++counter) {\n (task_pool.begin() + counter)->~iteration_task();\n }\n }\n\n aligned_space task_pool;\n std::size_t my_size;\n wait_context my_wait_context;\n wait_context_vertex& my_root_wait_context;\n task_group_context& my_execution_context;\n small_object_allocator my_allocator;\n}; // class forward_block_handling_task\n\n/** Body for parallel_for algorithm.\n * Allows to redirect operations under random access iterators range to the parallel_for algorithm.\n @ingroup algorithms **/\ntemplate \nclass parallel_for_body_wrapper {\n Iterator my_first;\n const Body& my_body;\n feeder_impl* my_feeder_ptr;\npublic:\n parallel_for_body_wrapper(Iterator first, const Body& body, feeder_impl* feeder_ptr)\n : my_first(first), my_body(body), my_feeder_ptr(feeder_ptr) {}\n\n void operator()(tbb::blocked_range range) const {\n#if __INTEL_COMPILER\n#pragma ivdep\n#endif\n for (std::size_t count = range.begin(); count != range.end(); count++) {\n parallel_for_each_operator_selector::call(my_body, *(my_first + count),\n my_feeder_ptr);\n }\n }\n}; // class parallel_for_body_wrapper\n\n\n/** Helper for getting iterators tag including inherited custom tags\n @ingroup algorithms */\ntemplate\nusing tag = typename std::iterator_traits::iterator_category;\n\n#if __TBB_CPP20_PRESENT\ntemplate \nstruct move_iterator_dispatch_helper {\n using type = It;\n};\n\n// Until C++23, std::move_iterator::iterator_concept always defines\n// to std::input_iterator_tag and hence std::forward_iterator concept\n// always evaluates to false, so std::move_iterator dispatch should be\n// made according to the base iterator type.\ntemplate \nstruct move_iterator_dispatch_helper> {\n using type = It;\n};\n\ntemplate \nusing iterator_tag_dispatch_impl =\n std::conditional_t,\n std::random_access_iterator_tag,\n std::conditional_t,\n std::forward_iterator_tag,\n std::input_iterator_tag>>;\n\ntemplate \nusing iterator_tag_dispatch =\n iterator_tag_dispatch_impl::type>;\n\n#else\ntemplate\nusing iterator_tag_dispatch = typename\n std::conditional<\n std::is_base_of>::value,\n std::random_access_iterator_tag,\n typename std::conditional<\n std::is_base_of>::value,\n std::forward_iterator_tag,\n std::input_iterator_tag\n >::type\n >::type;\n#endif // __TBB_CPP20_PRESENT\n\ntemplate \nusing feeder_is_required = tbb::detail::void_t(),\n std::declval::reference>(),\n std::declval&>()))>;\n\n// Creates feeder object only if the body can accept it\ntemplate \nstruct feeder_holder {\n feeder_holder( wait_context_vertex&, task_group_context&, const Body& ) {}\n\n feeder_impl* feeder_ptr() { return nullptr; }\n}; // class feeder_holder\n\ntemplate \nclass feeder_holder> {\npublic:\n feeder_holder( wait_context_vertex& w_context, task_group_context& context, const Body& body )\n : my_feeder(body, w_context, context) {}\n\n feeder_impl* feeder_ptr() { return &my_feeder; }\nprivate:\n feeder_impl my_feeder;\n}; // class feeder_holder\n\ntemplate \nclass for_each_root_task_base : public task {\npublic:\n for_each_root_task_base(Iterator first, Iterator last, const Body& body, wait_context_vertex& w_context, task_group_context& e_context)\n : my_first(first), my_last(last), my_wait_context(w_context), my_execution_context(e_context),\n my_body(body), my_feeder_holder(my_wait_context, my_execution_context, my_body)\n {\n my_wait_context.reserve();\n }\nprivate:\n task* cancel(execution_data&) override {\n this->my_wait_context.release();\n return nullptr;\n }\nprotected:\n Iterator my_first;\n Iterator my_last;\n wait_context_vertex& my_wait_context;\n task_group_context& my_execution_context;\n const Body& my_body;\n feeder_holder my_feeder_holder;\n}; // class for_each_root_task_base\n\n/** parallel_for_each algorithm root task - most generic version\n * Splits input range to blocks\n @ingroup algorithms **/\ntemplate >\nclass for_each_root_task : public for_each_root_task_base\n{\n using base_type = for_each_root_task_base;\npublic:\n using base_type::base_type;\nprivate:\n task* execute(execution_data& ed) override {\n using block_handling_type = input_block_handling_task;\n\n if (this->my_first == this->my_last) {\n this->my_wait_context.release();\n return nullptr;\n }\n\n this->my_wait_context.reserve();\n small_object_allocator alloc{};\n auto block_handling_task = alloc.new_object(ed, this->my_wait_context, this->my_execution_context,\n this->my_body, this->my_feeder_holder.feeder_ptr(),\n alloc);\n\n auto* block_iterator = block_handling_task->block_iteration_space.begin();\n for (; !(this->my_first == this->my_last) && block_handling_task->my_size < block_handling_type::max_block_size; ++this->my_first) {\n // Move semantics are automatically used when supported by the iterator\n new (block_iterator++) Item(*this->my_first);\n ++block_handling_task->my_size;\n }\n\n // Do not access this after spawn to avoid races\n spawn(*this, this->my_execution_context);\n return block_handling_task;\n }\n}; // class for_each_root_task - most generic implementation\n\n/** parallel_for_each algorithm root task - forward iterator based specialization\n * Splits input range to blocks\n @ingroup algorithms **/\ntemplate \nclass for_each_root_task\n : public for_each_root_task_base\n{\n using base_type = for_each_root_task_base;\npublic:\n using base_type::base_type;\nprivate:\n task* execute(execution_data& ed) override {\n using block_handling_type = forward_block_handling_task;\n if (this->my_first == this->my_last) {\n this->my_wait_context.release();\n return nullptr;\n }\n\n std::size_t block_size{0};\n Iterator first_block_element = this->my_first;\n for (; !(this->my_first == this->my_last) && block_size < block_handling_type::max_block_size; ++this->my_first) {\n ++block_size;\n }\n\n this->my_wait_context.reserve();\n small_object_allocator alloc{};\n auto block_handling_task = alloc.new_object(ed, first_block_element, block_size,\n this->my_wait_context, this->my_execution_context,\n this->my_body, this->my_feeder_holder.feeder_ptr(), alloc);\n\n // Do not access this after spawn to avoid races\n spawn(*this, this->my_execution_context);\n return block_handling_task;\n }\n}; // class for_each_root_task - forward iterator based specialization\n\n/** parallel_for_each algorithm root task - random access iterator based specialization\n * Splits input range to blocks\n @ingroup algorithms **/\ntemplate \nclass for_each_root_task\n : public for_each_root_task_base\n{\n using base_type = for_each_root_task_base;\npublic:\n using base_type::base_type;\nprivate:\n task* execute(execution_data&) override {\n tbb::parallel_for(\n tbb::blocked_range(0, std::distance(this->my_first, this->my_last)),\n parallel_for_body_wrapper(this->my_first, this->my_body, this->my_feeder_holder.feeder_ptr())\n , this->my_execution_context\n );\n\n this->my_wait_context.release();\n return nullptr;\n }\n}; // class for_each_root_task - random access iterator based specialization\n\n/** Helper for getting item type. If item type can be deduced from feeder - got it from feeder,\n if feeder is generic - got item type from range.\n @ingroup algorithms */\ntemplate\nauto feeder_argument_parser(void (Body::*)(Item, feeder&) const) -> FeederArg;\n\ntemplate\ndecltype(feeder_argument_parser(&Body::operator())) get_item_type_impl(int); // for (T, feeder)\ntemplate Item get_item_type_impl(...); // stub\n\ntemplate \nusing get_item_type = decltype(get_item_type_impl(0));\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\ntemplate \nusing feeder_item_type = std::remove_cvref_t>;\n\ntemplate \nconcept parallel_for_each_iterator_body =\n parallel_for_each_body, feeder_item_type>>;\n\ntemplate \nconcept parallel_for_each_range_body =\n parallel_for_each_body, feeder_item_type>>;\n#endif\n\n/** Implements parallel iteration over a range.\n @ingroup algorithms */\ntemplate\nvoid run_parallel_for_each( Iterator first, Iterator last, const Body& body, task_group_context& context)\n{\n if (!(first == last)) {\n using ItemType = get_item_type::value_type>;\n wait_context_vertex w_context(0);\n\n for_each_root_task root_task(first, last, body, w_context, context);\n\n execute_and_wait(root_task, context, w_context.get_context(), context);\n }\n}\n\n/** \\page parallel_for_each_body_req Requirements on parallel_for_each body\n Class \\c Body implementing the concept of parallel_for_each body must define:\n - \\code\n B::operator()(\n cv_item_type item,\n feeder& feeder\n ) const\n\n OR\n\n B::operator()( cv_item_type& item ) const\n \\endcode Process item.\n May be invoked concurrently for the same \\c this but different \\c item.\n\n - \\code item_type( const item_type& ) \\endcode\n Copy a work item.\n - \\code ~item_type() \\endcode Destroy a work item\n**/\n\n/** \\name parallel_for_each\n See also requirements on \\ref parallel_for_each_body_req \"parallel_for_each Body\". **/\n//@{\n//! Parallel iteration over a range, with optional addition of more work.\n/** @ingroup algorithms */\ntemplate\n __TBB_requires(std::input_iterator && parallel_for_each_iterator_body)\nvoid parallel_for_each(Iterator first, Iterator last, const Body& body) {\n task_group_context context(PARALLEL_FOR_EACH);\n run_parallel_for_each(first, last, body, context);\n}\n\ntemplate\n __TBB_requires(container_based_sequence && parallel_for_each_range_body)\nvoid parallel_for_each(Range& rng, const Body& body) {\n parallel_for_each(std::begin(rng), std::end(rng), body);\n}\n\ntemplate\n __TBB_requires(container_based_sequence && parallel_for_each_range_body)\nvoid parallel_for_each(const Range& rng, const Body& body) {\n parallel_for_each(std::begin(rng), std::end(rng), body);\n}\n\n//! Parallel iteration over a range, with optional addition of more work and user-supplied context\n/** @ingroup algorithms */\ntemplate\n __TBB_requires(std::input_iterator && parallel_for_each_iterator_body)\nvoid parallel_for_each(Iterator first, Iterator last, const Body& body, task_group_context& context) {\n run_parallel_for_each(first, last, body, context);\n}\n\ntemplate\n __TBB_requires(container_based_sequence && parallel_for_each_range_body)\nvoid parallel_for_each(Range& rng, const Body& body, task_group_context& context) {\n parallel_for_each(std::begin(rng), std::end(rng), body, context);\n}\n\ntemplate\n __TBB_requires(container_based_sequence && parallel_for_each_range_body)\nvoid parallel_for_each(const Range& rng, const Body& body, task_group_context& context) {\n parallel_for_each(std::begin(rng), std::end(rng), body, context);\n}\n\n} // namespace d2\n} // namespace detail\n//! @endcond\n//@}\n\ninline namespace v1 {\nusing detail::d2::parallel_for_each;\nusing detail::d1::feeder;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_parallel_for_each_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_invoke.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_invoke_H\n#define __TBB_parallel_invoke_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_task.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_small_object_pool.h\"\n\n#include \"task_group.h\"\n\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! Simple task object, executing user method\ntemplate\nstruct function_invoker : public task {\n function_invoker(const Function& function, WaitObject& wait_ctx) :\n my_function(function),\n parent_wait_ctx(wait_ctx)\n {}\n\n task* execute(execution_data& ed) override {\n my_function();\n parent_wait_ctx.release(ed);\n call_itt_task_notify(destroy, this);\n return nullptr;\n }\n\n task* cancel(execution_data& ed) override {\n parent_wait_ctx.release(ed);\n return nullptr;\n }\n\n const Function& my_function;\n WaitObject& parent_wait_ctx;\n}; // struct function_invoker\n\n//! Task object for managing subroots in trinary task trees.\n// Endowed with additional synchronization logic (compatible with wait object interfaces) to support\n// continuation passing execution. This task spawns 2 function_invoker tasks with first and second functors\n// and then executes first functor by itself. But only the last executed functor must destruct and deallocate\n// the subroot task.\ntemplate\nstruct invoke_subroot_task : public task {\n wait_context& root_wait_ctx;\n std::atomic ref_count{0};\n bool child_spawned = false;\n\n const F1& self_invoked_functor;\n function_invoker> f2_invoker;\n function_invoker> f3_invoker;\n\n task_group_context& my_execution_context;\n small_object_allocator my_allocator;\n\n invoke_subroot_task(const F1& f1, const F2& f2, const F3& f3, wait_context& wait_ctx, task_group_context& context,\n small_object_allocator& alloc) :\n root_wait_ctx(wait_ctx),\n self_invoked_functor(f1),\n f2_invoker(f2, *this),\n f3_invoker(f3, *this),\n my_execution_context(context),\n my_allocator(alloc)\n {\n root_wait_ctx.reserve();\n }\n\n void finalize(const execution_data& ed) {\n root_wait_ctx.release();\n\n my_allocator.delete_object(this, ed);\n }\n\n void release(const execution_data& ed) {\n __TBB_ASSERT(ref_count > 0, nullptr);\n call_itt_task_notify(releasing, this);\n if( --ref_count == 0 ) {\n call_itt_task_notify(acquired, this);\n finalize(ed);\n }\n }\n\n task* execute(execution_data& ed) override {\n ref_count.fetch_add(3, std::memory_order_relaxed);\n spawn(f3_invoker, my_execution_context);\n spawn(f2_invoker, my_execution_context);\n self_invoked_functor();\n\n release(ed);\n return nullptr;\n }\n\n task* cancel(execution_data& ed) override {\n if( ref_count > 0 ) { // detect children spawn\n release(ed);\n } else {\n finalize(ed);\n }\n return nullptr;\n }\n}; // struct subroot_task\n\nclass invoke_root_task {\npublic:\n invoke_root_task(wait_context& wc) : my_wait_context(wc) {}\n void release(const execution_data&) {\n my_wait_context.release();\n }\nprivate:\n wait_context& my_wait_context;\n};\n\ntemplate\nvoid invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1) {\n root_wait_ctx.reserve(1);\n invoke_root_task root(root_wait_ctx);\n function_invoker invoker1(f1, root);\n\n execute_and_wait(invoker1, context, root_wait_ctx, context);\n}\n\ntemplate\nvoid invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2) {\n root_wait_ctx.reserve(2);\n invoke_root_task root(root_wait_ctx);\n function_invoker invoker1(f1, root);\n function_invoker invoker2(f2, root);\n\n spawn(invoker1, context);\n execute_and_wait(invoker2, context, root_wait_ctx, context);\n}\n\ntemplate\nvoid invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2, const F3& f3) {\n root_wait_ctx.reserve(3);\n invoke_root_task root(root_wait_ctx);\n function_invoker invoker1(f1, root);\n function_invoker invoker2(f2, root);\n function_invoker invoker3(f3, root);\n\n //TODO: implement sub root for two tasks (measure performance)\n spawn(invoker1, context);\n spawn(invoker2, context);\n execute_and_wait(invoker3, context, root_wait_ctx, context);\n}\n\ntemplate\nvoid invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context,\n const F1& f1, const F2& f2, const F3& f3, const Fs&... fs) {\n small_object_allocator alloc{};\n auto sub_root = alloc.new_object>(f1, f2, f3, root_wait_ctx, context, alloc);\n spawn(*sub_root, context);\n\n invoke_recursive_separation(root_wait_ctx, context, fs...);\n}\n\ntemplate\nvoid parallel_invoke_impl(task_group_context& context, const Fs&... fs) {\n static_assert(sizeof...(Fs) >= 2, \"Parallel invoke may be called with at least two callable\");\n wait_context root_wait_ctx{0};\n\n invoke_recursive_separation(root_wait_ctx, context, fs...);\n}\n\ntemplate\nvoid parallel_invoke_impl(const F1& f1, const Fs&... fs) {\n static_assert(sizeof...(Fs) >= 1, \"Parallel invoke may be called with at least two callable\");\n task_group_context context(PARALLEL_INVOKE);\n wait_context root_wait_ctx{0};\n\n invoke_recursive_separation(root_wait_ctx, context, fs..., f1);\n}\n\n//! Passes last argument of variadic pack as first for handling user provided task_group_context\ntemplate \nstruct invoke_helper;\n\ntemplate \nstruct invoke_helper, T, Fs...> : invoke_helper, Fs...> {};\n\ntemplate \nstruct invoke_helper, T> {\n void operator()(Fs&&... args, T&& t) {\n parallel_invoke_impl(std::forward(t), std::forward(args)...);\n }\n};\n\n//! Parallel execution of several function objects\n// We need to pass parameter pack through forwarding reference,\n// since this pack may contain task_group_context that must be passed via lvalue non-const reference\ntemplate\nvoid parallel_invoke(Fs&&... fs) {\n invoke_helper, Fs...>()(std::forward(fs)...);\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::parallel_invoke;\n} // namespace v1\n\n} // namespace tbb\n#endif /* __TBB_parallel_invoke_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_pipeline.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_pipeline_H\n#define __TBB_parallel_pipeline_H\n\n#include \"detail/_pipeline_filters.h\"\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"task_group.h\"\n\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nTBB_EXPORT void __TBB_EXPORTED_FUNC parallel_pipeline(task_group_context&, std::size_t, const d1::filter_node&);\n}\n\nnamespace d1 {\n\nenum class filter_mode : unsigned int\n{\n //! processes multiple items in parallel and in no particular order\n parallel = base_filter::filter_is_out_of_order,\n //! processes items one at a time; all such filters process items in the same order\n serial_in_order = base_filter::filter_is_serial,\n //! processes items one at a time and in no particular order\n serial_out_of_order = base_filter::filter_is_serial | base_filter::filter_is_out_of_order\n};\n//! Class representing a chain of type-safe pipeline filters\n/** @ingroup algorithms */\ntemplate\nclass filter {\n filter_node_ptr my_root;\n filter( filter_node_ptr root ) : my_root(root) {}\n friend void parallel_pipeline( size_t, const filter&, task_group_context& );\n template\n friend filter make_filter( filter_mode, const Body& );\n template\n friend filter operator&( const filter&, const filter& );\npublic:\n filter() = default;\n filter( const filter& rhs ) : my_root(rhs.my_root) {}\n filter( filter&& rhs ) : my_root(std::move(rhs.my_root)) {}\n\n void operator=(const filter& rhs) {\n my_root = rhs.my_root;\n }\n void operator=( filter&& rhs ) {\n my_root = std::move(rhs.my_root);\n }\n\n template\n filter( filter_mode mode, const Body& body ) :\n my_root( new(r1::allocate_memory(sizeof(filter_node_leaf)))\n filter_node_leaf(static_cast(mode), body) ) {\n }\n\n filter& operator&=( const filter& right ) {\n *this = *this & right;\n return *this;\n }\n\n void clear() {\n // Like operator= with filter() on right side.\n my_root = nullptr;\n }\n};\n\n//! Create a filter to participate in parallel_pipeline\n/** @ingroup algorithms */\ntemplate\nfilter make_filter( filter_mode mode, const Body& body ) {\n return filter_node_ptr( new(r1::allocate_memory(sizeof(filter_node_leaf)))\n filter_node_leaf(static_cast(mode), body) );\n}\n\n//! Create a filter to participate in parallel_pipeline\n/** @ingroup algorithms */\ntemplate\nfilter, filter_output> make_filter( filter_mode mode, const Body& body ) {\n return make_filter, filter_output>(mode, body);\n}\n\n//! Composition of filters left and right.\n/** @ingroup algorithms */\ntemplate\nfilter operator&( const filter& left, const filter& right ) {\n __TBB_ASSERT(left.my_root,\"cannot use default-constructed filter as left argument of '&'\");\n __TBB_ASSERT(right.my_root,\"cannot use default-constructed filter as right argument of '&'\");\n return filter_node_ptr( new (r1::allocate_memory(sizeof(filter_node))) filter_node(left.my_root,right.my_root) );\n}\n\n#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\ntemplate\nfilter(filter_mode, Body)\n->filter, filter_output>;\n#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT\n\n//! Parallel pipeline over chain of filters with user-supplied context.\n/** @ingroup algorithms **/\ninline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain, task_group_context& context) {\n r1::parallel_pipeline(context, max_number_of_live_tokens, *filter_chain.my_root);\n}\n\n//! Parallel pipeline over chain of filters.\n/** @ingroup algorithms **/\ninline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain) {\n task_group_context context;\n parallel_pipeline(max_number_of_live_tokens, filter_chain, context);\n}\n\n//! Parallel pipeline over sequence of filters.\n/** @ingroup algorithms **/\ntemplate\nvoid parallel_pipeline(size_t max_number_of_live_tokens,\n const F1& filter1,\n const F2& filter2,\n FiltersContext&&... filters) {\n parallel_pipeline(max_number_of_live_tokens, filter1 & filter2, std::forward(filters)...);\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1\n{\nusing detail::d1::parallel_pipeline;\nusing detail::d1::filter;\nusing detail::d1::make_filter;\nusing detail::d1::filter_mode;\nusing detail::d1::flow_control;\n}\n} // tbb\n\n#endif /* __TBB_parallel_pipeline_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_reduce.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_reduce_H\n#define __TBB_parallel_reduce_H\n\n#include \n#include \"detail/_namespace_injection.h\"\n#include \"detail/_task.h\"\n#include \"detail/_aligned_space.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"detail/_range_common.h\"\n\n#include \"task_group.h\" // task_group_context\n#include \"partitioner.h\"\n#include \"profiling.h\"\n\nnamespace tbb {\nnamespace detail {\n#if __TBB_CPP20_CONCEPTS_PRESENT\ninline namespace d0 {\n\ntemplate \nconcept parallel_reduce_body = splittable &&\n requires( Body& body, const Range& range, Body& rhs ) {\n body(range);\n body.join(rhs);\n };\n\ntemplate \nconcept parallel_reduce_function = std::invocable&,\n const Range&, Value&&> &&\n std::convertible_to&,\n const Range&, Value&&>,\n Value>;\n\ntemplate \nconcept parallel_reduce_combine = std::invocable&,\n Value&&, Value&&> &&\n std::convertible_to&,\n Value&&, Value&&>,\n Value>;\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d1 {\n\n//! Tree node type for parallel_reduce.\n/** @ingroup algorithms */\n//TODO: consider folding tree via bypass execution(instead of manual folding)\n// for better cancellation and critical tasks handling (performance measurements required).\ntemplate\nstruct reduction_tree_node : public tree_node {\n tbb::detail::aligned_space zombie_space;\n Body& left_body;\n bool has_right_zombie{false};\n\n reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) :\n tree_node{parent, ref_count, alloc},\n left_body(input_left_body) /* gcc4.8 bug - braced-initialization doesn't work for class members of reference type */\n {}\n\n void join(task_group_context* context) {\n if (has_right_zombie && !context->is_group_execution_cancelled())\n left_body.join(*zombie_space.begin());\n }\n\n ~reduction_tree_node() {\n if( has_right_zombie ) zombie_space.begin()->~Body();\n }\n};\n\n//! Task type used to split the work of parallel_reduce.\n/** @ingroup algorithms */\ntemplate\nstruct start_reduce : public task {\n Range my_range;\n Body* my_body;\n node* my_parent;\n\n typename Partitioner::task_partition_type my_partition;\n small_object_allocator my_allocator;\n bool is_right_child;\n\n task* execute(execution_data&) override;\n task* cancel(execution_data&) override;\n void finalize(const execution_data&);\n\n using tree_node_type = reduction_tree_node;\n\n //! Constructor reduce root task.\n start_reduce( const Range& range, Body& body, Partitioner& partitioner, small_object_allocator& alloc ) :\n my_range(range),\n my_body(&body),\n my_parent(nullptr),\n my_partition(partitioner),\n my_allocator(alloc),\n is_right_child(false) {}\n //! Splitting constructor used to generate children.\n /** parent_ becomes left child. Newly constructed object is right child. */\n start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) :\n my_range(parent_.my_range, get_range_split_object(split_obj)),\n my_body(parent_.my_body),\n my_parent(nullptr),\n my_partition(parent_.my_partition, split_obj),\n my_allocator(alloc),\n is_right_child(true)\n {\n parent_.is_right_child = false;\n }\n //! Construct right child from the given range as response to the demand.\n /** parent_ remains left child. Newly constructed object is right child. */\n start_reduce( start_reduce& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) :\n my_range(r),\n my_body(parent_.my_body),\n my_parent(nullptr),\n my_partition(parent_.my_partition, split()),\n my_allocator(alloc),\n is_right_child(true)\n {\n my_partition.align_depth( d );\n parent_.is_right_child = false;\n }\n static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) {\n if ( !range.empty() ) {\n wait_node wn;\n small_object_allocator alloc{};\n auto reduce_task = alloc.new_object(range, body, partitioner, alloc);\n reduce_task->my_parent = &wn;\n execute_and_wait(*reduce_task, context, wn.m_wait, context);\n }\n }\n static void run(const Range& range, Body& body, Partitioner& partitioner) {\n // Bound context prevents exceptions from body to affect nesting or sibling algorithms,\n // and allows users to handle exceptions safely by wrapping parallel_reduce in the try-block.\n task_group_context context(PARALLEL_REDUCE);\n run(range, body, partitioner, context);\n }\n //! Run body for range, serves as callback for partitioner\n void run_body( Range &r ) {\n tbb::detail::invoke(*my_body, r);\n }\n\n //! spawn right task, serves as callback for partitioner\n void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) {\n offer_work_impl(ed, *this, split_obj);\n }\n //! spawn right task, serves as callback for partitioner\n void offer_work(const Range& r, depth_t d, execution_data& ed) {\n offer_work_impl(ed, *this, r, d);\n }\n\nprivate:\n template \n void offer_work_impl(execution_data& ed, Args&&... args) {\n small_object_allocator alloc{};\n // New right child\n auto right_child = alloc.new_object(ed, std::forward(args)..., alloc);\n\n // New root node as a continuation and ref count. Left and right child attach to the new parent.\n right_child->my_parent = my_parent = alloc.new_object(ed, my_parent, 2, *my_body, alloc);\n\n // Spawn the right sibling\n right_child->spawn_self(ed);\n }\n\n void spawn_self(execution_data& ed) {\n my_partition.spawn_task(*this, *context(ed));\n }\n};\n\n//! fold the tree and deallocate the task\ntemplate\nvoid start_reduce::finalize(const execution_data& ed) {\n // Get the current parent and wait object before an object destruction\n node* parent = my_parent;\n auto allocator = my_allocator;\n // Task execution finished - destroy it\n this->~start_reduce();\n // Unwind the tree decrementing the parent`s reference count\n fold_tree(parent, ed);\n allocator.deallocate(this, ed);\n}\n\n//! Execute parallel_reduce task\ntemplate\ntask* start_reduce::execute(execution_data& ed) {\n if (!is_same_affinity(ed)) {\n my_partition.note_affinity(execution_slot(ed));\n }\n my_partition.check_being_stolen(*this, ed);\n\n // The acquire barrier synchronizes the data pointed with my_body if the left\n // task has already finished.\n __TBB_ASSERT(my_parent, nullptr);\n if( is_right_child && my_parent->m_ref_count.load(std::memory_order_acquire) == 2 ) {\n tree_node_type* parent_ptr = static_cast(my_parent);\n my_body = static_cast(new( parent_ptr->zombie_space.begin() ) Body(*my_body, split()));\n parent_ptr->has_right_zombie = true;\n }\n __TBB_ASSERT(my_body != nullptr, \"Incorrect body value\");\n\n my_partition.execute(*this, my_range, ed);\n\n finalize(ed);\n return nullptr;\n}\n\n//! Cancel parallel_reduce task\ntemplate\ntask* start_reduce::cancel(execution_data& ed) {\n finalize(ed);\n return nullptr;\n}\n\n//! Tree node type for parallel_deterministic_reduce.\n/** @ingroup algorithms */\ntemplate\nstruct deterministic_reduction_tree_node : public tree_node {\n Body right_body;\n Body& left_body;\n\n deterministic_reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) :\n tree_node{parent, ref_count, alloc},\n right_body{input_left_body, detail::split()},\n left_body(input_left_body)\n {}\n\n void join(task_group_context* context) {\n if (!context->is_group_execution_cancelled())\n left_body.join(right_body);\n }\n};\n\n//! Task type used to split the work of parallel_deterministic_reduce.\n/** @ingroup algorithms */\ntemplate\nstruct start_deterministic_reduce : public task {\n Range my_range;\n Body& my_body;\n node* my_parent;\n\n typename Partitioner::task_partition_type my_partition;\n small_object_allocator my_allocator;\n\n task* execute(execution_data&) override;\n task* cancel(execution_data&) override;\n void finalize(const execution_data&);\n\n using tree_node_type = deterministic_reduction_tree_node;\n\n //! Constructor deterministic_reduce root task.\n start_deterministic_reduce( const Range& range, Partitioner& partitioner, Body& body, small_object_allocator& alloc ) :\n my_range(range),\n my_body(body),\n my_parent(nullptr),\n my_partition(partitioner),\n my_allocator(alloc) {}\n //! Splitting constructor used to generate children.\n /** parent_ becomes left child. Newly constructed object is right child. */\n start_deterministic_reduce( start_deterministic_reduce& parent_, typename Partitioner::split_type& split_obj, Body& body,\n small_object_allocator& alloc ) :\n my_range(parent_.my_range, get_range_split_object(split_obj)),\n my_body(body),\n my_parent(nullptr),\n my_partition(parent_.my_partition, split_obj),\n my_allocator(alloc) {}\n static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) {\n if ( !range.empty() ) {\n wait_node wn;\n small_object_allocator alloc{};\n auto deterministic_reduce_task =\n alloc.new_object(range, partitioner, body, alloc);\n deterministic_reduce_task->my_parent = &wn;\n execute_and_wait(*deterministic_reduce_task, context, wn.m_wait, context);\n }\n }\n static void run(const Range& range, Body& body, Partitioner& partitioner) {\n // Bound context prevents exceptions from body to affect nesting or sibling algorithms,\n // and allows users to handle exceptions safely by wrapping parallel_deterministic_reduce\n // in the try-block.\n task_group_context context(PARALLEL_REDUCE);\n run(range, body, partitioner, context);\n }\n //! Run body for range, serves as callback for partitioner\n void run_body( Range &r ) {\n tbb::detail::invoke(my_body, r);\n }\n //! Spawn right task, serves as callback for partitioner\n void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) {\n offer_work_impl(ed, *this, split_obj);\n }\nprivate:\n template \n void offer_work_impl(execution_data& ed, Args&&... args) {\n small_object_allocator alloc{};\n // New root node as a continuation and ref count. Left and right child attach to the new parent. Split the body.\n auto new_tree_node = alloc.new_object(ed, my_parent, 2, my_body, alloc);\n\n // New right child\n auto right_child = alloc.new_object(ed, std::forward(args)..., new_tree_node->right_body, alloc);\n\n right_child->my_parent = my_parent = new_tree_node;\n\n // Spawn the right sibling\n right_child->spawn_self(ed);\n }\n\n void spawn_self(execution_data& ed) {\n my_partition.spawn_task(*this, *context(ed));\n }\n};\n\n//! Fold the tree and deallocate the task\ntemplate\nvoid start_deterministic_reduce::finalize(const execution_data& ed) {\n // Get the current parent and wait object before an object destruction\n node* parent = my_parent;\n\n auto allocator = my_allocator;\n // Task execution finished - destroy it\n this->~start_deterministic_reduce();\n // Unwind the tree decrementing the parent`s reference count\n fold_tree(parent, ed);\n allocator.deallocate(this, ed);\n}\n\n//! Execute parallel_deterministic_reduce task\ntemplate\ntask* start_deterministic_reduce::execute(execution_data& ed) {\n if (!is_same_affinity(ed)) {\n my_partition.note_affinity(execution_slot(ed));\n }\n my_partition.check_being_stolen(*this, ed);\n\n my_partition.execute(*this, my_range, ed);\n\n finalize(ed);\n return nullptr;\n}\n\n//! Cancel parallel_deterministic_reduce task\ntemplate\ntask* start_deterministic_reduce::cancel(execution_data& ed) {\n finalize(ed);\n return nullptr;\n}\n\n\n//! Auxiliary class for parallel_reduce; for internal use only.\n/** The adaptor class that implements \\ref parallel_reduce_body_req \"parallel_reduce Body\"\n using given \\ref parallel_reduce_lambda_req \"anonymous function objects\".\n **/\n/** @ingroup algorithms */\ntemplate\nclass lambda_reduce_body {\n//TODO: decide if my_real_body, my_reduction, and my_identity_element should be copied or referenced\n// (might require some performance measurements)\n\n const Value& my_identity_element;\n const RealBody& my_real_body;\n const Reduction& my_reduction;\n Value my_value;\n lambda_reduce_body& operator= ( const lambda_reduce_body& other );\npublic:\n lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction )\n : my_identity_element(identity)\n , my_real_body(body)\n , my_reduction(reduction)\n , my_value(identity)\n { }\n lambda_reduce_body( const lambda_reduce_body& other ) = default;\n lambda_reduce_body( lambda_reduce_body& other, tbb::split )\n : my_identity_element(other.my_identity_element)\n , my_real_body(other.my_real_body)\n , my_reduction(other.my_reduction)\n , my_value(other.my_identity_element)\n { }\n void operator()(Range& range) {\n my_value = tbb::detail::invoke(my_real_body, range, std::move(my_value));\n }\n\n void join( lambda_reduce_body& rhs ) {\n my_value = tbb::detail::invoke(my_reduction, std::move(my_value), std::move(rhs.my_value));\n }\n\n __TBB_nodiscard Value&& result() && noexcept {\n return std::move(my_value);\n }\n};\n\n\n// Requirements on Range concept are documented in blocked_range.h\n\n/** \\page parallel_reduce_body_req Requirements on parallel_reduce body\n Class \\c Body implementing the concept of parallel_reduce body must define:\n - \\code Body::Body( Body&, split ); \\endcode Splitting constructor.\n Must be able to run concurrently with operator() and method \\c join\n - \\code Body::~Body(); \\endcode Destructor\n - \\code void Body::operator()( Range& r ); \\endcode Function call operator applying body to range \\c r\n and accumulating the result\n - \\code void Body::join( Body& b ); \\endcode Join results.\n The result in \\c b should be merged into the result of \\c this\n**/\n\n/** \\page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions)\n TO BE DOCUMENTED\n**/\n\n/** \\name parallel_reduce\n See also requirements on \\ref range_req \"Range\" and \\ref parallel_reduce_body_req \"parallel_reduce Body\". **/\n//@{\n\n//! Parallel iteration with reduction and default partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body ) {\n start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() );\n}\n\n//! Parallel iteration with reduction and simple_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) {\n start_reduce::run( range, body, partitioner );\n}\n\n//! Parallel iteration with reduction and auto_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) {\n start_reduce::run( range, body, partitioner );\n}\n\n//! Parallel iteration with reduction and static_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) {\n start_reduce::run( range, body, partitioner );\n}\n\n//! Parallel iteration with reduction and affinity_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) {\n start_reduce::run( range, body, partitioner );\n}\n\n//! Parallel iteration with reduction, default partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, task_group_context& context ) {\n start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context );\n}\n\n//! Parallel iteration with reduction, simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) {\n start_reduce::run( range, body, partitioner, context );\n}\n\n//! Parallel iteration with reduction, auto_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) {\n start_reduce::run( range, body, partitioner, context );\n}\n\n//! Parallel iteration with reduction, static_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) {\n start_reduce::run( range, body, partitioner, context );\n}\n\n//! Parallel iteration with reduction, affinity_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) {\n start_reduce::run( range, body, partitioner, context );\n}\n/** parallel_reduce overloads that work with anonymous function objects\n (see also \\ref parallel_reduce_lambda_req \"requirements on parallel_reduce anonymous function objects\"). **/\n\n//! Parallel iteration with reduction and default partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const __TBB_DEFAULT_PARTITIONER>\n ::run(range, body, __TBB_DEFAULT_PARTITIONER() );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction and simple_partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const simple_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const simple_partitioner>\n ::run(range, body, partitioner );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction and auto_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const auto_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const auto_partitioner>\n ::run( range, body, partitioner );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction and static_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const static_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const static_partitioner>\n ::run( range, body, partitioner );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction and affinity_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n affinity_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,affinity_partitioner>\n ::run( range, body, partitioner );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction, default partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const __TBB_DEFAULT_PARTITIONER>\n ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction, simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const simple_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const simple_partitioner>\n ::run( range, body, partitioner, context );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction, auto_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const auto_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const auto_partitioner>\n ::run( range, body, partitioner, context );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction, static_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const static_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,const static_partitioner>\n ::run( range, body, partitioner, context );\n return std::move(body).result();\n}\n\n//! Parallel iteration with reduction, affinity_partitioner and user-supplied context\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n affinity_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_reduce,affinity_partitioner>\n ::run( range, body, partitioner, context );\n return std::move(body).result();\n}\n\n//! Parallel iteration with deterministic reduction and default simple partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body ) {\n start_deterministic_reduce::run(range, body, simple_partitioner());\n}\n\n//! Parallel iteration with deterministic reduction and simple partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) {\n start_deterministic_reduce::run(range, body, partitioner);\n}\n\n//! Parallel iteration with deterministic reduction and static partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) {\n start_deterministic_reduce::run(range, body, partitioner);\n}\n\n//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) {\n start_deterministic_reduce::run( range, body, simple_partitioner(), context );\n}\n\n//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) {\n start_deterministic_reduce::run(range, body, partitioner, context);\n}\n\n//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_body)\nvoid parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) {\n start_deterministic_reduce::run(range, body, partitioner, context);\n}\n\n/** parallel_reduce overloads that work with anonymous function objects\n (see also \\ref parallel_reduce_lambda_req \"requirements on parallel_reduce anonymous function objects\"). **/\n\n//! Parallel iteration with deterministic reduction and default simple partitioner.\n// TODO: consider making static_partitioner the default\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) {\n return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner());\n}\n\n//! Parallel iteration with deterministic reduction and simple partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_deterministic_reduce, const simple_partitioner>\n ::run(range, body, partitioner);\n return std::move(body).result();\n}\n\n//! Parallel iteration with deterministic reduction and static partitioner.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_deterministic_reduce, const static_partitioner>\n ::run(range, body, partitioner);\n return std::move(body).result();\n}\n\n//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n task_group_context& context ) {\n return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context);\n}\n\n//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const simple_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_deterministic_reduce, const simple_partitioner>\n ::run(range, body, partitioner, context);\n return std::move(body).result();\n}\n\n//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context.\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_reduce_function &&\n parallel_reduce_combine)\nValue parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction,\n const static_partitioner& partitioner, task_group_context& context ) {\n lambda_reduce_body body(identity, real_body, reduction);\n start_deterministic_reduce, const static_partitioner>\n ::run(range, body, partitioner, context);\n return std::move(body).result();\n}\n//@}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::parallel_reduce;\nusing detail::d1::parallel_deterministic_reduce;\n// Split types\nusing detail::split;\nusing detail::proportional_split;\n} // namespace v1\n\n} // namespace tbb\n#endif /* __TBB_parallel_reduce_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_scan.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_scan_H\n#define __TBB_parallel_scan_H\n\n#include \n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_task.h\"\n\n#include \"profiling.h\"\n#include \"partitioner.h\"\n#include \"blocked_range.h\"\n#include \"task_group.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! Used to indicate that the initial scan is being performed.\n/** @ingroup algorithms */\nstruct pre_scan_tag {\n static bool is_final_scan() {return false;}\n operator bool() {return is_final_scan();}\n};\n\n//! Used to indicate that the final scan is being performed.\n/** @ingroup algorithms */\nstruct final_scan_tag {\n static bool is_final_scan() {return true;}\n operator bool() {return is_final_scan();}\n};\n\ntemplate\nstruct sum_node;\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\n} // namespace d1\nnamespace d0 {\n\ntemplate \nconcept parallel_scan_body = splittable &&\n requires( Body& body, const Range& range, Body& other ) {\n body(range, tbb::detail::d1::pre_scan_tag{});\n body(range, tbb::detail::d1::final_scan_tag{});\n body.reverse_join(other);\n body.assign(other);\n };\n\ntemplate \nconcept parallel_scan_function = std::invocable&,\n const Range&, const Value&, bool> &&\n std::convertible_to&,\n const Range&, const Value&, bool>,\n Value>;\n\ntemplate \nconcept parallel_scan_combine = std::invocable&,\n const Value&, const Value&> &&\n std::convertible_to&,\n const Value&, const Value&>,\n Value>;\n\n} // namespace d0\nnamespace d1 {\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\n\n//! Performs final scan for a leaf\n/** @ingroup algorithms */\ntemplate\nstruct final_sum : public task {\nprivate:\n using sum_node_type = sum_node;\n Body m_body;\n aligned_space m_range;\n //! Where to put result of last subrange, or nullptr if not last subrange.\n Body* m_stuff_last;\n\n wait_context& m_wait_context;\n sum_node_type* m_parent = nullptr;\npublic:\n small_object_allocator m_allocator;\n final_sum( Body& body, wait_context& w_o, small_object_allocator& alloc ) :\n m_body(body, split()), m_wait_context(w_o), m_allocator(alloc) {\n poison_pointer(m_stuff_last);\n }\n\n final_sum( final_sum& sum, small_object_allocator& alloc ) :\n m_body(sum.m_body, split()), m_wait_context(sum.m_wait_context), m_allocator(alloc) {\n poison_pointer(m_stuff_last);\n }\n\n ~final_sum() {\n m_range.begin()->~Range();\n }\n void finish_construction( sum_node_type* parent, const Range& range, Body* stuff_last ) {\n __TBB_ASSERT( m_parent == nullptr, nullptr );\n m_parent = parent;\n new( m_range.begin() ) Range(range);\n m_stuff_last = stuff_last;\n }\nprivate:\n sum_node_type* release_parent() {\n call_itt_task_notify(releasing, m_parent);\n if (m_parent) {\n auto parent = m_parent;\n m_parent = nullptr;\n if (parent->ref_count.fetch_sub(1) == 1) {\n return parent;\n }\n }\n else\n m_wait_context.release();\n return nullptr;\n }\n sum_node_type* finalize(const execution_data& ed){\n sum_node_type* next_task = release_parent();\n m_allocator.delete_object(this, ed);\n return next_task;\n }\n\npublic:\n task* execute(execution_data& ed) override {\n m_body( *m_range.begin(), final_scan_tag() );\n if( m_stuff_last )\n m_stuff_last->assign(m_body);\n\n return finalize(ed);\n }\n task* cancel(execution_data& ed) override {\n return finalize(ed);\n }\n template\n void operator()( const Range& r, Tag tag ) {\n m_body( r, tag );\n }\n void reverse_join( final_sum& a ) {\n m_body.reverse_join(a.m_body);\n }\n void reverse_join( Body& body ) {\n m_body.reverse_join(body);\n }\n void assign_to( Body& body ) {\n body.assign(m_body);\n }\n void self_destroy(const execution_data& ed) {\n m_allocator.delete_object(this, ed);\n }\n};\n\n//! Split work to be done in the scan.\n/** @ingroup algorithms */\ntemplate\nstruct sum_node : public task {\nprivate:\n using final_sum_type = final_sum;\npublic:\n final_sum_type *m_incoming;\n final_sum_type *m_body;\n Body *m_stuff_last;\nprivate:\n final_sum_type *m_left_sum;\n sum_node *m_left;\n sum_node *m_right;\n bool m_left_is_final;\n Range m_range;\n wait_context& m_wait_context;\n sum_node* m_parent;\n small_object_allocator m_allocator;\npublic:\n std::atomic ref_count{0};\n sum_node( const Range range, bool left_is_final_, sum_node* parent, wait_context& w_o, small_object_allocator& alloc ) :\n m_stuff_last(nullptr),\n m_left_sum(nullptr),\n m_left(nullptr),\n m_right(nullptr),\n m_left_is_final(left_is_final_),\n m_range(range),\n m_wait_context(w_o),\n m_parent(parent),\n m_allocator(alloc)\n {\n if( m_parent )\n m_parent->ref_count.fetch_add(1);\n // Poison fields that will be set by second pass.\n poison_pointer(m_body);\n poison_pointer(m_incoming);\n }\n\n ~sum_node() {\n if (m_parent)\n m_parent->ref_count.fetch_sub(1);\n }\nprivate:\n sum_node* release_parent() {\n call_itt_task_notify(releasing, m_parent);\n if (m_parent) {\n auto parent = m_parent;\n m_parent = nullptr;\n if (parent->ref_count.fetch_sub(1) == 1) {\n return parent;\n }\n }\n else\n m_wait_context.release();\n return nullptr;\n }\n task* create_child( const Range& range, final_sum_type& body, sum_node* child, final_sum_type* incoming, Body* stuff_last ) {\n if( child ) {\n __TBB_ASSERT( is_poisoned(child->m_body) && is_poisoned(child->m_incoming), nullptr );\n child->prepare_for_execution(body, incoming, stuff_last);\n return child;\n } else {\n body.finish_construction(this, range, stuff_last);\n return &body;\n }\n }\n\n sum_node* finalize(const execution_data& ed) {\n sum_node* next_task = release_parent();\n m_allocator.delete_object(this, ed);\n return next_task;\n }\n\npublic:\n void prepare_for_execution(final_sum_type& body, final_sum_type* incoming, Body *stuff_last) {\n this->m_body = &body;\n this->m_incoming = incoming;\n this->m_stuff_last = stuff_last;\n }\n task* execute(execution_data& ed) override {\n if( m_body ) {\n if( m_incoming )\n m_left_sum->reverse_join( *m_incoming );\n task* right_child = this->create_child(Range(m_range,split()), *m_left_sum, m_right, m_left_sum, m_stuff_last);\n task* left_child = m_left_is_final ? nullptr : this->create_child(m_range, *m_body, m_left, m_incoming, nullptr);\n ref_count = (left_child != nullptr) + (right_child != nullptr);\n m_body = nullptr;\n if( left_child ) {\n spawn(*right_child, *ed.context);\n return left_child;\n } else {\n return right_child;\n }\n } else {\n return finalize(ed);\n }\n }\n task* cancel(execution_data& ed) override {\n return finalize(ed);\n }\n void self_destroy(const execution_data& ed) {\n m_allocator.delete_object(this, ed);\n }\n template\n friend struct start_scan;\n\n template\n friend struct finish_scan;\n};\n\n//! Combine partial results\n/** @ingroup algorithms */\ntemplate\nstruct finish_scan : public task {\nprivate:\n using sum_node_type = sum_node;\n using final_sum_type = final_sum;\n final_sum_type** const m_sum_slot;\n sum_node_type*& m_return_slot;\n small_object_allocator m_allocator;\npublic:\n std::atomic m_right_zombie;\n sum_node_type& m_result;\n std::atomic ref_count{2};\n finish_scan* m_parent;\n wait_context& m_wait_context;\n task* execute(execution_data& ed) override {\n __TBB_ASSERT( m_result.ref_count.load() == static_cast((m_result.m_left!=nullptr)+(m_result.m_right!=nullptr)), nullptr );\n if( m_result.m_left )\n m_result.m_left_is_final = false;\n final_sum_type* right_zombie = m_right_zombie.load(std::memory_order_acquire);\n if( right_zombie && m_sum_slot )\n (*m_sum_slot)->reverse_join(*m_result.m_left_sum);\n __TBB_ASSERT( !m_return_slot, nullptr );\n if( right_zombie || m_result.m_right ) {\n m_return_slot = &m_result;\n } else {\n m_result.self_destroy(ed);\n }\n if( right_zombie && !m_sum_slot && !m_result.m_right ) {\n right_zombie->self_destroy(ed);\n m_right_zombie.store(nullptr, std::memory_order_relaxed);\n }\n return finalize(ed);\n }\n task* cancel(execution_data& ed) override {\n return finalize(ed);\n }\n finish_scan(sum_node_type*& return_slot, final_sum_type** sum, sum_node_type& result_, finish_scan* parent, wait_context& w_o, small_object_allocator& alloc) :\n m_sum_slot(sum),\n m_return_slot(return_slot),\n m_allocator(alloc),\n m_right_zombie(nullptr),\n m_result(result_),\n m_parent(parent),\n m_wait_context(w_o)\n {\n __TBB_ASSERT( !m_return_slot, nullptr );\n }\nprivate:\n finish_scan* release_parent() {\n call_itt_task_notify(releasing, m_parent);\n if (m_parent) {\n auto parent = m_parent;\n m_parent = nullptr;\n if (parent->ref_count.fetch_sub(1) == 1) {\n return parent;\n }\n }\n else\n m_wait_context.release();\n return nullptr;\n }\n finish_scan* finalize(const execution_data& ed) {\n finish_scan* next_task = release_parent();\n m_allocator.delete_object(this, ed);\n return next_task;\n }\n};\n\n//! Initial task to split the work\n/** @ingroup algorithms */\ntemplate\nstruct start_scan : public task {\nprivate:\n using sum_node_type = sum_node;\n using final_sum_type = final_sum;\n using finish_pass1_type = finish_scan;\n std::reference_wrapper m_return_slot;\n Range m_range;\n std::reference_wrapper m_body;\n typename Partitioner::partition_type m_partition;\n /** Non-null if caller is requesting total. */\n final_sum_type** m_sum_slot;\n bool m_is_final;\n bool m_is_right_child;\n\n finish_pass1_type* m_parent;\n small_object_allocator m_allocator;\n wait_context& m_wait_context;\n\n finish_pass1_type* release_parent() {\n call_itt_task_notify(releasing, m_parent);\n if (m_parent) {\n auto parent = m_parent;\n m_parent = nullptr;\n if (parent->ref_count.fetch_sub(1) == 1) {\n return parent;\n }\n }\n else\n m_wait_context.release();\n return nullptr;\n }\n\n finish_pass1_type* finalize( const execution_data& ed ) {\n finish_pass1_type* next_task = release_parent();\n m_allocator.delete_object(this, ed);\n return next_task;\n }\n\npublic:\n task* execute( execution_data& ) override;\n task* cancel( execution_data& ed ) override {\n return finalize(ed);\n }\n start_scan( sum_node_type*& return_slot, start_scan& parent, small_object_allocator& alloc ) :\n m_return_slot(return_slot),\n m_range(parent.m_range,split()),\n m_body(parent.m_body),\n m_partition(parent.m_partition,split()),\n m_sum_slot(parent.m_sum_slot),\n m_is_final(parent.m_is_final),\n m_is_right_child(true),\n m_parent(parent.m_parent),\n m_allocator(alloc),\n m_wait_context(parent.m_wait_context)\n {\n __TBB_ASSERT( !m_return_slot, nullptr );\n parent.m_is_right_child = false;\n }\n\n start_scan( sum_node_type*& return_slot, const Range& range, final_sum_type& body, const Partitioner& partitioner, wait_context& w_o, small_object_allocator& alloc ) :\n m_return_slot(return_slot),\n m_range(range),\n m_body(body),\n m_partition(partitioner),\n m_sum_slot(nullptr),\n m_is_final(true),\n m_is_right_child(false),\n m_parent(nullptr),\n m_allocator(alloc),\n m_wait_context(w_o)\n {\n __TBB_ASSERT( !m_return_slot, nullptr );\n }\n\n static void run( const Range& range, Body& body, const Partitioner& partitioner ) {\n if( !range.empty() ) {\n task_group_context context(PARALLEL_SCAN);\n\n using start_pass1_type = start_scan;\n sum_node_type* root = nullptr;\n wait_context w_ctx{1};\n small_object_allocator alloc{};\n\n auto& temp_body = *alloc.new_object(body, w_ctx, alloc);\n temp_body.reverse_join(body);\n\n auto& pass1 = *alloc.new_object(/*m_return_slot=*/root, range, temp_body, partitioner, w_ctx, alloc);\n\n execute_and_wait(pass1, context, w_ctx, context);\n if( root ) {\n root->prepare_for_execution(temp_body, nullptr, &body);\n w_ctx.reserve();\n execute_and_wait(*root, context, w_ctx, context);\n } else {\n temp_body.assign_to(body);\n temp_body.finish_construction(nullptr, range, nullptr);\n alloc.delete_object(&temp_body);\n }\n }\n }\n};\n\ntemplate\ntask* start_scan::execute( execution_data& ed ) {\n // Inspecting m_parent->result.left_sum would ordinarily be a race condition.\n // But we inspect it only if we are not a stolen task, in which case we\n // know that task assigning to m_parent->result.left_sum has completed.\n __TBB_ASSERT(!m_is_right_child || m_parent, \"right child is never an orphan\");\n bool treat_as_stolen = m_is_right_child && (is_stolen(ed) || &m_body.get()!=m_parent->m_result.m_left_sum);\n if( treat_as_stolen ) {\n // Invocation is for right child that has been really stolen or needs to be virtually stolen\n small_object_allocator alloc{};\n final_sum_type* right_zombie = alloc.new_object(m_body, alloc);\n m_parent->m_right_zombie.store(right_zombie, std::memory_order_release);\n m_body = *right_zombie;\n m_is_final = false;\n }\n task* next_task = nullptr;\n if( (m_is_right_child && !treat_as_stolen) || !m_range.is_divisible() || m_partition.should_execute_range(ed) ) {\n if( m_is_final )\n m_body(m_range, final_scan_tag());\n else if( m_sum_slot )\n m_body(m_range, pre_scan_tag());\n if( m_sum_slot )\n *m_sum_slot = &m_body.get();\n __TBB_ASSERT( !m_return_slot, nullptr );\n\n next_task = finalize(ed);\n } else {\n small_object_allocator alloc{};\n auto result = alloc.new_object(m_range,/*m_left_is_final=*/m_is_final, m_parent? &m_parent->m_result: nullptr, m_wait_context, alloc);\n\n auto new_parent = alloc.new_object(m_return_slot, m_sum_slot, *result, m_parent, m_wait_context, alloc);\n m_parent = new_parent;\n\n // Split off right child\n auto& right_child = *alloc.new_object(/*m_return_slot=*/result->m_right, *this, alloc);\n\n spawn(right_child, *ed.context);\n\n m_sum_slot = &result->m_left_sum;\n m_return_slot = result->m_left;\n\n __TBB_ASSERT( !m_return_slot, nullptr );\n next_task = this;\n }\n return next_task;\n}\n\ntemplate\nclass lambda_scan_body {\n Value m_sum_slot;\n const Value& identity_element;\n const Scan& m_scan;\n const ReverseJoin& m_reverse_join;\npublic:\n void operator=(const lambda_scan_body&) = delete;\n lambda_scan_body(const lambda_scan_body&) = default;\n\n lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join )\n : m_sum_slot(identity)\n , identity_element(identity)\n , m_scan(scan)\n , m_reverse_join(rev_join) {}\n\n lambda_scan_body( lambda_scan_body& b, split )\n : m_sum_slot(b.identity_element)\n , identity_element(b.identity_element)\n , m_scan(b.m_scan)\n , m_reverse_join(b.m_reverse_join) {}\n\n template\n void operator()( const Range& r, Tag tag ) {\n m_sum_slot = tbb::detail::invoke(m_scan, r, m_sum_slot, tag);\n }\n\n void reverse_join( lambda_scan_body& a ) {\n m_sum_slot = tbb::detail::invoke(m_reverse_join, a.m_sum_slot, m_sum_slot);\n }\n\n void assign( lambda_scan_body& b ) {\n m_sum_slot = b.m_sum_slot;\n }\n\n Value result() const {\n return m_sum_slot;\n }\n};\n\n// Requirements on Range concept are documented in blocked_range.h\n\n/** \\page parallel_scan_body_req Requirements on parallel_scan body\n Class \\c Body implementing the concept of parallel_scan body must define:\n - \\code Body::Body( Body&, split ); \\endcode Splitting constructor.\n Split \\c b so that \\c this and \\c b can accumulate separately\n - \\code Body::~Body(); \\endcode Destructor\n - \\code void Body::operator()( const Range& r, pre_scan_tag ); \\endcode\n Preprocess iterations for range \\c r\n - \\code void Body::operator()( const Range& r, final_scan_tag ); \\endcode\n Do final processing for iterations of range \\c r\n - \\code void Body::reverse_join( Body& a ); \\endcode\n Merge preprocessing state of \\c a into \\c this, where \\c a was\n created earlier from \\c b by b's splitting constructor\n**/\n\n/** \\name parallel_scan\n See also requirements on \\ref range_req \"Range\" and \\ref parallel_scan_body_req \"parallel_scan Body\". **/\n//@{\n\n//! Parallel prefix with default partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_body)\nvoid parallel_scan( const Range& range, Body& body ) {\n start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER());\n}\n\n//! Parallel prefix with simple_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_body)\nvoid parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) {\n start_scan::run(range, body, partitioner);\n}\n\n//! Parallel prefix with auto_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_body)\nvoid parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) {\n start_scan::run(range, body, partitioner);\n}\n\n//! Parallel prefix with default partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_function &&\n parallel_scan_combine)\nValue parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) {\n lambda_scan_body body(identity, scan, reverse_join);\n parallel_scan(range, body, __TBB_DEFAULT_PARTITIONER());\n return body.result();\n}\n\n//! Parallel prefix with simple_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_function &&\n parallel_scan_combine)\nValue parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join,\n const simple_partitioner& partitioner ) {\n lambda_scan_body body(identity, scan, reverse_join);\n parallel_scan(range, body, partitioner);\n return body.result();\n}\n\n//! Parallel prefix with auto_partitioner\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(tbb_range && parallel_scan_function &&\n parallel_scan_combine)\nValue parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join,\n const auto_partitioner& partitioner ) {\n lambda_scan_body body(identity, scan, reverse_join);\n parallel_scan(range, body, partitioner);\n return body.result();\n}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n using detail::d1::parallel_scan;\n using detail::d1::pre_scan_tag;\n using detail::d1::final_scan_tag;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_parallel_scan_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/parallel_sort.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_parallel_sort_H\n#define __TBB_parallel_sort_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"parallel_for.h\"\n#include \"blocked_range.h\"\n#include \"profiling.h\"\n\n#include \n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\n#if __TBB_CPP20_CONCEPTS_PRESENT\ninline namespace d0 {\n\n// TODO: consider using std::strict_weak_order concept\ntemplate \nconcept compare = requires( const std::remove_reference_t& comp, typename std::iterator_traits::reference value ) {\n // Forward via iterator_traits::reference\n { comp(typename std::iterator_traits::reference(value),\n typename std::iterator_traits::reference(value)) } -> std::convertible_to;\n};\n\n// Inspired by std::__PartiallyOrderedWith exposition only concept\ntemplate \nconcept less_than_comparable = requires( const std::remove_reference_t& lhs,\n const std::remove_reference_t& rhs ) {\n { lhs < rhs } -> boolean_testable;\n};\n\n} // namespace d0\n#endif // __TBB_CPP20_CONCEPTS_PRESENT\nnamespace d1 {\n\n//! Range used in quicksort to split elements into subranges based on a value.\n/** The split operation selects a splitter and places all elements less than or equal\n to the value in the first range and the remaining elements in the second range.\n @ingroup algorithms */\ntemplate\nclass quick_sort_range {\n std::size_t median_of_three( const RandomAccessIterator& array, std::size_t l, std::size_t m, std::size_t r ) const {\n return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp(array[l], array[r]) ? r : l ) )\n : ( comp(array[r], array[m]) ? m : ( comp(array[r], array[l]) ? r : l ) );\n }\n\n std::size_t pseudo_median_of_nine( const RandomAccessIterator& array, const quick_sort_range& range ) const {\n std::size_t offset = range.size / 8u;\n return median_of_three(array,\n median_of_three(array, 0 , offset, offset * 2),\n median_of_three(array, offset * 3, offset * 4, offset * 5),\n median_of_three(array, offset * 6, offset * 7, range.size - 1));\n\n }\n\n std::size_t split_range( quick_sort_range& range ) {\n RandomAccessIterator array = range.begin;\n RandomAccessIterator first_element = range.begin;\n std::size_t m = pseudo_median_of_nine(array, range);\n if( m != 0 ) std::iter_swap(array, array + m);\n\n std::size_t i = 0;\n std::size_t j = range.size;\n // Partition interval [i + 1,j - 1] with key *first_element.\n for(;;) {\n __TBB_ASSERT( i < j, nullptr );\n // Loop must terminate since array[l] == *first_element.\n do {\n --j;\n __TBB_ASSERT( i <= j, \"bad ordering relation?\" );\n } while( comp(*first_element, array[j]) );\n do {\n __TBB_ASSERT( i <= j, nullptr );\n if( i == j ) goto partition;\n ++i;\n } while( comp(array[i], *first_element) );\n if( i == j ) goto partition;\n std::iter_swap(array + i, array + j);\n }\npartition:\n // Put the partition key were it belongs\n std::iter_swap(array + j, first_element);\n // array[l..j) is less or equal to key.\n // array(j..r) is greater or equal to key.\n // array[j] is equal to key\n i = j + 1;\n std::size_t new_range_size = range.size - i;\n range.size = j;\n return new_range_size;\n }\n\npublic:\n quick_sort_range() = default;\n quick_sort_range( const quick_sort_range& ) = default;\n void operator=( const quick_sort_range& ) = delete;\n\n static constexpr std::size_t grainsize = 500;\n const Compare& comp;\n std::size_t size;\n RandomAccessIterator begin;\n\n quick_sort_range( RandomAccessIterator begin_, std::size_t size_, const Compare& comp_ ) :\n comp(comp_), size(size_), begin(begin_) {}\n\n bool empty() const { return size == 0; }\n bool is_divisible() const { return size >= grainsize; }\n\n quick_sort_range( quick_sort_range& range, split )\n : comp(range.comp)\n , size(split_range(range))\n // +1 accounts for the pivot element, which is at its correct place\n // already and, therefore, is not included into subranges.\n , begin(range.begin + range.size + 1) {}\n};\n\n//! Body class used to test if elements in a range are presorted\n/** @ingroup algorithms */\ntemplate\nclass quick_sort_pretest_body {\n const Compare& comp;\n task_group_context& context;\n\npublic:\n quick_sort_pretest_body() = default;\n quick_sort_pretest_body( const quick_sort_pretest_body& ) = default;\n void operator=( const quick_sort_pretest_body& ) = delete;\n\n quick_sort_pretest_body( const Compare& _comp, task_group_context& _context ) : comp(_comp), context(_context) {}\n\n void operator()( const blocked_range& range ) const {\n RandomAccessIterator my_end = range.end();\n\n int i = 0;\n //TODO: consider using std::is_sorted() for each 64 iterations (requires performance measurements)\n for( RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i ) {\n if( i % 64 == 0 && context.is_group_execution_cancelled() ) break;\n\n // The k - 1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1\n if( comp(*(k), *(k - 1)) ) {\n context.cancel_group_execution();\n break;\n }\n }\n }\n};\n\n//! Body class used to sort elements in a range that is smaller than the grainsize.\n/** @ingroup algorithms */\ntemplate\nstruct quick_sort_body {\n void operator()( const quick_sort_range& range ) const {\n std::sort(range.begin, range.begin + range.size, range.comp);\n }\n};\n\n//! Method to perform parallel_for based quick sort.\n/** @ingroup algorithms */\ntemplate\nvoid do_parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) {\n parallel_for(quick_sort_range(begin, end - begin, comp),\n quick_sort_body(),\n auto_partitioner());\n}\n\n//! Wrapper method to initiate the sort by calling parallel_for.\n/** @ingroup algorithms */\ntemplate\nvoid parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) {\n task_group_context my_context(PARALLEL_SORT);\n constexpr int serial_cutoff = 9;\n\n __TBB_ASSERT( begin + serial_cutoff < end, \"min_parallel_size is smaller than serial cutoff?\" );\n RandomAccessIterator k = begin;\n for( ; k != begin + serial_cutoff; ++k ) {\n if( comp(*(k + 1), *k) ) {\n do_parallel_quick_sort(begin, end, comp);\n return;\n }\n }\n\n // Check is input range already sorted\n parallel_for(blocked_range(k + 1, end),\n quick_sort_pretest_body(comp, my_context),\n auto_partitioner(),\n my_context);\n\n if( my_context.is_group_execution_cancelled() )\n do_parallel_quick_sort(begin, end, comp);\n}\n\n/** \\page parallel_sort_iter_req Requirements on iterators for parallel_sort\n Requirements on the iterator type \\c It and its value type \\c T for \\c parallel_sort:\n\n - \\code void iter_swap( It a, It b ) \\endcode Swaps the values of the elements the given\n iterators \\c a and \\c b are pointing to. \\c It should be a random access iterator.\n\n - \\code bool Compare::operator()( const T& x, const T& y ) \\endcode True if x comes before y;\n**/\n\n/** \\name parallel_sort\n See also requirements on \\ref parallel_sort_iter_req \"iterators for parallel_sort\". **/\n//@{\n\n#if __TBB_CPP20_CONCEPTS_PRESENT\ntemplate\nusing iter_value_type = typename std::iterator_traits::value_type;\n\ntemplate\nusing range_value_type = typename std::iterator_traits>::value_type;\n#endif\n\n//! Sorts the data in [begin,end) using the given comparator\n/** The compare function object is used for all comparisons between elements during sorting.\n The compare object must define a bool operator() function.\n @ingroup algorithms **/\ntemplate\n __TBB_requires(std::random_access_iterator &&\n compare &&\n std::movable>)\nvoid parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) {\n constexpr int min_parallel_size = 500;\n if( end > begin ) {\n if( end - begin < min_parallel_size ) {\n std::sort(begin, end, comp);\n } else {\n parallel_quick_sort(begin, end, comp);\n }\n }\n}\n\n//! Sorts the data in [begin,end) with a default comparator \\c std::less\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(std::random_access_iterator &&\n less_than_comparable> &&\n std::movable>)\nvoid parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) {\n parallel_sort(begin, end, std::less::value_type>());\n}\n\n//! Sorts the data in rng using the given comparator\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(container_based_sequence &&\n compare> &&\n std::movable>)\nvoid parallel_sort( Range&& rng, const Compare& comp ) {\n parallel_sort(std::begin(rng), std::end(rng), comp);\n}\n\n//! Sorts the data in rng with a default comparator \\c std::less\n/** @ingroup algorithms **/\ntemplate\n __TBB_requires(container_based_sequence &&\n less_than_comparable> &&\n std::movable>)\nvoid parallel_sort( Range&& rng ) {\n parallel_sort(std::begin(rng), std::end(rng));\n}\n//@}\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n using detail::d1::parallel_sort;\n} // namespace v1\n} // namespace tbb\n\n#endif /*__TBB_parallel_sort_H*/\n" }, { "path": "src/tbb/include/oneapi/tbb/partitioner.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_partitioner_H\n#define __TBB_partitioner_H\n\n#ifndef __TBB_INITIAL_CHUNKS\n// initial task divisions per thread\n#define __TBB_INITIAL_CHUNKS 2\n#endif\n#ifndef __TBB_RANGE_POOL_CAPACITY\n// maximum number of elements in range pool\n#define __TBB_RANGE_POOL_CAPACITY 8\n#endif\n#ifndef __TBB_INIT_DEPTH\n// initial value for depth of range pool\n#define __TBB_INIT_DEPTH 5\n#endif\n#ifndef __TBB_DEMAND_DEPTH_ADD\n// when imbalance is found range splits this value times more\n#define __TBB_DEMAND_DEPTH_ADD 1\n#endif\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_aligned_space.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_range_common.h\"\n#include \"detail/_task.h\"\n#include \"detail/_small_object_pool.h\"\n\n#include \"cache_aligned_allocator.h\"\n#include \"task_group.h\" // task_group_context\n#include \"task_arena.h\"\n\n#include \n#include \n#include \n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Workaround for overzealous compiler warnings\n // #pragma warning (push)\n // #pragma warning (disable: 4244)\n#endif\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass auto_partitioner;\nclass simple_partitioner;\nclass static_partitioner;\nclass affinity_partitioner;\nclass affinity_partition_type;\nclass affinity_partitioner_base;\n\ninline std::size_t get_initial_auto_partitioner_divisor() {\n const std::size_t factor = 4;\n return factor * static_cast(max_concurrency());\n}\n\n//! Defines entry point for affinity partitioner into oneTBB run-time library.\nclass affinity_partitioner_base: no_copy {\n friend class affinity_partitioner;\n friend class affinity_partition_type;\n //! Array that remembers affinities of tree positions to affinity_id.\n /** nullptr if my_size==0. */\n slot_id* my_array;\n //! Number of elements in my_array.\n std::size_t my_size;\n //! Zeros the fields.\n affinity_partitioner_base() : my_array(nullptr), my_size(0) {}\n //! Deallocates my_array.\n ~affinity_partitioner_base() { resize(0); }\n //! Resize my_array.\n /** Retains values if resulting size is the same. */\n void resize(unsigned factor) {\n // Check factor to avoid asking for number of workers while there might be no arena.\n unsigned max_threads_in_arena = static_cast(max_concurrency());\n std::size_t new_size = factor ? factor * max_threads_in_arena : 0;\n if (new_size != my_size) {\n if (my_array) {\n r1::cache_aligned_deallocate(my_array);\n // Following two assignments must be done here for sake of exception safety.\n my_array = nullptr;\n my_size = 0;\n }\n if (new_size) {\n my_array = static_cast(r1::cache_aligned_allocate(new_size * sizeof(slot_id)));\n std::fill_n(my_array, new_size, no_slot);\n my_size = new_size;\n }\n }\n }\n};\n\ntemplate struct start_for;\ntemplate struct start_scan;\ntemplate struct start_reduce;\ntemplate struct start_deterministic_reduce;\n\nstruct node {\n node* my_parent{};\n std::atomic m_ref_count{};\n\n node() = default;\n node(node* parent, int ref_count) :\n my_parent{parent}, m_ref_count{ref_count} {\n __TBB_ASSERT(ref_count > 0, \"The ref count must be positive\");\n }\n};\n\nstruct wait_node : node {\n wait_node() : node{ nullptr, 1 } {}\n wait_context m_wait{1};\n};\n\n//! Join task node that contains shared flag for stealing feedback\nstruct tree_node : public node {\n small_object_allocator m_allocator;\n std::atomic m_child_stolen{false};\n\n tree_node(node* parent, int ref_count, small_object_allocator& alloc)\n : node{parent, ref_count}\n , m_allocator{alloc} {}\n\n void join(task_group_context*) {/*dummy, required only for reduction algorithms*/};\n\n template \n static void mark_task_stolen(Task &t) {\n std::atomic &flag = static_cast(t.my_parent)->m_child_stolen;\n#if TBB_USE_PROFILING_TOOLS\n // Threading tools respect lock prefix but report false-positive data-race via plain store\n flag.exchange(true);\n#else\n flag.store(true, std::memory_order_relaxed);\n#endif // TBB_USE_PROFILING_TOOLS\n }\n template \n static bool is_peer_stolen(Task &t) {\n return static_cast(t.my_parent)->m_child_stolen.load(std::memory_order_relaxed);\n }\n};\n\n// Context used to check cancellation state during reduction join process\ntemplate\nvoid fold_tree(node* n, const execution_data& ed) {\n for (;;) {\n __TBB_ASSERT(n, nullptr);\n __TBB_ASSERT(n->m_ref_count.load(std::memory_order_relaxed) > 0, \"The refcount must be positive.\");\n call_itt_task_notify(releasing, n);\n if (--n->m_ref_count > 0) {\n return;\n }\n node* parent = n->my_parent;\n if (!parent) {\n break;\n };\n\n call_itt_task_notify(acquired, n);\n TreeNodeType* self = static_cast(n);\n self->join(ed.context);\n self->m_allocator.delete_object(self, ed);\n n = parent;\n }\n // Finish parallel for execution when the root (last node) is reached\n static_cast(n)->m_wait.release();\n}\n\n//! Depth is a relative depth of recursive division inside a range pool. Relative depth allows\n//! infinite absolute depth of the recursion for heavily unbalanced workloads with range represented\n//! by a number that cannot fit into machine word.\ntypedef unsigned char depth_t;\n\n//! Range pool stores ranges of type T in a circular buffer with MaxCapacity\ntemplate \nclass range_vector {\n depth_t my_head;\n depth_t my_tail;\n depth_t my_size;\n depth_t my_depth[MaxCapacity]; // relative depths of stored ranges\n tbb::detail::aligned_space my_pool;\n\npublic:\n //! initialize via first range in pool\n range_vector(const T& elem) : my_head(0), my_tail(0), my_size(1) {\n my_depth[0] = 0;\n new( static_cast(my_pool.begin()) ) T(elem);//TODO: std::move?\n }\n ~range_vector() {\n while( !empty() ) pop_back();\n }\n bool empty() const { return my_size == 0; }\n depth_t size() const { return my_size; }\n //! Populates range pool via ranges up to max depth or while divisible\n //! max_depth starts from 0, e.g. value 2 makes 3 ranges in the pool up to two 1/4 pieces\n void split_to_fill(depth_t max_depth) {\n while( my_size < MaxCapacity && is_divisible(max_depth) ) {\n depth_t prev = my_head;\n my_head = (my_head + 1) % MaxCapacity;\n new(my_pool.begin()+my_head) T(my_pool.begin()[prev]); // copy TODO: std::move?\n my_pool.begin()[prev].~T(); // instead of assignment\n new(my_pool.begin()+prev) T(my_pool.begin()[my_head], detail::split()); // do 'inverse' split\n my_depth[my_head] = ++my_depth[prev];\n my_size++;\n }\n }\n void pop_back() {\n __TBB_ASSERT(my_size > 0, \"range_vector::pop_back() with empty size\");\n my_pool.begin()[my_head].~T();\n my_size--;\n my_head = (my_head + MaxCapacity - 1) % MaxCapacity;\n }\n void pop_front() {\n __TBB_ASSERT(my_size > 0, \"range_vector::pop_front() with empty size\");\n my_pool.begin()[my_tail].~T();\n my_size--;\n my_tail = (my_tail + 1) % MaxCapacity;\n }\n T& back() {\n __TBB_ASSERT(my_size > 0, \"range_vector::back() with empty size\");\n return my_pool.begin()[my_head];\n }\n T& front() {\n __TBB_ASSERT(my_size > 0, \"range_vector::front() with empty size\");\n return my_pool.begin()[my_tail];\n }\n //! similarly to front(), returns depth of the first range in the pool\n depth_t front_depth() {\n __TBB_ASSERT(my_size > 0, \"range_vector::front_depth() with empty size\");\n return my_depth[my_tail];\n }\n depth_t back_depth() {\n __TBB_ASSERT(my_size > 0, \"range_vector::back_depth() with empty size\");\n return my_depth[my_head];\n }\n bool is_divisible(depth_t max_depth) {\n return back_depth() < max_depth && back().is_divisible();\n }\n};\n\n//! Provides default methods for partition objects and common algorithm blocks.\ntemplate \nstruct partition_type_base {\n typedef detail::split split_type;\n // decision makers\n void note_affinity( slot_id ) {}\n template \n bool check_being_stolen(Task&, const execution_data&) { return false; } // part of old should_execute_range()\n template split_type get_split() { return split(); }\n Partition& self() { return *static_cast(this); } // CRTP helper\n\n template\n void work_balance(StartType &start, Range &range, const execution_data&) {\n start.run_body( range ); // static partitioner goes here\n }\n\n template\n void execute(StartType &start, Range &range, execution_data& ed) {\n // The algorithm in a few words ([]-denotes calls to decision methods of partitioner):\n // [If this task is stolen, adjust depth and divisions if necessary, set flag].\n // If range is divisible {\n // Spread the work while [initial divisions left];\n // Create trap task [if necessary];\n // }\n // If not divisible or [max depth is reached], execute, else do the range pool part\n if ( range.is_divisible() ) {\n if ( self().is_divisible() ) {\n do { // split until is divisible\n typename Partition::split_type split_obj = self().template get_split();\n start.offer_work( split_obj, ed );\n } while ( range.is_divisible() && self().is_divisible() );\n }\n }\n self().work_balance(start, range, ed);\n }\n};\n\n//! Provides default splitting strategy for partition objects.\ntemplate \nstruct adaptive_mode : partition_type_base {\n typedef Partition my_partition;\n std::size_t my_divisor;\n // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves.\n // A task which has only one index must produce the right split without reserved index in order to avoid\n // it to be overwritten in note_affinity() of the created (right) task.\n // I.e. a task created deeper than the affinity array can remember must not save its affinity (LIFO order)\n static const unsigned factor = 1;\n adaptive_mode() : my_divisor(get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {}\n adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {}\n adaptive_mode(adaptive_mode&, const proportional_split&) : my_divisor(0)\n {\n // left blank as my_divisor gets overridden in the successors' constructors\n }\n /*! Override do_split methods in order to specify splitting strategy */\n std::size_t do_split(adaptive_mode &src, split) {\n return src.my_divisor /= 2u;\n }\n};\n\n\n//! Provides proportional splitting strategy for partition objects\ntemplate \nstruct proportional_mode : adaptive_mode {\n typedef Partition my_partition;\n using partition_type_base::self; // CRTP helper to get access to derived classes\n\n proportional_mode() : adaptive_mode() {}\n proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {}\n proportional_mode(proportional_mode &src, const proportional_split& split_obj)\n : adaptive_mode(src, split_obj)\n {\n self().my_divisor = do_split(src, split_obj);\n }\n std::size_t do_split(proportional_mode &src, const proportional_split& split_obj) {\n std::size_t portion = split_obj.right() * my_partition::factor;\n portion = (portion + my_partition::factor/2) & (0ul - my_partition::factor);\n src.my_divisor -= portion;\n return portion;\n }\n bool is_divisible() { // part of old should_execute_range()\n return self().my_divisor > my_partition::factor;\n }\n template \n proportional_split get_split() {\n // Create the proportion from partitioner internal resources (threads) that would be used:\n // - into proportional_mode constructor to split the partitioner\n // - if Range supports the proportional_split constructor it would use proposed proportion,\n // otherwise, the tbb::proportional_split object will be implicitly (for Range implementer)\n // casted to tbb::split\n\n std::size_t n = self().my_divisor / my_partition::factor;\n std::size_t right = n / 2;\n std::size_t left = n - right;\n return proportional_split(left, right);\n }\n};\n\nstatic std::size_t get_initial_partition_head() {\n int current_index = tbb::this_task_arena::current_thread_index();\n if (current_index == tbb::task_arena::not_initialized)\n current_index = 0;\n return size_t(current_index);\n}\n\n//! Provides default linear indexing of partitioner's sequence\ntemplate \nstruct linear_affinity_mode : proportional_mode {\n std::size_t my_head;\n std::size_t my_max_affinity;\n using proportional_mode::self;\n linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()),\n my_max_affinity(self().my_divisor) {}\n linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split())\n , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {}\n linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj)\n , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {}\n void spawn_task(task& t, task_group_context& ctx) {\n if (self().my_divisor) {\n spawn(t, ctx, slot_id(my_head));\n } else {\n spawn(t, ctx);\n }\n }\n};\n\nstatic bool is_stolen_task(const execution_data& ed) {\n return execution_slot(ed) != original_slot(ed);\n}\n\n/*! Determine work-balance phase implementing splitting & stealing actions */\ntemplate\nstruct dynamic_grainsize_mode : Mode {\n using Mode::self;\n enum {\n begin = 0,\n run,\n pass\n } my_delay;\n depth_t my_max_depth;\n static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY;\n dynamic_grainsize_mode(): Mode()\n , my_delay(begin)\n , my_max_depth(__TBB_INIT_DEPTH) {}\n dynamic_grainsize_mode(dynamic_grainsize_mode& p, split)\n : Mode(p, split())\n , my_delay(pass)\n , my_max_depth(p.my_max_depth) {}\n dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj)\n : Mode(p, split_obj)\n , my_delay(begin)\n , my_max_depth(p.my_max_depth) {}\n template \n bool check_being_stolen(Task &t, const execution_data& ed) { // part of old should_execute_range()\n if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree\n self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)?\n if( is_stolen_task(ed) && t.my_parent->m_ref_count >= 2 ) { // runs concurrently with the left task\n#if __TBB_USE_OPTIONAL_RTTI\n // RTTI is available, check whether the cast is valid\n // TODO: TBB_REVAMP_TODO __TBB_ASSERT(dynamic_cast(t.m_parent), 0);\n // correctness of the cast relies on avoiding the root task for which:\n // - initial value of my_divisor != 0 (protected by separate assertion)\n // - is_stolen_task() always returns false for the root task.\n#endif\n tree_node::mark_task_stolen(t);\n if( !my_max_depth ) my_max_depth++;\n my_max_depth += __TBB_DEMAND_DEPTH_ADD;\n return true;\n }\n }\n return false;\n }\n depth_t max_depth() { return my_max_depth; }\n void align_depth(depth_t base) {\n __TBB_ASSERT(base <= my_max_depth, nullptr);\n my_max_depth -= base;\n }\n template\n void work_balance(StartType &start, Range &range, execution_data& ed) {\n if( !range.is_divisible() || !self().max_depth() ) {\n start.run_body( range );\n }\n else { // do range pool\n range_vector range_pool(range);\n do {\n range_pool.split_to_fill(self().max_depth()); // fill range pool\n if( self().check_for_demand( start ) ) {\n if( range_pool.size() > 1 ) {\n start.offer_work( range_pool.front(), range_pool.front_depth(), ed );\n range_pool.pop_front();\n continue;\n }\n if( range_pool.is_divisible(self().max_depth()) ) // was not enough depth to fork a task\n continue; // note: next split_to_fill() should split range at least once\n }\n start.run_body( range_pool.back() );\n range_pool.pop_back();\n } while( !range_pool.empty() && !ed.context->is_group_execution_cancelled() );\n }\n }\n template \n bool check_for_demand(Task& t) {\n if ( pass == my_delay ) {\n if ( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array\n return true; // do not do my_max_depth++ here, but be sure range_pool is splittable once more\n else if ( self().my_divisor && my_max_depth ) { // make balancing task\n self().my_divisor = 0; // once for each task; depth will be decreased in align_depth()\n return true;\n }\n else if ( tree_node::is_peer_stolen(t) ) {\n my_max_depth += __TBB_DEMAND_DEPTH_ADD;\n return true;\n }\n } else if( begin == my_delay ) {\n my_delay = pass;\n }\n return false;\n }\n};\n\nclass auto_partition_type: public dynamic_grainsize_mode > {\npublic:\n auto_partition_type( const auto_partitioner& ) {\n my_divisor *= __TBB_INITIAL_CHUNKS;\n }\n auto_partition_type( auto_partition_type& src, split)\n : dynamic_grainsize_mode >(src, split()) {}\n bool is_divisible() { // part of old should_execute_range()\n if( my_divisor > 1 ) return true;\n if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead\n // keep same fragmentation while splitting for the local task pool\n my_max_depth--;\n my_divisor = 0; // decrease max_depth once per task\n return true;\n } else return false;\n }\n template \n bool check_for_demand(Task& t) {\n if (tree_node::is_peer_stolen(t)) {\n my_max_depth += __TBB_DEMAND_DEPTH_ADD;\n return true;\n } else return false;\n }\n void spawn_task(task& t, task_group_context& ctx) {\n spawn(t, ctx);\n }\n};\n\nclass simple_partition_type: public partition_type_base {\npublic:\n simple_partition_type( const simple_partitioner& ) {}\n simple_partition_type( const simple_partition_type&, split ) {}\n //! simplified algorithm\n template\n void execute(StartType &start, Range &range, execution_data& ed) {\n split_type split_obj = split(); // start.offer_work accepts split_type as reference\n while( range.is_divisible() )\n start.offer_work( split_obj, ed );\n start.run_body( range );\n }\n void spawn_task(task& t, task_group_context& ctx) {\n spawn(t, ctx);\n }\n};\n\nclass static_partition_type : public linear_affinity_mode {\npublic:\n typedef detail::proportional_split split_type;\n static_partition_type( const static_partitioner& ) {}\n static_partition_type( static_partition_type& p, const proportional_split& split_obj )\n : linear_affinity_mode(p, split_obj) {}\n};\n\nclass affinity_partition_type : public dynamic_grainsize_mode > {\n static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units\n slot_id* my_array;\npublic:\n static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task\n typedef detail::proportional_split split_type;\n affinity_partition_type( affinity_partitioner_base& ap ) {\n __TBB_ASSERT( (factor&(factor-1))==0, \"factor must be power of two\" );\n ap.resize(factor);\n my_array = ap.my_array;\n my_max_depth = factor_power + 1;\n __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, nullptr );\n }\n affinity_partition_type(affinity_partition_type& p, split)\n : dynamic_grainsize_mode >(p, split())\n , my_array(p.my_array) {}\n affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj)\n : dynamic_grainsize_mode >(p, split_obj)\n , my_array(p.my_array) {}\n void note_affinity(slot_id id) {\n if( my_divisor )\n my_array[my_head] = id;\n }\n void spawn_task(task& t, task_group_context& ctx) {\n if (my_divisor) {\n if (!my_array[my_head]) {\n // TODO: consider new ideas with my_array for both affinity and static partitioner's, then code reuse\n spawn(t, ctx, slot_id(my_head / factor));\n } else {\n spawn(t, ctx, my_array[my_head]);\n }\n } else {\n spawn(t, ctx);\n }\n }\n};\n\n//! A simple partitioner\n/** Divides the range until the range is not divisible.\n @ingroup algorithms */\nclass simple_partitioner {\npublic:\n simple_partitioner() {}\nprivate:\n template friend struct start_for;\n template friend struct start_reduce;\n template friend struct start_deterministic_reduce;\n template friend struct start_scan;\n // new implementation just extends existing interface\n typedef simple_partition_type task_partition_type;\n // TODO: consider to make split_type public\n typedef simple_partition_type::split_type split_type;\n\n // for parallel_scan only\n class partition_type {\n public:\n bool should_execute_range(const execution_data& ) {return false;}\n partition_type( const simple_partitioner& ) {}\n partition_type( const partition_type&, split ) {}\n };\n};\n\n//! An auto partitioner\n/** The range is initial divided into several large chunks.\n Chunks are further subdivided into smaller pieces if demand detected and they are divisible.\n @ingroup algorithms */\nclass auto_partitioner {\npublic:\n auto_partitioner() {}\n\nprivate:\n template friend struct start_for;\n template friend struct start_reduce;\n template friend struct start_deterministic_reduce;\n template friend struct start_scan;\n // new implementation just extends existing interface\n typedef auto_partition_type task_partition_type;\n // TODO: consider to make split_type public\n typedef auto_partition_type::split_type split_type;\n\n //! Backward-compatible partition for auto and affinity partition objects.\n class partition_type {\n size_t num_chunks;\n static const size_t VICTIM_CHUNKS = 4;\n public:\n bool should_execute_range(const execution_data& ed) {\n if( num_chunks friend struct start_for;\n template friend struct start_reduce;\n template friend struct start_deterministic_reduce;\n template friend struct start_scan;\n // new implementation just extends existing interface\n typedef static_partition_type task_partition_type;\n // TODO: consider to make split_type public\n typedef static_partition_type::split_type split_type;\n};\n\n//! An affinity partitioner\nclass affinity_partitioner : affinity_partitioner_base {\npublic:\n affinity_partitioner() {}\n\nprivate:\n template friend struct start_for;\n template friend struct start_reduce;\n template friend struct start_deterministic_reduce;\n template friend struct start_scan;\n // new implementation just extends existing interface\n typedef affinity_partition_type task_partition_type;\n // TODO: consider to make split_type public\n typedef affinity_partition_type::split_type split_type;\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n// Partitioners\nusing detail::d1::auto_partitioner;\nusing detail::d1::simple_partitioner;\nusing detail::d1::static_partitioner;\nusing detail::d1::affinity_partitioner;\n// Split types\nusing detail::split;\nusing detail::proportional_split;\n} // namespace v1\n\n} // namespace tbb\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // #pragma warning (pop)\n#endif // warning 4244 is back\n\n#undef __TBB_INITIAL_CHUNKS\n#undef __TBB_RANGE_POOL_CAPACITY\n#undef __TBB_INIT_DEPTH\n\n#endif /* __TBB_partitioner_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/profiling.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_profiling_H\n#define __TBB_profiling_H\n\n#include \"detail/_config.h\"\n#include \n\n#include \n\nnamespace tbb {\nnamespace detail {\ninline namespace d0 {\n // include list of index names\n #define TBB_STRING_RESOURCE(index_name,str) index_name,\n enum string_resource_index : std::uintptr_t {\n #include \"detail/_string_resource.h\"\n NUM_STRINGS\n };\n #undef TBB_STRING_RESOURCE\n\n enum itt_relation\n {\n __itt_relation_is_unknown = 0,\n __itt_relation_is_dependent_on, /**< \"A is dependent on B\" means that A cannot start until B completes */\n __itt_relation_is_sibling_of, /**< \"A is sibling of B\" means that A and B were created as a group */\n __itt_relation_is_parent_of, /**< \"A is parent of B\" means that A created B */\n __itt_relation_is_continuation_of, /**< \"A is continuation of B\" means that A assumes the dependencies of B */\n __itt_relation_is_child_of, /**< \"A is child of B\" means that A was created by B (inverse of is_parent_of) */\n __itt_relation_is_continued_by, /**< \"A is continued by B\" means that B assumes the dependencies of A (inverse of is_continuation_of) */\n __itt_relation_is_predecessor_to /**< \"A is predecessor to B\" means that B cannot start until A completes (inverse of is_dependent_on) */\n };\n\n//! Unicode support\n#if (_WIN32||_WIN64)\n //! Unicode character type. Always wchar_t on Windows.\n using tchar = wchar_t;\n#else /* !WIN */\n using tchar = char;\n#endif /* !WIN */\n\n} // namespace d0\n} // namespace detail\n} // namespace tbb\n\n#include \n#if _WIN32||_WIN64\n#include /* mbstowcs_s */\n#endif\n// Need these to work regardless of tools support\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n enum notify_type {prepare=0, cancel, acquired, releasing, destroy};\n enum itt_domain_enum { ITT_DOMAIN_FLOW=0, ITT_DOMAIN_MAIN=1, ITT_DOMAIN_ALGO=2, ITT_NUM_DOMAINS };\n} // namespace d1\n\nnamespace r1 {\n TBB_EXPORT void __TBB_EXPORTED_FUNC call_itt_notify(int t, void* ptr);\n TBB_EXPORT void __TBB_EXPORTED_FUNC create_itt_sync(void* ptr, const tchar* objtype, const tchar* objname);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_make_task_group(d1::itt_domain_enum domain, void* group, unsigned long long group_extra,\n void* parent, unsigned long long parent_extra, string_resource_index name_index);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_begin(d1::itt_domain_enum domain, void* task, unsigned long long task_extra,\n void* parent, unsigned long long parent_extra, string_resource_index name_index);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_end(d1::itt_domain_enum domain);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_set_sync_name(void* obj, const tchar* name);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_str_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra,\n string_resource_index key, const char* value);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_ptr_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra,\n string_resource_index key, void* value);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_relation_add(d1::itt_domain_enum domain, void* addr0, unsigned long long addr0_extra,\n itt_relation relation, void* addr1, unsigned long long addr1_extra);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_begin(d1::itt_domain_enum domain, void* region, unsigned long long region_extra,\n void* parent, unsigned long long parent_extra, string_resource_index /* name_index */);\n TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_end(d1::itt_domain_enum domain, void* region, unsigned long long region_extra);\n} // namespace r1\n\nnamespace d1 {\n#if TBB_USE_PROFILING_TOOLS && (_WIN32||_WIN64)\n inline std::size_t multibyte_to_widechar(wchar_t* wcs, const char* mbs, std::size_t bufsize) {\n std::size_t len;\n mbstowcs_s(&len, wcs, bufsize, mbs, _TRUNCATE);\n return len; // mbstowcs_s counts null terminator\n }\n#endif\n\n#if TBB_USE_PROFILING_TOOLS\n inline void create_itt_sync(void *ptr, const char *objtype, const char *objname) {\n#if (_WIN32||_WIN64)\n std::size_t len_type = multibyte_to_widechar(nullptr, objtype, 0);\n wchar_t *type = new wchar_t[len_type];\n multibyte_to_widechar(type, objtype, len_type);\n std::size_t len_name = multibyte_to_widechar(nullptr, objname, 0);\n wchar_t *name = new wchar_t[len_name];\n multibyte_to_widechar(name, objname, len_name);\n#else // WIN\n const char *type = objtype;\n const char *name = objname;\n#endif\n r1::create_itt_sync(ptr, type, name);\n\n#if (_WIN32||_WIN64)\n delete[] type;\n delete[] name;\n#endif // WIN\n }\n\n// Distinguish notifications on task for reducing overheads\n#if TBB_USE_PROFILING_TOOLS == 2\n inline void call_itt_task_notify(d1::notify_type t, void *ptr) {\n r1::call_itt_notify(static_cast(t), ptr);\n }\n#else\n inline void call_itt_task_notify(d1::notify_type, void *) {}\n#endif // TBB_USE_PROFILING_TOOLS\n\n inline void call_itt_notify(d1::notify_type t, void *ptr) {\n r1::call_itt_notify(static_cast(t), ptr);\n }\n\n#if (_WIN32||_WIN64) && !__MINGW32__\n inline void itt_set_sync_name(void* obj, const wchar_t* name) {\n r1::itt_set_sync_name(obj, name);\n }\n inline void itt_set_sync_name(void* obj, const char* name) {\n std::size_t len_name = multibyte_to_widechar(nullptr, name, 0);\n wchar_t *obj_name = new wchar_t[len_name];\n multibyte_to_widechar(obj_name, name, len_name);\n r1::itt_set_sync_name(obj, obj_name);\n delete[] obj_name;\n }\n#else\n inline void itt_set_sync_name( void* obj, const char* name) {\n r1::itt_set_sync_name(obj, name);\n }\n#endif //WIN\n\n inline void itt_make_task_group(itt_domain_enum domain, void* group, unsigned long long group_extra,\n void* parent, unsigned long long parent_extra, string_resource_index name_index) {\n r1::itt_make_task_group(domain, group, group_extra, parent, parent_extra, name_index);\n }\n\n inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra,\n string_resource_index key, const char *value ) {\n r1::itt_metadata_str_add( domain, addr, addr_extra, key, value );\n }\n\n inline void register_node_addr(itt_domain_enum domain, void *addr, unsigned long long addr_extra,\n string_resource_index key, void *value) {\n r1::itt_metadata_ptr_add(domain, addr, addr_extra, key, value);\n }\n\n inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra,\n itt_relation relation, void *addr1, unsigned long long addr1_extra ) {\n r1::itt_relation_add( domain, addr0, addr0_extra, relation, addr1, addr1_extra );\n }\n\n inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra,\n void *parent, unsigned long long parent_extra, string_resource_index name_index ) {\n r1::itt_task_begin( domain, task, task_extra, parent, parent_extra, name_index );\n }\n\n inline void itt_task_end( itt_domain_enum domain ) {\n r1::itt_task_end( domain );\n }\n\n inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra,\n void *parent, unsigned long long parent_extra, string_resource_index name_index ) {\n r1::itt_region_begin( domain, region, region_extra, parent, parent_extra, name_index );\n }\n\n inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) {\n r1::itt_region_end( domain, region, region_extra );\n }\n#else\n inline void create_itt_sync(void* /*ptr*/, const char* /*objtype*/, const char* /*objname*/) {}\n\n inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {}\n\n inline void call_itt_task_notify(notify_type /*t*/, void* /*ptr*/) {}\n#endif // TBB_USE_PROFILING_TOOLS\n\n#if TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2)\nclass event {\n/** This class supports user event traces through itt.\n Common use-case is tagging data flow graph tasks (data-id)\n and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/\n// TODO: Replace implementation by itt user event api.\n\n const std::string my_name;\n\n static void emit_trace(const std::string &input) {\n itt_metadata_str_add( ITT_DOMAIN_FLOW, nullptr, FLOW_NULL, USER_EVENT, ( \"FGA::DATAID::\" + input ).c_str() );\n }\n\npublic:\n event(const std::string &input)\n : my_name( input )\n { }\n\n void emit() {\n emit_trace(my_name);\n }\n\n static void emit(const std::string &description) {\n emit_trace(description);\n }\n\n};\n#else // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2)\n// Using empty struct if user event tracing is disabled:\nstruct event {\n event(const std::string &) { }\n\n void emit() { }\n\n static void emit(const std::string &) { }\n};\n#endif // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2)\n} // namespace d1\n} // namespace detail\n\nnamespace profiling {\n using detail::d1::event;\n}\n} // namespace tbb\n\n\n#endif /* __TBB_profiling_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/queuing_mutex.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_queuing_mutex_H\n#define __TBB_queuing_mutex_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_mutex_common.h\"\n\n#include \"profiling.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n//! Queuing mutex with local-only spinning.\n/** @ingroup synchronization */\nclass queuing_mutex {\npublic:\n //! Construct unacquired mutex.\n queuing_mutex() noexcept {\n create_itt_sync(this, \"tbb::queuing_mutex\", \"\");\n };\n\n queuing_mutex(const queuing_mutex&) = delete;\n queuing_mutex& operator=(const queuing_mutex&) = delete;\n\n //! The scoped locking pattern\n /** It helps to avoid the common problem of forgetting to release lock.\n It also nicely provides the \"node\" for queuing locks. */\n class scoped_lock {\n //! Reset fields to mean \"no lock held\".\n void reset() {\n m_mutex = nullptr;\n }\n\n public:\n //! Construct lock that has not acquired a mutex.\n /** Equivalent to zero-initialization of *this. */\n scoped_lock() = default;\n\n //! Acquire lock on given mutex.\n scoped_lock(queuing_mutex& m) {\n acquire(m);\n }\n\n //! Release lock (if lock is held).\n ~scoped_lock() {\n if (m_mutex) release();\n }\n\n //! No Copy\n scoped_lock( const scoped_lock& ) = delete;\n scoped_lock& operator=( const scoped_lock& ) = delete;\n\n //! Acquire lock on given mutex.\n void acquire( queuing_mutex& m ) {\n __TBB_ASSERT(!m_mutex, \"scoped_lock is already holding a mutex\");\n\n // Must set all fields before the exchange, because once the\n // exchange executes, *this becomes accessible to other threads.\n m_mutex = &m;\n m_next.store(nullptr, std::memory_order_relaxed);\n m_going.store(0U, std::memory_order_relaxed);\n\n // x86 compare exchange operation always has a strong fence\n // \"sending\" the fields initialized above to other processors.\n scoped_lock* pred = m.q_tail.exchange(this);\n if (pred) {\n call_itt_notify(prepare, &m);\n __TBB_ASSERT(pred->m_next.load(std::memory_order_relaxed) == nullptr, \"the predecessor has another successor!\");\n\n pred->m_next.store(this, std::memory_order_release);\n spin_wait_while_eq(m_going, 0U);\n }\n call_itt_notify(acquired, &m);\n\n }\n\n //! Acquire lock on given mutex if free (i.e. non-blocking)\n bool try_acquire( queuing_mutex& m ) {\n __TBB_ASSERT(!m_mutex, \"scoped_lock is already holding a mutex\");\n\n // Must set all fields before the compare_exchange_strong, because once the\n // compare_exchange_strong executes, *this becomes accessible to other threads.\n m_next.store(nullptr, std::memory_order_relaxed);\n m_going.store(0U, std::memory_order_relaxed);\n\n scoped_lock* expected = nullptr;\n // The compare_exchange_strong must have release semantics, because we are\n // \"sending\" the fields initialized above to other processors.\n // x86 compare exchange operation always has a strong fence\n if (!m.q_tail.compare_exchange_strong(expected, this, std::memory_order_acq_rel))\n return false;\n\n m_mutex = &m;\n\n call_itt_notify(acquired, &m);\n return true;\n }\n\n //! Release lock.\n void release()\n {\n __TBB_ASSERT(this->m_mutex, \"no lock acquired\");\n\n call_itt_notify(releasing, this->m_mutex);\n\n if (m_next.load(std::memory_order_relaxed) == nullptr) {\n scoped_lock* expected = this;\n if (m_mutex->q_tail.compare_exchange_strong(expected, nullptr)) {\n // this was the only item in the queue, and the queue is now empty.\n reset();\n return;\n }\n // Someone in the queue\n spin_wait_while_eq(m_next, nullptr);\n }\n m_next.load(std::memory_order_acquire)->m_going.store(1U, std::memory_order_release);\n\n reset();\n }\n\n private:\n //! The pointer to the mutex owned, or nullptr if not holding a mutex.\n queuing_mutex* m_mutex{nullptr};\n\n //! The pointer to the next competitor for a mutex\n std::atomic m_next{nullptr};\n\n //! The local spin-wait variable\n /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of\n zero-initialization. Defining it as an entire word instead of\n a byte seems to help performance slightly. */\n std::atomic m_going{0U};\n };\n\n // Mutex traits\n static constexpr bool is_rw_mutex = false;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = true;\n\nprivate:\n //! The last competitor requesting the lock\n std::atomic q_tail{nullptr};\n\n};\n\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(queuing_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(queuing_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif //WIN\n#else\ninline void set_name(queuing_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(queuing_mutex&, const wchar_t*) {}\n#endif //WIN\n#endif\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::queuing_mutex;\n} // namespace v1\nnamespace profiling {\n using detail::d1::set_name;\n}\n} // namespace tbb\n\n#endif /* __TBB_queuing_mutex_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/queuing_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_queuing_rw_mutex_H\n#define __TBB_queuing_rw_mutex_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_mutex_common.h\"\n\n#include \"profiling.h\"\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\nstruct queuing_rw_mutex_impl;\n}\nnamespace d1 {\n\n//! Queuing reader-writer mutex with local-only spinning.\n/** Adapted from Krieger, Stumm, et al. pseudocode at\n https://www.researchgate.net/publication/221083709_A_Fair_Fast_Scalable_Reader-Writer_Lock\n @ingroup synchronization */\nclass queuing_rw_mutex {\n friend r1::queuing_rw_mutex_impl;\npublic:\n //! Construct unacquired mutex.\n queuing_rw_mutex() noexcept {\n create_itt_sync(this, \"tbb::queuing_rw_mutex\", \"\");\n }\n\n //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-null\n ~queuing_rw_mutex() {\n __TBB_ASSERT(q_tail.load(std::memory_order_relaxed) == nullptr, \"destruction of an acquired mutex\");\n }\n\n //! No Copy\n queuing_rw_mutex(const queuing_rw_mutex&) = delete;\n queuing_rw_mutex& operator=(const queuing_rw_mutex&) = delete;\n\n //! The scoped locking pattern\n /** It helps to avoid the common problem of forgetting to release lock.\n It also nicely provides the \"node\" for queuing locks. */\n class scoped_lock {\n friend r1::queuing_rw_mutex_impl;\n //! Initialize fields to mean \"no lock held\".\n void initialize() {\n my_mutex = nullptr;\n my_internal_lock.store(0, std::memory_order_relaxed);\n my_going.store(0, std::memory_order_relaxed);\n#if TBB_USE_ASSERT\n my_state = 0xFF; // Set to invalid state\n my_next.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed);\n my_prev.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed);\n#endif /* TBB_USE_ASSERT */\n }\n\n public:\n //! Construct lock that has not acquired a mutex.\n /** Equivalent to zero-initialization of *this. */\n scoped_lock() {initialize();}\n\n //! Acquire lock on given mutex.\n scoped_lock( queuing_rw_mutex& m, bool write=true ) {\n initialize();\n acquire(m,write);\n }\n\n //! Release lock (if lock is held).\n ~scoped_lock() {\n if( my_mutex ) release();\n }\n\n //! No Copy\n scoped_lock(const scoped_lock&) = delete;\n scoped_lock& operator=(const scoped_lock&) = delete;\n\n //! Acquire lock on given mutex.\n void acquire( queuing_rw_mutex& m, bool write=true );\n\n //! Acquire lock on given mutex if free (i.e. non-blocking)\n bool try_acquire( queuing_rw_mutex& m, bool write=true );\n\n //! Release lock.\n void release();\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n bool upgrade_to_writer();\n\n //! Downgrade writer to become a reader.\n bool downgrade_to_reader();\n\n bool is_writer() const;\n\n private:\n //! The pointer to the mutex owned, or nullptr if not holding a mutex.\n queuing_rw_mutex* my_mutex;\n\n //! The 'pointer' to the previous and next competitors for a mutex\n std::atomic my_prev;\n std::atomic my_next;\n\n using state_t = unsigned char ;\n\n //! State of the request: reader, writer, active reader, other service states\n std::atomic my_state;\n\n //! The local spin-wait variable\n /** Corresponds to \"spin\" in the pseudocode but inverted for the sake of zero-initialization */\n std::atomic my_going;\n\n //! A tiny internal lock\n std::atomic my_internal_lock;\n };\n\n // Mutex traits\n static constexpr bool is_rw_mutex = true;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = true;\n\nprivate:\n //! The last competitor requesting the lock\n std::atomic q_tail{nullptr};\n};\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(queuing_rw_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(queuing_rw_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif //WIN\n#else\ninline void set_name(queuing_rw_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(queuing_rw_mutex&, const wchar_t*) {}\n#endif //WIN\n#endif\n} // namespace d1\n\nnamespace r1 {\nTBB_EXPORT void acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool);\nTBB_EXPORT bool try_acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool);\nTBB_EXPORT void release(d1::queuing_rw_mutex::scoped_lock&);\nTBB_EXPORT bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock&);\nTBB_EXPORT bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock&);\nTBB_EXPORT bool is_writer(const d1::queuing_rw_mutex::scoped_lock&);\n} // namespace r1\n\nnamespace d1 {\n\n\ninline void queuing_rw_mutex::scoped_lock::acquire(queuing_rw_mutex& m,bool write) {\n r1::acquire(m, *this, write);\n}\n\ninline bool queuing_rw_mutex::scoped_lock::try_acquire(queuing_rw_mutex& m, bool write) {\n return r1::try_acquire(m, *this, write);\n}\n\ninline void queuing_rw_mutex::scoped_lock::release() {\n r1::release(*this);\n}\n\ninline bool queuing_rw_mutex::scoped_lock::upgrade_to_writer() {\n return r1::upgrade_to_writer(*this);\n}\n\ninline bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() {\n return r1::downgrade_to_reader(*this);\n}\n\ninline bool queuing_rw_mutex::scoped_lock::is_writer() const {\n return r1::is_writer(*this);\n}\n} // namespace d1\n\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::queuing_rw_mutex;\n} // namespace v1\nnamespace profiling {\n using detail::d1::set_name;\n}\n} // namespace tbb\n\n#endif /* __TBB_queuing_rw_mutex_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/rw_mutex.h", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_rw_mutex_H\n#define __TBB_rw_mutex_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_waitable_atomic.h\"\n#include \"detail/_scoped_lock.h\"\n#include \"detail/_mutex_common.h\"\n#include \"profiling.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\nclass rw_mutex {\npublic:\n //! Constructors\n rw_mutex() noexcept : m_state(0) {\n create_itt_sync(this, \"tbb::rw_mutex\", \"\");\n }\n\n //! Destructor\n ~rw_mutex() {\n __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), \"destruction of an acquired mutex\");\n }\n\n //! No Copy\n rw_mutex(const rw_mutex&) = delete;\n rw_mutex& operator=(const rw_mutex&) = delete;\n\n using scoped_lock = rw_scoped_lock;\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = true;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\n\n //! Acquire lock\n void lock() {\n call_itt_notify(prepare, this);\n while (!try_lock()) {\n if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers\n m_state |= WRITER_PENDING;\n }\n\n auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); };\n adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT);\n }\n\n call_itt_notify(acquired, this);\n }\n\n //! Try acquiring lock (non-blocking)\n /** Return true if lock acquired; false otherwise. */\n bool try_lock() {\n // for a writer: only possible to acquire if no active readers or writers\n // Use relaxed memory fence is OK here because\n // Acquire memory fence guaranteed by compare_exchange_strong()\n state_type s = m_state.load(std::memory_order_relaxed);\n if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101\n if (m_state.compare_exchange_strong(s, WRITER)) {\n call_itt_notify(acquired, this);\n return true; // successfully stored writer flag\n }\n }\n return false;\n }\n\n //! Release lock\n void unlock() {\n call_itt_notify(releasing, this);\n state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state\n\n if (curr_state & WRITER_PENDING) {\n r1::notify_by_address(this, WRITER_CONTEXT);\n } else {\n // It's possible that WRITER sleeps without WRITER_PENDING,\n // because other thread might clear this bit at upgrade()\n r1::notify_by_address_all(this);\n }\n }\n\n //! Lock shared ownership mutex\n void lock_shared() {\n call_itt_notify(prepare, this);\n while (!try_lock_shared()) {\n state_type has_writer = WRITER | WRITER_PENDING;\n auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); };\n adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT);\n }\n __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, \"invalid state of a read lock: no readers\");\n }\n\n //! Try lock shared ownership mutex\n bool try_lock_shared() {\n // for a reader: acquire if no active or waiting writers\n // Use relaxed memory fence is OK here because\n // Acquire memory fence guaranteed by fetch_add()\n state_type has_writer = WRITER | WRITER_PENDING;\n if (!(m_state.load(std::memory_order_relaxed) & has_writer)) {\n if (m_state.fetch_add(ONE_READER) & has_writer) {\n m_state -= ONE_READER;\n r1::notify_by_address(this, WRITER_CONTEXT);\n } else {\n call_itt_notify(acquired, this);\n return true; // successfully stored increased number of readers\n }\n }\n return false;\n }\n\n //! Unlock shared ownership mutex\n void unlock_shared() {\n __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, \"invalid state of a read lock: no readers\");\n call_itt_notify(releasing, this);\n\n state_type curr_state = (m_state -= ONE_READER); // Returns current state\n\n if (curr_state & (WRITER_PENDING)) {\n r1::notify_by_address(this, WRITER_CONTEXT);\n } else {\n // It's possible that WRITER sleeps without WRITER_PENDING,\n // because other thread might clear this bit at upgrade()\n r1::notify_by_address_all(this);\n }\n }\n\nprivate:\n /** Internal non ISO C++ standard API **/\n //! This API is used through the scoped_lock class\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n bool upgrade() {\n state_type s = m_state.load(std::memory_order_relaxed);\n __TBB_ASSERT(s & READERS, \"invalid state before upgrade: no readers \");\n // Check and set writer-pending flag.\n // Required conditions: either no pending writers, or we are the only reader\n // (with multiple readers and pending writer, another upgrade could have been requested)\n while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) {\n if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) {\n auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; };\n while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) {\n adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT);\n }\n\n __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER),\n \"invalid state when upgrading to writer\");\n // Both new readers and writers are blocked at this time\n m_state -= (ONE_READER + WRITER_PENDING);\n return true; // successfully upgraded\n }\n }\n // Slow reacquire\n unlock_shared();\n lock();\n return false;\n }\n\n //! Downgrade writer to a reader\n void downgrade() {\n __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr),\n call_itt_notify(releasing, this);\n m_state += (ONE_READER - WRITER);\n\n if (!(m_state & WRITER_PENDING)) {\n r1::notify_by_address(this, READER_CONTEXT);\n }\n\n __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, \"invalid state after downgrade: no readers\");\n }\n\n using state_type = std::intptr_t;\n static constexpr state_type WRITER = 1;\n static constexpr state_type WRITER_PENDING = 2;\n static constexpr state_type READERS = ~(WRITER | WRITER_PENDING);\n static constexpr state_type ONE_READER = 4;\n static constexpr state_type BUSY = WRITER | READERS;\n\n using context_type = std::uintptr_t;\n static constexpr context_type WRITER_CONTEXT = 0;\n static constexpr context_type READER_CONTEXT = 1;\n friend scoped_lock;\n //! State of lock\n /** Bit 0 = writer is holding lock\n Bit 1 = request by a writer to acquire lock (hint to readers to wait)\n Bit 2..N = number of readers holding lock */\n std::atomic m_state;\n}; // class rw_mutex\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::rw_mutex;\n} // namespace v1\n\n} // namespace tbb\n\n#endif // __TBB_rw_mutex_H\n" }, { "path": "src/tbb/include/oneapi/tbb/scalable_allocator.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_scalable_allocator_H\n#define __TBB_scalable_allocator_H\n\n#ifdef __cplusplus\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_namespace_injection.h\"\n#include \n#include \n#include /* std::bad_alloc() */\n#else\n#include \"oneapi/tbb/detail/_export.h\"\n#include /* Need ptrdiff_t and size_t from here. */\n#if !defined(_MSC_VER) || defined(__clang__)\n#include /* Need intptr_t from here. */\n#endif\n#endif\n\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n#include \n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif /* __cplusplus */\n\n#if _MSC_VER\n #define __TBB_EXPORTED_FUNC __cdecl\n#else\n #define __TBB_EXPORTED_FUNC\n#endif\n\n/** The \"malloc\" analogue to allocate block of memory of size bytes.\n * @ingroup memory_allocation */\nTBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size);\n\n/** The \"free\" analogue to discard a previously allocated piece of memory.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr);\n\n/** The \"realloc\" analogue complementing scalable_malloc.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size);\n\n/** The \"calloc\" analogue complementing scalable_malloc.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size);\n\n/** The \"posix_memalign\" analogue.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size);\n\n/** The \"_aligned_malloc\" analogue.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment);\n\n/** The \"_aligned_realloc\" analogue.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment);\n\n/** The \"_aligned_free\" analogue.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr);\n\n/** The analogue of _msize/malloc_size/malloc_usable_size.\n Returns the usable size of a memory block previously allocated by scalable_*,\n or 0 (zero) if ptr does not point to such a block.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr);\n\n/* Results for scalable_allocation_* functions */\ntypedef enum {\n TBBMALLOC_OK,\n TBBMALLOC_INVALID_PARAM,\n TBBMALLOC_UNSUPPORTED,\n TBBMALLOC_NO_MEMORY,\n TBBMALLOC_NO_EFFECT\n} ScalableAllocationResult;\n\n/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.\n scalable_allocation_mode call has priority over environment variable. */\ntypedef enum {\n TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */\n /* deprecated, kept for backward compatibility only */\n USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,\n /* try to limit memory consumption value (Bytes), clean internal buffers\n if limit is exceeded, but not prevents from requesting memory from OS */\n TBBMALLOC_SET_SOFT_HEAP_LIMIT,\n /* Lower bound for the size (Bytes), that is interpreted as huge\n * and not released during regular cleanup operations. */\n TBBMALLOC_SET_HUGE_SIZE_THRESHOLD\n} AllocationModeParam;\n\n/** Set TBB allocator-specific allocation modes.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);\n\ntypedef enum {\n /* Clean internal allocator buffers for all threads.\n Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,\n TBBMALLOC_OK if some memory released from buffers. */\n TBBMALLOC_CLEAN_ALL_BUFFERS,\n /* Clean internal allocator buffer for current thread only.\n Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */\n TBBMALLOC_CLEAN_THREAD_BUFFERS\n} ScalableAllocationCmd;\n\n/** Call TBB allocator-specific commands.\n @ingroup memory_allocation */\nTBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);\n\n#ifdef __cplusplus\n} /* extern \"C\" */\n#endif /* __cplusplus */\n\n#ifdef __cplusplus\n\n//! The namespace rml contains components of low-level memory pool interface.\nnamespace rml {\nclass MemoryPool;\n\ntypedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes);\n// returns non-zero in case of error\ntypedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes);\n\nstruct MemPoolPolicy {\n enum {\n TBBMALLOC_POOL_VERSION = 1\n };\n\n rawAllocType pAlloc;\n rawFreeType pFree;\n // granularity of pAlloc allocations. 0 means default used.\n std::size_t granularity;\n int version;\n // all memory consumed at 1st pAlloc call and never returned,\n // no more pAlloc calls after 1st\n unsigned fixedPool : 1,\n // memory consumed but returned only at pool termination\n keepAllMemory : 1,\n reserved : 30;\n\n MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,\n std::size_t granularity_ = 0, bool fixedPool_ = false,\n bool keepAllMemory_ = false) :\n pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),\n fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),\n reserved(0) {}\n};\n\n// enums have same values as appropriate enums from ScalableAllocationResult\n// TODO: use ScalableAllocationResult in pool_create directly\nenum MemPoolError {\n // pool created successfully\n POOL_OK = TBBMALLOC_OK,\n // invalid policy parameters found\n INVALID_POLICY = TBBMALLOC_INVALID_PARAM,\n // requested pool policy is not supported by allocator library\n UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,\n // lack of memory during pool creation\n NO_MEMORY = TBBMALLOC_NO_MEMORY,\n // action takes no effect\n NO_EFFECT = TBBMALLOC_NO_EFFECT\n};\n\nTBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy,\n rml::MemoryPool **pool);\n\nTBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool);\nTBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size);\nTBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size);\nTBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment);\nTBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment);\nTBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool);\nTBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object);\nTBBMALLOC_EXPORT MemoryPool *pool_identify(void *object);\nTBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object);\n\n} // namespace rml\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n// keep throw in a separate function to prevent code bloat\ntemplate\nvoid throw_exception(const E &e) {\n#if TBB_USE_EXCEPTIONS\n throw e;\n#else\n suppress_unused_warning(e);\n#endif\n}\n\ntemplate\nclass scalable_allocator {\npublic:\n using value_type = T;\n using propagate_on_container_move_assignment = std::true_type;\n\n //! Always defined for TBB containers\n using is_always_equal = std::true_type;\n\n scalable_allocator() = default;\n template scalable_allocator(const scalable_allocator&) noexcept {}\n\n //! Allocate space for n objects.\n __TBB_nodiscard T* allocate(std::size_t n) {\n T* p = static_cast(scalable_malloc(n * sizeof(value_type)));\n if (!p) {\n throw_exception(std::bad_alloc());\n }\n return p;\n }\n\n //! Free previously allocated block of memory\n void deallocate(T* p, std::size_t) {\n scalable_free(p);\n }\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n using pointer = value_type*;\n using const_pointer = const value_type*;\n using reference = value_type&;\n using const_reference = const value_type&;\n using difference_type = std::ptrdiff_t;\n using size_type = std::size_t;\n template struct rebind {\n using other = scalable_allocator;\n };\n //! Largest value for which method allocate might succeed.\n size_type max_size() const noexcept {\n size_type absolutemax = static_cast(-1) / sizeof (value_type);\n return (absolutemax > 0 ? absolutemax : 1);\n }\n template\n void construct(U *p, Args&&... args)\n { ::new((void *)p) U(std::forward(args)...); }\n void destroy(pointer p) { p->~value_type(); }\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n#endif // TBB_ALLOCATOR_TRAITS_BROKEN\n\n};\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n template<>\n class scalable_allocator {\n public:\n using pointer = void*;\n using const_pointer = const void*;\n using value_type = void;\n template struct rebind {\n using other = scalable_allocator;\n };\n };\n#endif\n\ntemplate\ninline bool operator==(const scalable_allocator&, const scalable_allocator&) noexcept { return true; }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate\ninline bool operator!=(const scalable_allocator&, const scalable_allocator&) noexcept { return false; }\n#endif\n\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n\n//! C++17 memory resource implementation for scalable allocator\n//! ISO C++ Section 23.12.2\nclass scalable_resource_impl : public std::pmr::memory_resource {\nprivate:\n void* do_allocate(std::size_t bytes, std::size_t alignment) override {\n void* p = scalable_aligned_malloc(bytes, alignment);\n if (!p) {\n throw_exception(std::bad_alloc());\n }\n return p;\n }\n\n void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override {\n scalable_free(ptr);\n }\n\n //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any\n //! other instance of this class\n bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {\n return this == &other ||\n#if __TBB_USE_OPTIONAL_RTTI\n dynamic_cast(&other) != nullptr;\n#else\n false;\n#endif\n }\n};\n\n//! Global scalable allocator memory resource provider\ninline std::pmr::memory_resource* scalable_memory_resource() noexcept {\n static tbb::detail::d1::scalable_resource_impl scalable_res;\n return &scalable_res;\n}\n\n#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::scalable_allocator;\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\nusing detail::d1::scalable_memory_resource;\n#endif\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __cplusplus */\n\n#endif /* __TBB_scalable_allocator_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/spin_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_spin_mutex_H\n#define __TBB_spin_mutex_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_mutex_common.h\"\n\n#include \"profiling.h\"\n\n#include \"detail/_assert.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_scoped_lock.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n#if __TBB_TSX_INTRINSICS_PRESENT\nclass rtm_mutex;\n#endif\n\n/** A spin_mutex is a low-level synchronization primitive.\n While locked, it causes the waiting threads to spin in a loop until the lock is released.\n It should be used only for locking short critical sections\n (typically less than 20 instructions) when fairness is not an issue.\n If zero-initialized, the mutex is considered unheld.\n @ingroup synchronization */\nclass spin_mutex {\npublic:\n //! Constructors\n spin_mutex() noexcept : m_flag(false) {\n create_itt_sync(this, \"tbb::spin_mutex\", \"\");\n };\n\n //! Destructor\n ~spin_mutex() = default;\n\n //! No Copy\n spin_mutex(const spin_mutex&) = delete;\n spin_mutex& operator=(const spin_mutex&) = delete;\n\n using scoped_lock = unique_scoped_lock;\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = false;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\n\n //! Acquire lock\n /** Spin if the lock is taken */\n void lock() {\n atomic_backoff backoff;\n call_itt_notify(prepare, this);\n while (m_flag.exchange(true)) backoff.pause();\n call_itt_notify(acquired, this);\n }\n\n //! Try acquiring lock (non-blocking)\n /** Return true if lock acquired; false otherwise. */\n bool try_lock() {\n bool result = !m_flag.exchange(true);\n if (result) {\n call_itt_notify(acquired, this);\n }\n return result;\n }\n\n //! Release lock\n void unlock() {\n call_itt_notify(releasing, this);\n m_flag.store(false, std::memory_order_release);\n }\n\nprotected:\n std::atomic m_flag;\n}; // class spin_mutex\n\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(spin_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(spin_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif //WIN\n#else\ninline void set_name(spin_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(spin_mutex&, const wchar_t*) {}\n#endif // WIN\n#endif\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::spin_mutex;\n} // namespace v1\nnamespace profiling {\n using detail::d1::set_name;\n}\n} // namespace tbb\n\n#include \"detail/_rtm_mutex.h\"\n\nnamespace tbb {\ninline namespace v1 {\n#if __TBB_TSX_INTRINSICS_PRESENT\n using speculative_spin_mutex = detail::d1::rtm_mutex;\n#else\n using speculative_spin_mutex = detail::d1::spin_mutex;\n#endif\n}\n}\n\n#endif /* __TBB_spin_mutex_H */\n\n" }, { "path": "src/tbb/include/oneapi/tbb/spin_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_spin_rw_mutex_H\n#define __TBB_spin_rw_mutex_H\n\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_mutex_common.h\"\n\n#include \"profiling.h\"\n\n#include \"detail/_assert.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_scoped_lock.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n#if __TBB_TSX_INTRINSICS_PRESENT\nclass rtm_rw_mutex;\n#endif\n\n//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference\n/** @ingroup synchronization */\nclass spin_rw_mutex {\npublic:\n //! Constructors\n spin_rw_mutex() noexcept : m_state(0) {\n create_itt_sync(this, \"tbb::spin_rw_mutex\", \"\");\n }\n\n //! Destructor\n ~spin_rw_mutex() {\n __TBB_ASSERT(!m_state, \"destruction of an acquired mutex\");\n }\n\n //! No Copy\n spin_rw_mutex(const spin_rw_mutex&) = delete;\n spin_rw_mutex& operator=(const spin_rw_mutex&) = delete;\n\n using scoped_lock = rw_scoped_lock;\n\n //! Mutex traits\n static constexpr bool is_rw_mutex = true;\n static constexpr bool is_recursive_mutex = false;\n static constexpr bool is_fair_mutex = false;\n\n //! Acquire lock\n void lock() {\n call_itt_notify(prepare, this);\n for (atomic_backoff backoff; ; backoff.pause()) {\n state_type s = m_state.load(std::memory_order_relaxed);\n if (!(s & BUSY)) { // no readers, no writers\n if (m_state.compare_exchange_strong(s, WRITER))\n break; // successfully stored writer flag\n backoff.reset(); // we could be very close to complete op.\n } else if (!(s & WRITER_PENDING)) { // no pending writers\n m_state |= WRITER_PENDING;\n }\n }\n call_itt_notify(acquired, this);\n }\n\n //! Try acquiring lock (non-blocking)\n /** Return true if lock acquired; false otherwise. */\n bool try_lock() {\n // for a writer: only possible to acquire if no active readers or writers\n state_type s = m_state.load(std::memory_order_relaxed);\n if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101\n if (m_state.compare_exchange_strong(s, WRITER)) {\n call_itt_notify(acquired, this);\n return true; // successfully stored writer flag\n }\n }\n return false;\n }\n\n //! Release lock\n void unlock() {\n call_itt_notify(releasing, this);\n m_state &= READERS;\n }\n\n //! Lock shared ownership mutex\n void lock_shared() {\n call_itt_notify(prepare, this);\n for (atomic_backoff b; ; b.pause()) {\n state_type s = m_state.load(std::memory_order_relaxed);\n if (!(s & (WRITER | WRITER_PENDING))) { // no writer or write requests\n state_type prev_state = m_state.fetch_add(ONE_READER);\n if (!(prev_state & WRITER)) {\n break; // successfully stored increased number of readers\n }\n // writer got there first, undo the increment\n m_state -= ONE_READER;\n }\n }\n call_itt_notify(acquired, this);\n __TBB_ASSERT(m_state & READERS, \"invalid state of a read lock: no readers\");\n }\n\n //! Try lock shared ownership mutex\n bool try_lock_shared() {\n // for a reader: acquire if no active or waiting writers\n state_type s = m_state.load(std::memory_order_relaxed);\n if (!(s & (WRITER | WRITER_PENDING))) { // no writers\n state_type prev_state = m_state.fetch_add(ONE_READER);\n if (!(prev_state & WRITER)) { // got the lock\n call_itt_notify(acquired, this);\n return true; // successfully stored increased number of readers\n }\n // writer got there first, undo the increment\n m_state -= ONE_READER;\n }\n return false;\n }\n\n //! Unlock shared ownership mutex\n void unlock_shared() {\n __TBB_ASSERT(m_state & READERS, \"invalid state of a read lock: no readers\");\n call_itt_notify(releasing, this);\n m_state -= ONE_READER;\n }\n\nprotected:\n /** Internal non ISO C++ standard API **/\n //! This API is used through the scoped_lock class\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n bool upgrade() {\n state_type s = m_state.load(std::memory_order_relaxed);\n __TBB_ASSERT(s & READERS, \"invalid state before upgrade: no readers \");\n // Check and set writer-pending flag.\n // Required conditions: either no pending writers, or we are the only reader\n // (with multiple readers and pending writer, another upgrade could have been requested)\n while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) {\n if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) {\n atomic_backoff backoff;\n while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) backoff.pause();\n __TBB_ASSERT((m_state & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), \"invalid state when upgrading to writer\");\n // Both new readers and writers are blocked at this time\n m_state -= (ONE_READER + WRITER_PENDING);\n return true; // successfully upgraded\n }\n }\n // Slow reacquire\n unlock_shared();\n lock();\n return false;\n }\n\n //! Downgrade writer to a reader\n void downgrade() {\n call_itt_notify(releasing, this);\n m_state += (ONE_READER - WRITER);\n __TBB_ASSERT(m_state & READERS, \"invalid state after downgrade: no readers\");\n }\n\n using state_type = std::intptr_t;\n static constexpr state_type WRITER = 1;\n static constexpr state_type WRITER_PENDING = 2;\n static constexpr state_type READERS = ~(WRITER | WRITER_PENDING);\n static constexpr state_type ONE_READER = 4;\n static constexpr state_type BUSY = WRITER | READERS;\n friend scoped_lock;\n //! State of lock\n /** Bit 0 = writer is holding lock\n Bit 1 = request by a writer to acquire lock (hint to readers to wait)\n Bit 2..N = number of readers holding lock */\n std::atomic m_state;\n}; // class spin_rw_mutex\n\n#if TBB_USE_PROFILING_TOOLS\ninline void set_name(spin_rw_mutex& obj, const char* name) {\n itt_set_sync_name(&obj, name);\n}\n#if (_WIN32||_WIN64)\ninline void set_name(spin_rw_mutex& obj, const wchar_t* name) {\n itt_set_sync_name(&obj, name);\n}\n#endif // WIN\n#else\ninline void set_name(spin_rw_mutex&, const char*) {}\n#if (_WIN32||_WIN64)\ninline void set_name(spin_rw_mutex&, const wchar_t*) {}\n#endif // WIN\n#endif\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::spin_rw_mutex;\n} // namespace v1\nnamespace profiling {\n using detail::d1::set_name;\n}\n} // namespace tbb\n\n#include \"detail/_rtm_rw_mutex.h\"\n\nnamespace tbb {\ninline namespace v1 {\n#if __TBB_TSX_INTRINSICS_PRESENT\n using speculative_spin_rw_mutex = detail::d1::rtm_rw_mutex;\n#else\n using speculative_spin_rw_mutex = detail::d1::spin_rw_mutex;\n#endif\n}\n}\n\n#endif /* __TBB_spin_rw_mutex_H */\n\n" }, { "path": "src/tbb/include/oneapi/tbb/task.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_task_H\n#define __TBB_task_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_task.h\"\n\nnamespace tbb {\ninline namespace v1 {\nnamespace task {\n#if __TBB_RESUMABLE_TASKS\n using detail::d1::suspend_point;\n using detail::d1::resume;\n using detail::d1::suspend;\n#endif /* __TBB_RESUMABLE_TASKS */\n using detail::d1::current_context;\n} // namespace task\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_task_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/task_arena.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_task_arena_H\n#define __TBB_task_arena_H\n\n#include \"detail/_config.h\"\n\n#include \"detail/_aligned_space.h\"\n#include \"detail/_attach.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"detail/_task.h\"\n\n#include \"detail/_task_handle.h\"\n\n#if __TBB_ARENA_BINDING\n#include \"info.h\"\n#endif /*__TBB_ARENA_BINDING*/\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\n\ntemplate\nclass task_arena_function : public delegate_base {\n F &my_func;\n aligned_space my_return_storage;\n bool my_constructed{false};\n // The function should be called only once.\n bool operator()() const override {\n new (my_return_storage.begin()) R(my_func());\n return true;\n }\npublic:\n task_arena_function(F& f) : my_func(f) {}\n // The function can be called only after operator() and only once.\n R consume_result() {\n my_constructed = true;\n return std::move(*(my_return_storage.begin()));\n }\n ~task_arena_function() override {\n if (my_constructed) {\n my_return_storage.begin()->~R();\n }\n }\n};\n\ntemplate\nclass task_arena_function : public delegate_base {\n F &my_func;\n bool operator()() const override {\n my_func();\n return true;\n }\npublic:\n task_arena_function(F& f) : my_func(f) {}\n void consume_result() const {}\n\n friend class task_arena_base;\n};\n\nclass task_arena_base;\nclass task_scheduler_observer;\n} // namespace d1\n\nnamespace r1 {\nclass arena;\nstruct task_arena_impl;\n\nTBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool);\nTBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_arena_base&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC terminate(d1::task_arena_base&);\nTBB_EXPORT bool __TBB_EXPORTED_FUNC attach(d1::task_arena_base&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::task_arena_base&);\nTBB_EXPORT int __TBB_EXPORTED_FUNC max_concurrency(const d1::task_arena_base*);\nTBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base& d, std::intptr_t);\n\nTBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_arena_base*);\nTBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_group_context&, d1::task_arena_base*);\nTBB_EXPORT void __TBB_EXPORTED_FUNC submit(d1::task&, d1::task_group_context&, arena*, std::uintptr_t);\n} // namespace r1\n\nnamespace d2 {\ninline void enqueue_impl(task_handle&& th, d1::task_arena_base* ta) {\n __TBB_ASSERT(th != nullptr, \"Attempt to schedule empty task_handle\");\n\n auto& ctx = task_handle_accessor::ctx_of(th);\n\n // Do not access th after release\n r1::enqueue(*task_handle_accessor::release(th), ctx, ta);\n}\n} //namespace d2\n\nnamespace d1 {\n\nstatic constexpr unsigned num_priority_levels = 3;\nstatic constexpr int priority_stride = INT_MAX / (num_priority_levels + 1);\n\nclass task_arena_base {\n friend struct r1::task_arena_impl;\n friend void r1::observe(d1::task_scheduler_observer&, bool);\npublic:\n enum class priority : int {\n low = 1 * priority_stride,\n normal = 2 * priority_stride,\n high = 3 * priority_stride\n };\n#if __TBB_ARENA_BINDING\n using constraints = tbb::detail::d1::constraints;\n#endif /*__TBB_ARENA_BINDING*/\nprotected:\n //! Special settings\n intptr_t my_version_and_traits;\n\n std::atomic my_initialization_state;\n\n //! nullptr if not currently initialized.\n std::atomic my_arena;\n static_assert(sizeof(std::atomic) == sizeof(r1::arena*),\n \"To preserve backward compatibility we need the equal size of an atomic pointer and a pointer\");\n\n //! Concurrency level for deferred initialization\n int my_max_concurrency;\n\n //! Reserved slots for external threads\n unsigned my_num_reserved_slots;\n\n //! Arena priority\n priority my_priority;\n\n //! The NUMA node index to which the arena will be attached\n numa_node_id my_numa_id;\n\n //! The core type index to which arena will be attached\n core_type_id my_core_type;\n\n //! Number of threads per core\n int my_max_threads_per_core;\n\n // Backward compatibility checks.\n core_type_id core_type() const {\n return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_core_type : automatic;\n }\n int max_threads_per_core() const {\n return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_max_threads_per_core : automatic;\n }\n\n enum {\n default_flags = 0\n , core_type_support_flag = 1\n };\n\n task_arena_base(int max_concurrency, unsigned reserved_for_masters, priority a_priority)\n : my_version_and_traits(default_flags | core_type_support_flag)\n , my_initialization_state(do_once_state::uninitialized)\n , my_arena(nullptr)\n , my_max_concurrency(max_concurrency)\n , my_num_reserved_slots(reserved_for_masters)\n , my_priority(a_priority)\n , my_numa_id(automatic)\n , my_core_type(automatic)\n , my_max_threads_per_core(automatic)\n {}\n\n#if __TBB_ARENA_BINDING\n task_arena_base(const constraints& constraints_, unsigned reserved_for_masters, priority a_priority)\n : my_version_and_traits(default_flags | core_type_support_flag)\n , my_initialization_state(do_once_state::uninitialized)\n , my_arena(nullptr)\n , my_max_concurrency(constraints_.max_concurrency)\n , my_num_reserved_slots(reserved_for_masters)\n , my_priority(a_priority)\n , my_numa_id(constraints_.numa_id)\n , my_core_type(constraints_.core_type)\n , my_max_threads_per_core(constraints_.max_threads_per_core)\n {}\n#endif /*__TBB_ARENA_BINDING*/\npublic:\n //! Typedef for number of threads that is automatic.\n static const int automatic = -1;\n static const int not_initialized = -2;\n};\n\ntemplate\nR isolate_impl(F& f) {\n task_arena_function func(f);\n r1::isolate_within_arena(func, /*isolation*/ 0);\n return func.consume_result();\n}\n\ntemplate \nclass enqueue_task : public task {\n small_object_allocator m_allocator;\n const F m_func;\n\n void finalize(const execution_data& ed) {\n m_allocator.delete_object(this, ed);\n }\n task* execute(execution_data& ed) override {\n m_func();\n finalize(ed);\n return nullptr;\n }\n task* cancel(execution_data&) override {\n __TBB_ASSERT_RELEASE(false, \"Unhandled exception from enqueue task is caught\");\n return nullptr;\n }\npublic:\n enqueue_task(const F& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(f) {}\n enqueue_task(F&& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(std::move(f)) {}\n};\n\ntemplate\nvoid enqueue_impl(F&& f, task_arena_base* ta) {\n small_object_allocator alloc{};\n r1::enqueue(*alloc.new_object::type>>(std::forward(f), alloc), ta);\n}\n/** 1-to-1 proxy representation class of scheduler's arena\n * Constructors set up settings only, real construction is deferred till the first method invocation\n * Destructor only removes one of the references to the inner arena representation.\n * Final destruction happens when all the references (and the work) are gone.\n */\nclass task_arena : public task_arena_base {\n\n void mark_initialized() {\n __TBB_ASSERT( my_arena.load(std::memory_order_relaxed), \"task_arena initialization is incomplete\" );\n my_initialization_state.store(do_once_state::initialized, std::memory_order_release);\n }\n\n template\n R execute_impl(F& f) {\n initialize();\n task_arena_function func(f);\n r1::execute(*this, func);\n return func.consume_result();\n }\npublic:\n //! Creates task_arena with certain concurrency limits\n /** Sets up settings only, real construction is deferred till the first method invocation\n * @arg max_concurrency specifies total number of slots in arena where threads work\n * @arg reserved_for_masters specifies number of slots to be used by external threads only.\n * Value of 1 is default and reflects behavior of implicit arenas.\n **/\n task_arena(int max_concurrency_ = automatic, unsigned reserved_for_masters = 1,\n priority a_priority = priority::normal)\n : task_arena_base(max_concurrency_, reserved_for_masters, a_priority)\n {}\n\n#if __TBB_ARENA_BINDING\n //! Creates task arena pinned to certain NUMA node\n task_arena(const constraints& constraints_, unsigned reserved_for_masters = 1,\n priority a_priority = priority::normal)\n : task_arena_base(constraints_, reserved_for_masters, a_priority)\n {}\n\n //! Copies settings from another task_arena\n task_arena(const task_arena &s) // copy settings but not the reference or instance\n : task_arena_base(\n constraints{}\n .set_numa_id(s.my_numa_id)\n .set_max_concurrency(s.my_max_concurrency)\n .set_core_type(s.my_core_type)\n .set_max_threads_per_core(s.my_max_threads_per_core)\n , s.my_num_reserved_slots, s.my_priority)\n {}\n#else\n //! Copies settings from another task_arena\n task_arena(const task_arena& a) // copy settings but not the reference or instance\n : task_arena_base(a.my_max_concurrency, a.my_num_reserved_slots, a.my_priority)\n {}\n#endif /*__TBB_ARENA_BINDING*/\n\n //! Tag class used to indicate the \"attaching\" constructor\n struct attach {};\n\n //! Creates an instance of task_arena attached to the current arena of the thread\n explicit task_arena( attach )\n : task_arena_base(automatic, 1, priority::normal) // use default settings if attach fails\n {\n if (r1::attach(*this)) {\n mark_initialized();\n }\n }\n\n //! Creates an instance of task_arena attached to the current arena of the thread\n explicit task_arena(d1::attach)\n : task_arena(attach{})\n {}\n\n //! Forces allocation of the resources for the task_arena as specified in constructor arguments\n void initialize() {\n atomic_do_once([this]{ r1::initialize(*this); }, my_initialization_state);\n }\n\n //! Overrides concurrency level and forces initialization of internal representation\n void initialize(int max_concurrency_, unsigned reserved_for_masters = 1,\n priority a_priority = priority::normal)\n {\n __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), \"Impossible to modify settings of an already initialized task_arena\");\n if( !is_active() ) {\n my_max_concurrency = max_concurrency_;\n my_num_reserved_slots = reserved_for_masters;\n my_priority = a_priority;\n r1::initialize(*this);\n mark_initialized();\n }\n }\n\n#if __TBB_ARENA_BINDING\n void initialize(constraints constraints_, unsigned reserved_for_masters = 1,\n priority a_priority = priority::normal)\n {\n __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), \"Impossible to modify settings of an already initialized task_arena\");\n if( !is_active() ) {\n my_numa_id = constraints_.numa_id;\n my_max_concurrency = constraints_.max_concurrency;\n my_core_type = constraints_.core_type;\n my_max_threads_per_core = constraints_.max_threads_per_core;\n my_num_reserved_slots = reserved_for_masters;\n my_priority = a_priority;\n r1::initialize(*this);\n mark_initialized();\n }\n }\n#endif /*__TBB_ARENA_BINDING*/\n\n //! Attaches this instance to the current arena of the thread\n void initialize(attach) {\n // TODO: decide if this call must be thread-safe\n __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), \"Impossible to modify settings of an already initialized task_arena\");\n if( !is_active() ) {\n if ( !r1::attach(*this) ) {\n r1::initialize(*this);\n }\n mark_initialized();\n }\n }\n\n //! Attaches this instance to the current arena of the thread\n void initialize(d1::attach) {\n initialize(attach{});\n }\n\n //! Removes the reference to the internal arena representation.\n //! Not thread safe wrt concurrent invocations of other methods.\n void terminate() {\n if( is_active() ) {\n r1::terminate(*this);\n my_initialization_state.store(do_once_state::uninitialized, std::memory_order_relaxed);\n }\n }\n\n //! Removes the reference to the internal arena representation, and destroys the external object.\n //! Not thread safe wrt concurrent invocations of other methods.\n ~task_arena() {\n terminate();\n }\n\n //! Returns true if the arena is active (initialized); false otherwise.\n //! The name was chosen to match a task_scheduler_init method with the same semantics.\n bool is_active() const {\n return my_initialization_state.load(std::memory_order_acquire) == do_once_state::initialized;\n }\n\n //! Enqueues a task into the arena to process a functor, and immediately returns.\n //! Does not require the calling thread to join the arena\n\n template\n void enqueue(F&& f) {\n initialize();\n enqueue_impl(std::forward(f), this);\n }\n\n //! Enqueues a task into the arena to process a functor wrapped in task_handle, and immediately returns.\n //! Does not require the calling thread to join the arena\n void enqueue(d2::task_handle&& th) {\n initialize();\n d2::enqueue_impl(std::move(th), this);\n }\n\n //! Joins the arena and executes a mutable functor, then returns\n //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion\n //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread\n //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void).\n template\n auto execute(F&& f) -> decltype(f()) {\n return execute_impl(f);\n }\n\n#if __TBB_EXTRA_DEBUG\n //! Returns my_num_reserved_slots\n int debug_reserved_slots() const {\n // Handle special cases inside the library\n return my_num_reserved_slots;\n }\n\n //! Returns my_max_concurrency\n int debug_max_concurrency() const {\n // Handle special cases inside the library\n return my_max_concurrency;\n }\n\n //! Wait for all work in the arena to be completed\n //! Even submitted by other application threads\n //! Joins arena if/when possible (in the same way as execute())\n void debug_wait_until_empty() {\n initialize();\n r1::wait(*this);\n }\n#endif //__TBB_EXTRA_DEBUG\n\n //! Returns the maximal number of threads that can work inside the arena\n int max_concurrency() const {\n // Handle special cases inside the library\n return (my_max_concurrency > 1) ? my_max_concurrency : r1::max_concurrency(this);\n }\n\n friend void submit(task& t, task_arena& ta, task_group_context& ctx, bool as_critical) {\n __TBB_ASSERT(ta.is_active(), nullptr);\n call_itt_task_notify(releasing, &t);\n r1::submit(t, ctx, ta.my_arena.load(std::memory_order_relaxed), as_critical ? 1 : 0);\n }\n};\n\n//! Executes a mutable functor in isolation within the current task arena.\n//! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void).\ntemplate\ninline auto isolate(F&& f) -> decltype(f()) {\n return isolate_impl(f);\n}\n\n//! Returns the index, aka slot number, of the calling thread in its current arena\ninline int current_thread_index() {\n slot_id idx = r1::execution_slot(nullptr);\n return idx == slot_id(-1) ? task_arena_base::not_initialized : int(idx);\n}\n\n#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS\ninline bool is_inside_task() {\n return nullptr != current_context();\n}\n#endif //__TBB_PREVIEW_TASK_GROUP_EXTENSIONS\n\n//! Returns the maximal number of threads that can work inside the arena\ninline int max_concurrency() {\n return r1::max_concurrency(nullptr);\n}\n\ninline void enqueue(d2::task_handle&& th) {\n d2::enqueue_impl(std::move(th), nullptr);\n}\n\ntemplate\ninline void enqueue(F&& f) {\n enqueue_impl(std::forward(f), nullptr);\n}\n\nusing r1::submit;\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::task_arena;\nusing detail::d1::attach;\n\n#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS\nusing detail::d1::is_inside_task;\n#endif\n\nnamespace this_task_arena {\nusing detail::d1::current_thread_index;\nusing detail::d1::max_concurrency;\nusing detail::d1::isolate;\n\nusing detail::d1::enqueue;\n} // namespace this_task_arena\n\n} // inline namespace v1\n\n} // namespace tbb\n#endif /* __TBB_task_arena_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/task_group.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_task_group_H\n#define __TBB_task_group_H\n\n#include \"detail/_config.h\"\n#include \"detail/_namespace_injection.h\"\n#include \"detail/_assert.h\"\n#include \"detail/_utils.h\"\n#include \"detail/_template_helpers.h\"\n#include \"detail/_exception.h\"\n#include \"detail/_task.h\"\n#include \"detail/_small_object_pool.h\"\n#include \"detail/_intrusive_list_node.h\"\n#include \"detail/_task_handle.h\"\n\n#include \"profiling.h\"\n\n#include \n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Suppress warning: structure was padded due to alignment specifier\n // #pragma warning(push)\n // #pragma warning(disable:4324)\n#endif\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass delegate_base;\nclass task_arena_base;\nclass task_group_context;\n}\n\nnamespace r1 {\n// Forward declarations\nclass tbb_exception_ptr;\nclass cancellation_disseminator;\nclass thread_data;\nclass task_dispatcher;\ntemplate \nclass context_guard_helper;\nstruct task_arena_impl;\nclass context_list;\n\nTBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base&, std::intptr_t);\n\nTBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_group_context&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::task_group_context&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC reset(d1::task_group_context&);\nTBB_EXPORT bool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context&);\nTBB_EXPORT bool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context&);\nTBB_EXPORT void __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context&);\n\nstruct task_group_context_impl;\n}\n\nnamespace d2 {\n\nnamespace {\ntemplate\nd1::task* task_ptr_or_nullptr(F&& f);\n}\n\ntemplate\nclass function_task : public task_handle_task {\n //TODO: apply empty base optimization here\n const F m_func;\n\nprivate:\n d1::task* execute(d1::execution_data& ed) override {\n __TBB_ASSERT(ed.context == &this->ctx(), \"The task group context should be used for all tasks\");\n task* res = task_ptr_or_nullptr(m_func);\n finalize(&ed);\n return res;\n }\n d1::task* cancel(d1::execution_data& ed) override {\n finalize(&ed);\n return nullptr;\n }\npublic:\n template\n function_task(FF&& f, d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, d1::small_object_allocator& alloc)\n : task_handle_task{vertex, ctx, alloc},\n m_func(std::forward(f)) {}\n};\n\n#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS\nnamespace {\n template\n d1::task* task_ptr_or_nullptr_impl(std::false_type, F&& f){\n task_handle th = std::forward(f)();\n return task_handle_accessor::release(th);\n }\n\n template\n d1::task* task_ptr_or_nullptr_impl(std::true_type, F&& f){\n std::forward(f)();\n return nullptr;\n }\n\n template\n d1::task* task_ptr_or_nullptr(F&& f){\n using is_void_t = std::is_void<\n decltype(std::forward(f)())\n >;\n\n return task_ptr_or_nullptr_impl(is_void_t{}, std::forward(f));\n }\n}\n#else\nnamespace {\n template\n d1::task* task_ptr_or_nullptr(F&& f){\n std::forward(f)();\n return nullptr;\n }\n} // namespace\n#endif // __TBB_PREVIEW_TASK_GROUP_EXTENSIONS\n} // namespace d2\n\nnamespace d1 {\n\n// This structure is left here for backward compatibility check\nstruct context_list_node {\n std::atomic prev{};\n std::atomic next{};\n};\n\n//! Used to form groups of tasks\n/** @ingroup task_scheduling\n The context services explicit cancellation requests from user code, and unhandled\n exceptions intercepted during tasks execution. Intercepting an exception results\n in generating internal cancellation requests (which is processed in exactly the\n same way as external ones).\n\n The context is associated with one or more root tasks and defines the cancellation\n group that includes all the descendants of the corresponding root task(s). Association\n is established when a context object is passed as an argument to the task::allocate_root()\n method. See task_group_context::task_group_context for more details.\n\n The context can be bound to another one, and other contexts can be bound to it,\n forming a tree-like structure: parent -> this -> children. Arrows here designate\n cancellation propagation direction. If a task in a cancellation group is cancelled\n all the other tasks in this group and groups bound to it (as children) get cancelled too.\n**/\nclass task_group_context : no_copy {\npublic:\n enum traits_type {\n fp_settings = 1 << 1,\n concurrent_wait = 1 << 2,\n default_traits = 0\n };\n enum kind_type {\n isolated,\n bound\n };\nprivate:\n //! Space for platform-specific FPU settings.\n /** Must only be accessed inside TBB binaries, and never directly in user\n code or inline methods. */\n std::uint64_t my_cpu_ctl_env;\n\n //! Specifies whether cancellation was requested for this task group.\n std::atomic my_cancellation_requested;\n\n //! Versioning for run-time checks and behavioral traits of the context.\n enum class task_group_context_version : std::uint8_t {\n unused = 1 // ensure that new versions, if any, will not clash with previously used ones\n };\n task_group_context_version my_version;\n\n //! The context traits.\n struct context_traits {\n bool fp_settings : 1;\n bool concurrent_wait : 1;\n bool bound : 1;\n bool reserved1 : 1;\n bool reserved2 : 1;\n bool reserved3 : 1;\n bool reserved4 : 1;\n bool reserved5 : 1;\n } my_traits;\n\n static_assert(sizeof(context_traits) == 1, \"Traits shall fit into one byte.\");\n\n static constexpr std::uint8_t may_have_children = 1;\n //! The context internal state (currently only may_have_children).\n std::atomic my_may_have_children;\n\n enum class state : std::uint8_t {\n created,\n locked,\n isolated,\n bound,\n dead,\n proxy = std::uint8_t(-1) //the context is not the real one, but proxy to other one\n };\n\n //! The synchronization machine state to manage lifetime.\n std::atomic my_state;\n\n union {\n //! Pointer to the context of the parent cancellation group. nullptr for isolated contexts.\n task_group_context* my_parent;\n\n //! Pointer to the actual context 'this' context represents a proxy of.\n task_group_context* my_actual_context;\n };\n\n //! Thread data instance that registered this context in its list.\n r1::context_list* my_context_list;\n static_assert(sizeof(std::atomic) == sizeof(r1::context_list*), \"To preserve backward compatibility these types should have the same size\");\n\n //! Used to form the thread specific list of contexts without additional memory allocation.\n /** A context is included into the list of the current thread when its binding to\n its parent happens. Any context can be present in the list of one thread only. **/\n intrusive_list_node my_node;\n static_assert(sizeof(intrusive_list_node) == sizeof(context_list_node), \"To preserve backward compatibility these types should have the same size\");\n\n //! Pointer to the container storing exception being propagated across this task group.\n std::atomic my_exception;\n static_assert(sizeof(std::atomic) == sizeof(r1::tbb_exception_ptr*),\n \"backward compatibility check\");\n\n //! Used to set and maintain stack stitching point for Intel Performance Tools.\n void* my_itt_caller;\n\n //! Description of algorithm for scheduler based instrumentation.\n string_resource_index my_name;\n\n char padding[max_nfs_size\n - sizeof(std::uint64_t) // my_cpu_ctl_env\n - sizeof(std::atomic) // my_cancellation_requested\n - sizeof(std::uint8_t) // my_version\n - sizeof(context_traits) // my_traits\n - sizeof(std::atomic) // my_state\n - sizeof(std::atomic) // my_state\n - sizeof(task_group_context*) // my_parent\n - sizeof(r1::context_list*) // my_context_list\n - sizeof(intrusive_list_node) // my_node\n - sizeof(std::atomic) // my_exception\n - sizeof(void*) // my_itt_caller\n - sizeof(string_resource_index) // my_name\n ];\n\n task_group_context(context_traits t, string_resource_index name)\n : my_version{task_group_context_version::unused}, my_name{name}\n {\n my_traits = t; // GCC4.8 issues warning list initialization for bitset (missing-field-initializers)\n r1::initialize(*this);\n }\n\n task_group_context(task_group_context* actual_context)\n : my_version{task_group_context_version::unused}\n , my_state{state::proxy}\n , my_actual_context{actual_context}\n {\n __TBB_ASSERT(my_actual_context, \"Passed pointer value points to nothing.\");\n my_name = actual_context->my_name;\n\n // no need to initialize 'this' context as it acts as a proxy for my_actual_context, which\n // initialization is a user-side responsibility.\n }\n\n static context_traits make_traits(kind_type relation_with_parent, std::uintptr_t user_traits) {\n context_traits ct;\n ct.fp_settings = (user_traits & fp_settings) == fp_settings;\n ct.concurrent_wait = (user_traits & concurrent_wait) == concurrent_wait;\n ct.bound = relation_with_parent == bound;\n ct.reserved1 = ct.reserved2 = ct.reserved3 = ct.reserved4 = ct.reserved5 = false;\n return ct;\n }\n\n bool is_proxy() const {\n return my_state.load(std::memory_order_relaxed) == state::proxy;\n }\n\n task_group_context& actual_context() noexcept {\n if (is_proxy()) {\n __TBB_ASSERT(my_actual_context, \"Actual task_group_context is not set.\");\n return *my_actual_context;\n }\n return *this;\n }\n\n const task_group_context& actual_context() const noexcept {\n if (is_proxy()) {\n __TBB_ASSERT(my_actual_context, \"Actual task_group_context is not set.\");\n return *my_actual_context;\n }\n return *this;\n }\n\npublic:\n //! Default & binding constructor.\n /** By default a bound context is created. That is this context will be bound\n (as child) to the context of the currently executing task . Cancellation\n requests passed to the parent context are propagated to all the contexts\n bound to it. Similarly priority change is propagated from the parent context\n to its children.\n\n If task_group_context::isolated is used as the argument, then the tasks associated\n with this context will never be affected by events in any other context.\n\n Creating isolated contexts involve much less overhead, but they have limited\n utility. Normally when an exception occurs in an algorithm that has nested\n ones running, it is desirably to have all the nested algorithms cancelled\n as well. Such a behavior requires nested algorithms to use bound contexts.\n\n There is one good place where using isolated algorithms is beneficial. It is\n an external thread. That is if a particular algorithm is invoked directly from\n the external thread (not from a TBB task), supplying it with explicitly\n created isolated context will result in a faster algorithm startup.\n\n VERSIONING NOTE:\n Implementation(s) of task_group_context constructor(s) cannot be made\n entirely out-of-line because the run-time version must be set by the user\n code. This will become critically important for binary compatibility, if\n we ever have to change the size of the context object. **/\n\n task_group_context(kind_type relation_with_parent = bound,\n std::uintptr_t t = default_traits)\n : task_group_context(make_traits(relation_with_parent, t), CUSTOM_CTX) {}\n\n // Custom constructor for instrumentation of oneTBB algorithm\n task_group_context(string_resource_index name )\n : task_group_context(make_traits(bound, default_traits), name) {}\n\n // Do not introduce any logic on user side since it might break state propagation assumptions\n ~task_group_context() {\n // When 'this' serves as a proxy, the initialization does not happen - nor should the\n // destruction.\n if (!is_proxy())\n {\n r1::destroy(*this);\n }\n }\n\n //! Forcefully reinitializes the context after the task tree it was associated with is completed.\n /** Because the method assumes that all the tasks that used to be associated with\n this context have already finished, calling it while the context is still\n in use somewhere in the task hierarchy leads to undefined behavior.\n\n IMPORTANT: This method is not thread safe!\n\n The method does not change the context's parent if it is set. **/\n void reset() {\n r1::reset(actual_context());\n }\n\n //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups.\n /** \\return false if cancellation has already been requested, true otherwise.\n\n Note that canceling never fails. When false is returned, it just means that\n another thread (or this one) has already sent cancellation request to this\n context or to one of its ancestors (if this context is bound). It is guaranteed\n that when this method is concurrently called on the same not yet cancelled\n context, true will be returned by one and only one invocation. **/\n bool cancel_group_execution() {\n return r1::cancel_group_execution(actual_context());\n }\n\n //! Returns true if the context received cancellation request.\n bool is_group_execution_cancelled() {\n return r1::is_group_execution_cancelled(actual_context());\n }\n\n#if __TBB_FP_CONTEXT\n //! Captures the current FPU control settings to the context.\n /** Because the method assumes that all the tasks that used to be associated with\n this context have already finished, calling it while the context is still\n in use somewhere in the task hierarchy leads to undefined behavior.\n\n IMPORTANT: This method is not thread safe!\n\n The method does not change the FPU control settings of the context's parent. **/\n void capture_fp_settings() {\n r1::capture_fp_settings(actual_context());\n }\n#endif\n\n //! Returns the user visible context trait\n std::uintptr_t traits() const {\n std::uintptr_t t{};\n const task_group_context& ctx = actual_context();\n t |= ctx.my_traits.fp_settings ? fp_settings : 0;\n t |= ctx.my_traits.concurrent_wait ? concurrent_wait : 0;\n return t;\n }\nprivate:\n //// TODO: cleanup friends\n friend class r1::cancellation_disseminator;\n friend class r1::thread_data;\n friend class r1::task_dispatcher;\n template \n friend class r1::context_guard_helper;\n friend struct r1::task_arena_impl;\n friend struct r1::task_group_context_impl;\n friend class d2::task_group_base;\n}; // class task_group_context\n\nstatic_assert(sizeof(task_group_context) == 128, \"Wrong size of task_group_context\");\n\ninline bool is_current_task_group_canceling() {\n task_group_context* ctx = current_context();\n return ctx ? ctx->is_group_execution_cancelled() : false;\n}\n\n} // namespace d1\n\nnamespace d2 {\n\nenum task_group_status {\n not_complete,\n complete,\n canceled\n};\n\nclass task_group;\nclass structured_task_group;\n#if TBB_PREVIEW_ISOLATED_TASK_GROUP\nclass isolated_task_group;\n#endif\n\ntemplate \nclass function_stack_task : public d1::task {\n const F& m_func;\n d1::wait_tree_vertex_interface* m_wait_tree_vertex;\n\n void finalize() {\n m_wait_tree_vertex->release();\n }\n task* execute(d1::execution_data&) override {\n task* res = d2::task_ptr_or_nullptr(m_func);\n finalize();\n return res;\n }\n task* cancel(d1::execution_data&) override {\n finalize();\n return nullptr;\n }\npublic:\n function_stack_task(const F& f, d1::wait_tree_vertex_interface* vertex) : m_func(f), m_wait_tree_vertex(vertex) {\n m_wait_tree_vertex->reserve();\n }\n};\n\nclass task_group_base : no_copy {\nprotected:\n d1::wait_context_vertex m_wait_vertex;\n d1::task_group_context m_context;\n\n template\n task_group_status internal_run_and_wait(const F& f) {\n function_stack_task t{ f, r1::get_thread_reference_vertex(&m_wait_vertex) };\n\n bool cancellation_status = false;\n try_call([&] {\n execute_and_wait(t, context(), m_wait_vertex.get_context(), context());\n }).on_completion([&] {\n // TODO: the reset method is not thread-safe. Ensure the correct behavior.\n cancellation_status = context().is_group_execution_cancelled();\n context().reset();\n });\n return cancellation_status ? canceled : complete;\n }\n\n task_group_status internal_run_and_wait(d2::task_handle&& h) {\n __TBB_ASSERT(h != nullptr, \"Attempt to schedule empty task_handle\");\n\n using acs = d2::task_handle_accessor;\n __TBB_ASSERT(&acs::ctx_of(h) == &context(), \"Attempt to schedule task_handle into different task_group\");\n\n bool cancellation_status = false;\n try_call([&] {\n execute_and_wait(*acs::release(h), context(), m_wait_vertex.get_context(), context());\n }).on_completion([&] {\n // TODO: the reset method is not thread-safe. Ensure the correct behavior.\n cancellation_status = context().is_group_execution_cancelled();\n context().reset();\n });\n return cancellation_status ? canceled : complete;\n }\n\n template\n d1::task* prepare_task(F&& f) {\n d1::small_object_allocator alloc{};\n return alloc.new_object::type>>(std::forward(f),\n r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc);\n }\n\n d1::task_group_context& context() noexcept {\n return m_context.actual_context();\n }\n\n template\n d2::task_handle prepare_task_handle(F&& f) {\n d1::small_object_allocator alloc{};\n using function_task_t = d2::function_task::type>;\n d2::task_handle_task* function_task_p = alloc.new_object(std::forward(f),\n r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc);\n\n return d2::task_handle_accessor::construct(function_task_p);\n }\n\npublic:\n task_group_base(uintptr_t traits = 0)\n : m_wait_vertex(0)\n , m_context(d1::task_group_context::bound, d1::task_group_context::default_traits | traits)\n {}\n\n task_group_base(d1::task_group_context& ctx)\n : m_wait_vertex(0)\n , m_context(&ctx)\n {}\n\n ~task_group_base() noexcept(false) {\n if (m_wait_vertex.continue_execution()) {\n#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT\n bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0;\n#else\n bool stack_unwinding_in_progress = std::uncaught_exception();\n#endif\n // Always attempt to do proper cleanup to avoid inevitable memory corruption\n // in case of missing wait (for the sake of better testability & debuggability)\n if (!context().is_group_execution_cancelled())\n cancel();\n d1::wait(m_wait_vertex.get_context(), context());\n if (!stack_unwinding_in_progress)\n throw_exception(exception_id::missing_wait);\n }\n }\n\n task_group_status wait() {\n bool cancellation_status = false;\n try_call([&] {\n d1::wait(m_wait_vertex.get_context(), context());\n }).on_completion([&] {\n // TODO: the reset method is not thread-safe. Ensure the correct behavior.\n cancellation_status = m_context.is_group_execution_cancelled();\n context().reset();\n });\n return cancellation_status ? canceled : complete;\n }\n\n void cancel() {\n context().cancel_group_execution();\n }\n}; // class task_group_base\n\nclass task_group : public task_group_base {\npublic:\n task_group() : task_group_base(d1::task_group_context::concurrent_wait) {}\n task_group(d1::task_group_context& ctx) : task_group_base(ctx) {}\n\n template\n void run(F&& f) {\n d1::spawn(*prepare_task(std::forward(f)), context());\n }\n\n void run(d2::task_handle&& h) {\n __TBB_ASSERT(h != nullptr, \"Attempt to schedule empty task_handle\");\n\n using acs = d2::task_handle_accessor;\n __TBB_ASSERT(&acs::ctx_of(h) == &context(), \"Attempt to schedule task_handle into different task_group\");\n\n d1::spawn(*acs::release(h), context());\n }\n\n template\n d2::task_handle defer(F&& f) {\n return prepare_task_handle(std::forward(f));\n\n }\n\n template\n task_group_status run_and_wait(const F& f) {\n return internal_run_and_wait(f);\n }\n\n task_group_status run_and_wait(d2::task_handle&& h) {\n return internal_run_and_wait(std::move(h));\n }\n}; // class task_group\n\n#if TBB_PREVIEW_ISOLATED_TASK_GROUP\nclass spawn_delegate : public d1::delegate_base {\n d1::task* task_to_spawn;\n d1::task_group_context& context;\n bool operator()() const override {\n spawn(*task_to_spawn, context);\n return true;\n }\npublic:\n spawn_delegate(d1::task* a_task, d1::task_group_context& ctx)\n : task_to_spawn(a_task), context(ctx)\n {}\n};\n\nclass wait_delegate : public d1::delegate_base {\n bool operator()() const override {\n status = tg.wait();\n return true;\n }\nprotected:\n task_group& tg;\n task_group_status& status;\npublic:\n wait_delegate(task_group& a_group, task_group_status& tgs)\n : tg(a_group), status(tgs) {}\n};\n\ntemplate\nclass run_wait_delegate : public wait_delegate {\n F& func;\n bool operator()() const override {\n status = tg.run_and_wait(func);\n return true;\n }\npublic:\n run_wait_delegate(task_group& a_group, F& a_func, task_group_status& tgs)\n : wait_delegate(a_group, tgs), func(a_func) {}\n};\n\nclass isolated_task_group : public task_group {\n intptr_t this_isolation() {\n return reinterpret_cast(this);\n }\npublic:\n isolated_task_group() : task_group() {}\n\n isolated_task_group(d1::task_group_context& ctx) : task_group(ctx) {}\n\n template\n void run(F&& f) {\n spawn_delegate sd(prepare_task(std::forward(f)), context());\n r1::isolate_within_arena(sd, this_isolation());\n }\n\n void run(d2::task_handle&& h) {\n __TBB_ASSERT(h != nullptr, \"Attempt to schedule empty task_handle\");\n\n using acs = d2::task_handle_accessor;\n __TBB_ASSERT(&acs::ctx_of(h) == &context(), \"Attempt to schedule task_handle into different task_group\");\n\n spawn_delegate sd(acs::release(h), context());\n r1::isolate_within_arena(sd, this_isolation());\n }\n\n template\n task_group_status run_and_wait( const F& f ) {\n task_group_status result = not_complete;\n run_wait_delegate rwd(*this, f, result);\n r1::isolate_within_arena(rwd, this_isolation());\n __TBB_ASSERT(result != not_complete, \"premature exit from wait?\");\n return result;\n }\n\n task_group_status wait() {\n task_group_status result = not_complete;\n wait_delegate wd(*this, result);\n r1::isolate_within_arena(wd, this_isolation());\n __TBB_ASSERT(result != not_complete, \"premature exit from wait?\");\n return result;\n }\n}; // class isolated_task_group\n#endif // TBB_PREVIEW_ISOLATED_TASK_GROUP\n} // namespace d2\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::task_group_context;\nusing detail::d2::task_group;\n#if TBB_PREVIEW_ISOLATED_TASK_GROUP\nusing detail::d2::isolated_task_group;\n#endif\n\nusing detail::d2::task_group_status;\nusing detail::d2::not_complete;\nusing detail::d2::complete;\nusing detail::d2::canceled;\n\nusing detail::d1::is_current_task_group_canceling;\nusing detail::r1::missing_wait;\n\nusing detail::d2::task_handle;\n}\n\n} // namespace tbb\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning(pop) // 4324 warning\n#endif\n\n#endif // __TBB_task_group_H\n" }, { "path": "src/tbb/include/oneapi/tbb/task_scheduler_observer.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_task_scheduler_observer_H\n#define __TBB_task_scheduler_observer_H\n\n#include \n\n#include \"detail/_namespace_injection.h\"\n\n#include \"task_arena.h\"\n\n\nnamespace tbb {\nnamespace detail {\n\nnamespace d1 {\nclass task_scheduler_observer;\n}\n\nnamespace r1 {\nclass observer_proxy;\nclass observer_list;\n\n//! Enable or disable observation\n/** For local observers the method can be used only when the current thread\nhas the task scheduler initialized or is attached to an arena.\nRepeated calls with the same state are no-ops. **/\nTBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool state = true);\n}\n\nnamespace d1 {\nclass task_scheduler_observer {\n friend class r1::observer_proxy;\n friend class r1::observer_list;\n friend void r1::observe(d1::task_scheduler_observer&, bool);\n\n //! Pointer to the proxy holding this observer.\n /** Observers are proxied by the scheduler to maintain persistent lists of them. **/\n std::atomic my_proxy{ nullptr };\n\n //! Counter preventing the observer from being destroyed while in use by the scheduler.\n /** Valid only when observation is on. **/\n std::atomic my_busy_count{ 0 };\n\n //! Contains task_arena pointer\n task_arena* my_task_arena{ nullptr };\npublic:\n //! Returns true if observation is enabled, false otherwise.\n bool is_observing() const { return my_proxy.load(std::memory_order_relaxed) != nullptr; }\n\n //! Entry notification\n /** Invoked from inside observe(true) call and whenever a worker enters the arena\n this observer is associated with. If a thread is already in the arena when\n the observer is activated, the entry notification is called before it\n executes the first stolen task. **/\n virtual void on_scheduler_entry( bool /*is_worker*/ ) {}\n\n //! Exit notification\n /** Invoked from inside observe(false) call and whenever a worker leaves the\n arena this observer is associated with. **/\n virtual void on_scheduler_exit( bool /*is_worker*/ ) {}\n\n //! Construct local or global observer in inactive state (observation disabled).\n /** For a local observer entry/exit notifications are invoked whenever a worker\n thread joins/leaves the arena of the observer's owner thread. If a thread is\n already in the arena when the observer is activated, the entry notification is\n called before it executes the first stolen task. **/\n explicit task_scheduler_observer() = default;\n\n //! Construct local observer for a given arena in inactive state (observation disabled).\n /** entry/exit notifications are invoked whenever a thread joins/leaves arena.\n If a thread is already in the arena when the observer is activated, the entry notification\n is called before it executes the first stolen task. **/\n explicit task_scheduler_observer(task_arena& a) : my_task_arena(&a) {}\n\n /** Destructor protects instance of the observer from concurrent notification.\n It is recommended to disable observation before destructor of a derived class starts,\n otherwise it can lead to concurrent notification callback on partly destroyed object **/\n virtual ~task_scheduler_observer() {\n if (my_proxy.load(std::memory_order_acquire)) {\n observe(false);\n }\n }\n\n //! Enable or disable observation\n /** Warning: concurrent invocations of this method are not safe.\n Repeated calls with the same state are no-ops. **/\n void observe(bool state = true) {\n if( state && !my_proxy.load(std::memory_order_relaxed) ) {\n __TBB_ASSERT( my_busy_count.load(std::memory_order_relaxed) == 0, \"Inconsistent state of task_scheduler_observer instance\");\n }\n r1::observe(*this, state);\n }\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n using detail::d1::task_scheduler_observer;\n}\n} // namespace tbb\n\n\n// Provided for backwards compatibility.\nnamespace tbb {\nnamespace interface6 {\nclass task_scheduler_observer;\n}\nnamespace internal {\n\nclass task_scheduler_observer_v3 {\n friend class tbb::detail::r1::observer_proxy;\n friend class tbb::detail::r1::observer_list;\n friend class interface6::task_scheduler_observer;\n\n //! Pointer to the proxy holding this observer.\n /** Observers are proxied by the scheduler to maintain persistent lists of them. **/\n tbb::detail::r1::observer_proxy* my_proxy;\n\n //! Counter preventing the observer from being destroyed while in use by the scheduler.\n /** Valid only when observation is on. **/\n std::atomic my_busy_count;\n\npublic:\n //! Enable or disable observation\n /** For local observers the method can be used only when the current thread\n has the task scheduler initialized or is attached to an arena.\n\n Repeated calls with the same state are no-ops. **/\n void __TBB_EXPORTED_METHOD observe( bool state=true );\n\n //! Returns true if observation is enabled, false otherwise.\n bool is_observing() const {return my_proxy!=NULL;}\n\n //! Construct observer with observation disabled.\n task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); }\n\n //! Entry notification\n /** Invoked from inside observe(true) call and whenever a worker enters the arena\n this observer is associated with. If a thread is already in the arena when\n the observer is activated, the entry notification is called before it\n executes the first stolen task.\n\n Obsolete semantics. For global observers it is called by a thread before\n the first steal since observation became enabled. **/\n virtual void on_scheduler_entry( bool /*is_worker*/ ) {}\n\n //! Exit notification\n /** Invoked from inside observe(false) call and whenever a worker leaves the\n arena this observer is associated with.\n\n Obsolete semantics. For global observers it is called by a thread before\n the first steal since observation became enabled. **/\n virtual void on_scheduler_exit( bool /*is_worker*/ ) {}\n\n //! Destructor automatically switches observation off if it is enabled.\n virtual ~task_scheduler_observer_v3() { if(my_proxy) observe(false);}\n};\n\n} // namespace internal\n\nnamespace interface6 {\nclass task_scheduler_observer : public internal::task_scheduler_observer_v3 {\n friend class internal::task_scheduler_observer_v3;\n friend class tbb::detail::r1::observer_proxy;\n friend class tbb::detail::r1::observer_list;\n\n /** Negative numbers with the largest absolute value to minimize probability\n of coincidence in case of a bug in busy count usage. **/\n // TODO: take more high bits for version number\n static const intptr_t v6_trait = (intptr_t)((~(uintptr_t)0 >> 1) + 1);\n\n //! contains task_arena pointer or tag indicating local or global semantics of the observer\n intptr_t my_context_tag;\n enum { global_tag = 0, implicit_tag = 1 };\n\npublic:\n //! Construct local or global observer in inactive state (observation disabled).\n /** For a local observer entry/exit notifications are invoked whenever a worker\n thread joins/leaves the arena of the observer's owner thread. If a thread is\n already in the arena when the observer is activated, the entry notification is\n called before it executes the first stolen task. **/\n /** TODO: Obsolete.\n Global observer semantics is obsolete as it violates master thread isolation\n guarantees and is not composable. Thus the current default behavior of the\n constructor is obsolete too and will be changed in one of the future versions\n of the library. **/\n explicit task_scheduler_observer( bool local = false ) {\n my_context_tag = local? implicit_tag : global_tag;\n }\n\n //! Construct local observer for a given arena in inactive state (observation disabled).\n /** entry/exit notifications are invoked whenever a thread joins/leaves arena.\n If a thread is already in the arena when the observer is activated, the entry notification\n is called before it executes the first stolen task. **/\n explicit task_scheduler_observer( task_arena & a) {\n my_context_tag = (intptr_t)&a;\n }\n\n /** Destructor protects instance of the observer from concurrent notification.\n It is recommended to disable observation before destructor of a derived class starts,\n otherwise it can lead to concurrent notification callback on partly destroyed object **/\n virtual ~task_scheduler_observer() { if(my_proxy) observe(false); }\n\n //! Enable or disable observation\n /** Warning: concurrent invocations of this method are not safe.\n Repeated calls with the same state are no-ops. **/\n void observe( bool state=true ) {\n if( state && !my_proxy ) {\n __TBB_ASSERT( !my_busy_count, \"Inconsistent state of task_scheduler_observer instance\");\n my_busy_count.store(v6_trait);\n }\n internal::task_scheduler_observer_v3::observe(state);\n }\n};\n\n} //namespace interface6\n\n} // namespace tbb\n\n#endif /* __TBB_task_scheduler_observer_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/tbb_allocator.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbb_allocator_H\n#define __TBB_tbb_allocator_H\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"detail/_namespace_injection.h\"\n#include \n#include \n\n#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\n\nnamespace r1 {\nTBB_EXPORT void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size);\nTBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_memory(void* p);\nTBB_EXPORT bool __TBB_EXPORTED_FUNC is_tbbmalloc_used();\n}\n\nnamespace d1 {\n\ntemplate\nclass tbb_allocator {\npublic:\n using value_type = T;\n using propagate_on_container_move_assignment = std::true_type;\n\n //! Always defined for TBB containers (supported since C++17 for std containers)\n using is_always_equal = std::true_type;\n\n //! Specifies current allocator\n enum malloc_type {\n scalable,\n standard\n };\n\n tbb_allocator() = default;\n template tbb_allocator(const tbb_allocator&) noexcept {}\n\n //! Allocate space for n objects.\n __TBB_nodiscard T* allocate(std::size_t n) {\n return static_cast(r1::allocate_memory(n * sizeof(value_type)));\n }\n\n //! Free previously allocated block of memory.\n void deallocate(T* p, std::size_t) {\n r1::deallocate_memory(p);\n }\n\n //! Returns current allocator\n static malloc_type allocator_type() {\n return r1::is_tbbmalloc_used() ? standard : scalable;\n }\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n using pointer = value_type*;\n using const_pointer = const value_type*;\n using reference = value_type&;\n using const_reference = const value_type&;\n using difference_type = std::ptrdiff_t;\n using size_type = std::size_t;\n template struct rebind {\n using other = tbb_allocator;\n };\n //! Largest value for which method allocate might succeed.\n size_type max_size() const noexcept {\n size_type max = ~(std::size_t(0)) / sizeof(value_type);\n return (max > 0 ? max : 1);\n }\n template\n void construct(U *p, Args&&... args)\n { ::new (p) U(std::forward(args)...); }\n void destroy( pointer p ) { p->~value_type(); }\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n#endif // TBB_ALLOCATOR_TRAITS_BROKEN\n};\n\n#if TBB_ALLOCATOR_TRAITS_BROKEN\n template<>\n class tbb_allocator {\n public:\n using pointer = void*;\n using const_pointer = const void*;\n using value_type = void;\n template struct rebind {\n using other = tbb_allocator;\n };\n };\n#endif\n\ntemplate\ninline bool operator==(const tbb_allocator&, const tbb_allocator&) noexcept { return true; }\n\n#if !__TBB_CPP20_COMPARISONS_PRESENT\ntemplate\ninline bool operator!=(const tbb_allocator&, const tbb_allocator&) noexcept { return false; }\n#endif\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\nusing detail::d1::tbb_allocator;\n} // namespace v1\n} // namespace tbb\n\n#endif /* __TBB_tbb_allocator_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/tbbmalloc_proxy.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n/*\nReplacing the standard memory allocation routines in Microsoft* C/C++ RTL\n(malloc/free, global new/delete, etc.) with the TBB memory allocator.\n\nInclude the following header to a source of any binary which is loaded during\napplication startup\n\n#include \"oneapi/tbb/tbbmalloc_proxy.h\"\n\nor add following parameters to the linker options for the binary which is\nloaded during application startup. It can be either exe-file or dll.\n\nFor win32\ntbbmalloc_proxy.lib /INCLUDE:\"___TBB_malloc_proxy\"\nwin64\ntbbmalloc_proxy.lib /INCLUDE:\"__TBB_malloc_proxy\"\n*/\n\n#ifndef __TBB_tbbmalloc_proxy_H\n#define __TBB_tbbmalloc_proxy_H\n\n#if _MSC_VER\n\n#ifdef _DEBUG\n #pragma comment(lib, \"tbbmalloc_proxy_debug.lib\")\n#else\n #pragma comment(lib, \"tbbmalloc_proxy.lib\")\n#endif\n\n#if defined(_WIN64)\n #pragma comment(linker, \"/include:__TBB_malloc_proxy\")\n#else\n #pragma comment(linker, \"/include:___TBB_malloc_proxy\")\n#endif\n\n#else\n/* Primarily to support MinGW */\n\nextern \"C\" void __TBB_malloc_proxy();\nstruct __TBB_malloc_proxy_caller {\n __TBB_malloc_proxy_caller() { __TBB_malloc_proxy(); }\n} volatile __TBB_malloc_proxy_helper_object;\n\n#endif // _MSC_VER\n\n/* Public Windows API */\nextern \"C\" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr);\n\n#endif //__TBB_tbbmalloc_proxy_H\n" }, { "path": "src/tbb/include/oneapi/tbb/tick_count.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tick_count_H\n#define __TBB_tick_count_H\n\n#include \n\n#include \"detail/_namespace_injection.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n\n//! Absolute timestamp\n/** @ingroup timing */\nclass tick_count {\npublic:\n using clock_type = typename std::conditional::type;\n\n //! Relative time interval.\n class interval_t : public clock_type::duration {\n public:\n //! Construct a time interval representing zero time duration\n interval_t() : clock_type::duration(clock_type::duration::zero()) {}\n\n //! Construct a time interval representing sec seconds time duration\n explicit interval_t( double sec )\n : clock_type::duration(std::chrono::duration_cast(std::chrono::duration(sec))) {}\n\n //! Return the length of a time interval in seconds\n double seconds() const {\n return std::chrono::duration_cast>(*this).count();\n }\n\n //! Extract the intervals from the tick_counts and subtract them.\n friend interval_t operator-( const tick_count& t1, const tick_count& t0 );\n\n //! Add two intervals.\n friend interval_t operator+( const interval_t& i, const interval_t& j ) {\n return interval_t(std::chrono::operator+(i, j));\n }\n\n //! Subtract two intervals.\n friend interval_t operator-( const interval_t& i, const interval_t& j ) {\n return interval_t(std::chrono::operator-(i, j));\n }\n\n private:\n explicit interval_t( clock_type::duration value_ ) : clock_type::duration(value_) {}\n };\n\n tick_count() = default;\n\n //! Return current time.\n static tick_count now() {\n return clock_type::now();\n }\n\n //! Subtract two timestamps to get the time interval between\n friend interval_t operator-( const tick_count& t1, const tick_count& t0 ) {\n return tick_count::interval_t(t1.my_time_point - t0.my_time_point);\n }\n\n //! Return the resolution of the clock in seconds per tick.\n static double resolution() {\n return static_cast(interval_t::period::num) / interval_t::period::den;\n }\n\nprivate:\n clock_type::time_point my_time_point;\n tick_count( clock_type::time_point tp ) : my_time_point(tp) {}\n};\n\n} // namespace d1\n} // namespace detail\n\ninline namespace v1 {\n using detail::d1::tick_count;\n} // namespace v1\n\n} // namespace tbb\n\n#endif /* __TBB_tick_count_H */\n" }, { "path": "src/tbb/include/oneapi/tbb/version.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_version_H\n#define __TBB_version_H\n\n// Exclude all includes during .rc files compilation\n#ifndef RC_INVOKED\n #include \"detail/_config.h\"\n #include \"detail/_namespace_injection.h\"\n#else\n #define __TBB_STRING_AUX(x) #x\n #define __TBB_STRING(x) __TBB_STRING_AUX(x)\n#endif\n\n// Product version\n#define TBB_VERSION_MAJOR 2022\n// Update version\n#define TBB_VERSION_MINOR 0\n// \"Patch\" version for custom releases\n#define TBB_VERSION_PATCH 0\n// Suffix string\n#define __TBB_VERSION_SUFFIX \"\"\n// Full official version string\n#define TBB_VERSION_STRING \\\n __TBB_STRING(TBB_VERSION_MAJOR) \".\" \\\n __TBB_STRING(TBB_VERSION_MINOR) \".\" \\\n __TBB_STRING(TBB_VERSION_PATCH) \\\n __TBB_VERSION_SUFFIX\n\n// OneAPI oneTBB specification version\n#define ONETBB_SPEC_VERSION \"1.0\"\n// Full interface version\n#define TBB_INTERFACE_VERSION 12140\n// Major interface version\n#define TBB_INTERFACE_VERSION_MAJOR (TBB_INTERFACE_VERSION/1000)\n// Minor interface version\n#define TBB_INTERFACE_VERSION_MINOR (TBB_INTERFACE_VERSION%1000/10)\n\n// The binary compatibility version\n// To be used in SONAME, manifests, etc.\n#define __TBB_BINARY_VERSION 12\n\n//! TBB_VERSION support\n#ifndef TBB_ENDL\n#define TBB_ENDL \"\\n\"\n#endif\n\n//TBB_REVAMP_TODO: consider enabling version_string.ver generation\n//TBB_REVAMP_TODO: #include \"version_string.ver\"\n\n#define __TBB_ONETBB_SPEC_VERSION(N) #N \": SPECIFICATION VERSION\\t\" ONETBB_SPEC_VERSION TBB_ENDL\n#define __TBB_VERSION_NUMBER(N) #N \": VERSION\\t\\t\" TBB_VERSION_STRING TBB_ENDL\n#define __TBB_INTERFACE_VERSION_NUMBER(N) #N \": INTERFACE VERSION\\t\" __TBB_STRING(TBB_INTERFACE_VERSION) TBB_ENDL\n\n#ifndef TBB_USE_DEBUG\n #define __TBB_VERSION_USE_DEBUG(N) #N \": TBB_USE_DEBUG\\tundefined\" TBB_ENDL\n#elif TBB_USE_DEBUG==0\n #define __TBB_VERSION_USE_DEBUG(N) #N \": TBB_USE_DEBUG\\t0\" TBB_ENDL\n#elif TBB_USE_DEBUG==1\n #define __TBB_VERSION_USE_DEBUG(N) #N \": TBB_USE_DEBUG\\t1\" TBB_ENDL\n#elif TBB_USE_DEBUG==2\n #define __TBB_VERSION_USE_DEBUG(N) #N \": TBB_USE_DEBUG\\t2\" TBB_ENDL\n#else\n #error Unexpected value for TBB_USE_DEBUG\n#endif\n\n#ifndef TBB_USE_ASSERT\n #define __TBB_VERSION_USE_ASSERT(N) #N \": TBB_USE_ASSERT\\tundefined\" TBB_ENDL\n#elif TBB_USE_ASSERT==0\n #define __TBB_VERSION_USE_ASSERT(N) #N \": TBB_USE_ASSERT\\t0\" TBB_ENDL\n#elif TBB_USE_ASSERT==1\n #define __TBB_VERSION_USE_ASSERT(N) #N \": TBB_USE_ASSERT\\t1\" TBB_ENDL\n#elif TBB_USE_ASSERT==2\n #define __TBB_VERSION_USE_ASSERT(N) #N \": TBB_USE_ASSERT\\t2\" TBB_ENDL\n#else\n #error Unexpected value for TBB_USE_ASSERT\n#endif\n\n#define TBB_VERSION_STRINGS_P(N) \\\n __TBB_ONETBB_SPEC_VERSION(N) \\\n __TBB_VERSION_NUMBER(N) \\\n __TBB_INTERFACE_VERSION_NUMBER(N) \\\n __TBB_VERSION_USE_DEBUG(N) \\\n __TBB_VERSION_USE_ASSERT(N)\n\n#define TBB_VERSION_STRINGS TBB_VERSION_STRINGS_P(oneTBB)\n#define TBBMALLOC_VERSION_STRINGS TBB_VERSION_STRINGS_P(TBBmalloc)\n\n//! The function returns the version string for the Intel(R) oneAPI Threading Building Blocks (oneTBB)\n//! shared library being used.\n/**\n * The returned pointer is an address of a string in the shared library.\n * It can be different than the TBB_VERSION_STRING obtained at compile time.\n */\nextern \"C\" TBB_EXPORT const char* __TBB_EXPORTED_FUNC TBB_runtime_version();\n\n//! The function returns the interface version of the oneTBB shared library being used.\n/**\n * The returned version is determined at runtime, not at compile/link time.\n * It can be different than the value of TBB_INTERFACE_VERSION obtained at compile time.\n */\nextern \"C\" TBB_EXPORT int __TBB_EXPORTED_FUNC TBB_runtime_interface_version();\n\n#endif // __TBB_version_H\n" }, { "path": "src/tbb/include/oneapi/tbb.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbb_H\n#define __TBB_tbb_H\n\n/**\n This header bulk-includes declarations or definitions of all the functionality\n provided by TBB (save for tbbmalloc and 3rd party dependent headers).\n\n If you use only a few TBB constructs, consider including specific headers only.\n Any header listed below can be included independently of others.\n**/\n\n#include \"oneapi/tbb/blocked_range.h\"\n#include \"oneapi/tbb/blocked_range2d.h\"\n#include \"oneapi/tbb/blocked_range3d.h\"\n#if TBB_PREVIEW_BLOCKED_RANGE_ND\n#include \"tbb/blocked_rangeNd.h\"\n#endif\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/combinable.h\"\n#include \"oneapi/tbb/concurrent_hash_map.h\"\n#if TBB_PREVIEW_CONCURRENT_LRU_CACHE\n#include \"tbb/concurrent_lru_cache.h\"\n#endif\n#include \"oneapi/tbb/collaborative_call_once.h\"\n#include \"oneapi/tbb/concurrent_priority_queue.h\"\n#include \"oneapi/tbb/concurrent_queue.h\"\n#include \"oneapi/tbb/concurrent_unordered_map.h\"\n#include \"oneapi/tbb/concurrent_unordered_set.h\"\n#include \"oneapi/tbb/concurrent_map.h\"\n#include \"oneapi/tbb/concurrent_set.h\"\n#include \"oneapi/tbb/concurrent_vector.h\"\n#include \"oneapi/tbb/enumerable_thread_specific.h\"\n#include \"oneapi/tbb/flow_graph.h\"\n#include \"oneapi/tbb/global_control.h\"\n#include \"oneapi/tbb/info.h\"\n#include \"oneapi/tbb/null_mutex.h\"\n#include \"oneapi/tbb/null_rw_mutex.h\"\n#include \"oneapi/tbb/parallel_for.h\"\n#include \"oneapi/tbb/parallel_for_each.h\"\n#include \"oneapi/tbb/parallel_invoke.h\"\n#include \"oneapi/tbb/parallel_pipeline.h\"\n#include \"oneapi/tbb/parallel_reduce.h\"\n#include \"oneapi/tbb/parallel_scan.h\"\n#include \"oneapi/tbb/parallel_sort.h\"\n#include \"oneapi/tbb/partitioner.h\"\n#include \"oneapi/tbb/queuing_mutex.h\"\n#include \"oneapi/tbb/queuing_rw_mutex.h\"\n#include \"oneapi/tbb/spin_mutex.h\"\n#include \"oneapi/tbb/spin_rw_mutex.h\"\n#include \"oneapi/tbb/mutex.h\"\n#include \"oneapi/tbb/rw_mutex.h\"\n#include \"oneapi/tbb/task.h\"\n#include \"oneapi/tbb/task_arena.h\"\n#include \"oneapi/tbb/task_group.h\"\n#include \"oneapi/tbb/task_scheduler_observer.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/tick_count.h\"\n#include \"oneapi/tbb/version.h\"\n\n#endif /* __TBB_tbb_H */\n" }, { "path": "src/tbb/include/tbb/blocked_range.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/blocked_range.h\"\n" }, { "path": "src/tbb/include/tbb/blocked_range2d.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/blocked_range2d.h\"\n" }, { "path": "src/tbb/include/tbb/blocked_range3d.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/blocked_range3d.h\"\n" }, { "path": "src/tbb/include/tbb/blocked_rangeNd.h", "content": "/*\n Copyright (c) 2017-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/blocked_rangeNd.h\"\n" }, { "path": "src/tbb/include/tbb/cache_aligned_allocator.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/cache_aligned_allocator.h\"\n" }, { "path": "src/tbb/include/tbb/collaborative_call_once.h", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/collaborative_call_once.h\"\n" }, { "path": "src/tbb/include/tbb/combinable.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/combinable.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_hash_map.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_hash_map.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_lru_cache.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_lru_cache.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_map.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_map.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_priority_queue.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_priority_queue.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_queue.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_queue.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_set.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_set.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_unordered_map.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_unordered_map.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_unordered_set.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_unordered_set.h\"\n" }, { "path": "src/tbb/include/tbb/concurrent_vector.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/concurrent_vector.h\"\n" }, { "path": "src/tbb/include/tbb/enumerable_thread_specific.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/enumerable_thread_specific.h\"\n" }, { "path": "src/tbb/include/tbb/flow_graph.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/flow_graph.h\"\n" }, { "path": "src/tbb/include/tbb/flow_graph_abstractions.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/flow_graph_abstractions.h\"\n" }, { "path": "src/tbb/include/tbb/global_control.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/global_control.h\"\n" }, { "path": "src/tbb/include/tbb/info.h", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/info.h\"\n" }, { "path": "src/tbb/include/tbb/memory_pool.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/memory_pool.h\"\n" }, { "path": "src/tbb/include/tbb/mutex.h", "content": "/*\n Copyright (c) 2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/mutex.h\"\n" }, { "path": "src/tbb/include/tbb/null_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/null_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/null_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/null_rw_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_for.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_for.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_for_each.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_for_each.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_invoke.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_invoke.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_pipeline.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_pipeline.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_reduce.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_reduce.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_scan.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_scan.h\"\n" }, { "path": "src/tbb/include/tbb/parallel_sort.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/parallel_sort.h\"\n" }, { "path": "src/tbb/include/tbb/partitioner.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/partitioner.h\"\n" }, { "path": "src/tbb/include/tbb/profiling.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/profiling.h\"\n" }, { "path": "src/tbb/include/tbb/queuing_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/queuing_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/queuing_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/queuing_rw_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/rw_mutex.h", "content": "/*\n Copyright (c) 2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/rw_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/scalable_allocator.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/scalable_allocator.h\"\n" }, { "path": "src/tbb/include/tbb/spin_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/spin_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/spin_rw_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/spin_rw_mutex.h\"\n" }, { "path": "src/tbb/include/tbb/task.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/task.h\"\n" }, { "path": "src/tbb/include/tbb/task_arena.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/task_arena.h\"\n" }, { "path": "src/tbb/include/tbb/task_group.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/task_group.h\"\n" }, { "path": "src/tbb/include/tbb/task_scheduler_observer.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/task_scheduler_observer.h\"\n" }, { "path": "src/tbb/include/tbb/tbb.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb.h\"\n" }, { "path": "src/tbb/include/tbb/tbb_allocator.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/tbb_allocator.h\"\n" }, { "path": "src/tbb/include/tbb/tbbmalloc_proxy.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/tbbmalloc_proxy.h\"\n" }, { "path": "src/tbb/include/tbb/tick_count.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/tick_count.h\"\n" }, { "path": "src/tbb/include/tbb/version.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"../oneapi/tbb/version.h\"\n" }, { "path": "src/tbb/integration/cmake/generate_vars.cmake", "content": "# Copyright (c) 2020-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Reuired parameters:\n# SOURCE_DIR - incoming path to oneTBB source directory.\n# BINARY_DIR - incoming path to oneTBB build directory.\n# BIN_PATH - incoming path to oneTBB binaries directory.\n# TBB_INSTALL_VARS - install vars generation trigger\n# TBB_CMAKE_INSTALL_LIBDIR - subdir for shared object files installation path (used only in TBB_INSTALL_VARS mode)\n# VARS_TEMPLATE - path to the vars template file\n# VARS_NAME - name of the output vars script\n\nset(INPUT_FILE \"${SOURCE_DIR}/integration/${VARS_TEMPLATE}\")\nset(OUTPUT_FILE \"${BIN_PATH}/${VARS_NAME}\")\n\nfile(TO_NATIVE_PATH \"${SOURCE_DIR}\" TBBROOT_REPLACEMENT)\nfile(TO_NATIVE_PATH \"${BIN_PATH}\" LIBRARY_PATH_REPLACEMENT)\nif (WIN32)\n file(TO_NATIVE_PATH \"${BIN_PATH}\" BINARY_PATH_REPLACEMENT)\nendif()\n\nif (NOT EXISTS ${OUTPUT_FILE})\n configure_file(${INPUT_FILE} ${OUTPUT_FILE} @ONLY)\nendif()\n\nif (TBB_INSTALL_VARS)\n set(OUTPUT_FILE \"${BINARY_DIR}/internal_install_vars\")\n if (UNIX)\n set(TBBROOT_REPLACEMENT \"$(cd $(dirname \\${BASH_SOURCE}) && pwd -P)/..\")\n set(LIBRARY_PATH_REPLACEMENT \"$TBBROOT/${TBB_CMAKE_INSTALL_LIBDIR}/\")\n set(CMAKE_ENVIRONMENT_SOURCING_STRING \"CMAKE_PREFIX_PATH=\\\"\\${TBBROOT}/${TBB_CMAKE_INSTALL_LIBDIR}/cmake/TBB:${CMAKE_PREFIX_PATH}\\\"; export CMAKE_PREFIX_PATH\")\n else()\n set(TBBROOT_REPLACEMENT \"%~d0%~p0..\")\n set(LIBRARY_PATH_REPLACEMENT \"%TBBROOT%\\\\${TBB_CMAKE_INSTALL_LIBDIR}\")\n set(BINARY_PATH_REPLACEMENT \"%TBBROOT%\\\\bin\")\n set(CMAKE_ENVIRONMENT_SOURCING_STRING \"set \\\"CMAKE_PREFIX_PATH=%TBBROOT%\\\\${TBB_CMAKE_INSTALL_LIBDIR}\\\\cmake\\\\TBB;%CMAKE_PREFIX_PATH%\\\"\")\n endif()\n\n configure_file( ${INPUT_FILE} ${OUTPUT_FILE} @ONLY )\nendif()\n" }, { "path": "src/tbb/integration/linux/env/vars.sh", "content": "#!/bin/sh\n# shellcheck shell=sh\n#\n# Copyright (c) 2005-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The script is setting up environment for oneTBB.\n# Supported arguments:\n# intel64|ia32 - architecture, intel64 is default.\n\n# Get absolute path to script. Gets a relative path as argument and outputs an absolute path.\nget_script_path() (\n script_path=\"$1\"\n while [ -L \"$script_path\" ] ; do\n script_dir=$(command dirname -- \"$script_path\")\n script_dir=$(cd \"$script_dir\" && command pwd -P)\n script_path=\"$(readlink \"$script_path\")\"\n case $script_path in\n (/*) ;;\n (*) script_path=\"$script_dir/$script_path\" ;;\n esac\n done\n script_dir=$(command dirname -- \"$script_path\")\n script_dir=$(cd \"$script_dir\" && command pwd -P)\n printf \"%s\" \"$script_dir\"\n)\n\n_vars_get_proc_name() {\n if [ -n \"${ZSH_VERSION:-}\" ] ; then\n script=\"$(ps -p \"$$\" -o comm=)\"\n else\n script=\"$1\"\n while [ -L \"$script\" ] ; do\n script=\"$(readlink \"$script\")\"\n done\n fi\n basename -- \"$script\"\n}\n\n_vars_this_script_name=\"vars.sh\"\nif [ \"$_vars_this_script_name\" = \"$(_vars_get_proc_name \"$0\")\" ] ; then\n echo \":: ERROR: Incorrect usage: this script must be sourced.\"\n echo \" Usage: . path/to/${_vars_this_script_name}\"\n return 255 2>/dev/null || exit 255\nfi\n\n# Prepend path segment(s) to path-like env vars (PATH, CPATH, etc.).\n\n# prepend_path() avoids dangling \":\" that affects some env vars (PATH and CPATH)\n# PATH > https://www.gnu.org/software/libc/manual/html_node/Standard-Environment.html\n\n# Usage:\n# env_var=$(prepend_path \"$prepend_to_var\" \"$existing_env_var\")\n# export env_var\n#\n# Inputs:\n# $1 == path segment to be prepended to $2\n# $2 == value of existing path-like environment variable\n\nprepend_path() (\n path_to_add=\"$1\"\n path_is_now=\"$2\"\n\n if [ \"\" = \"${path_is_now}\" ] ; then # avoid dangling \":\"\n printf \"%s\" \"${path_to_add}\"\n else\n printf \"%s\" \"${path_to_add}:${path_is_now}\"\n fi\n)\n\n# Extract the name and location of this sourced script.\n\n# Generally, \"ps -o comm=\" is limited to a 15 character result, but it works\n# fine for this usage, because we are primarily interested in finding the name\n# of the execution shell, not the name of any calling script.\n\nvars_script_name=\"\"\nvars_script_shell=\"$(ps -p \"$$\" -o comm=)\"\n# ${var:-} needed to pass \"set -eu\" checks\nif [ -n \"${ZSH_VERSION:-}\" ] && [ -n \"${ZSH_EVAL_CONTEXT:-}\" ] ; then # zsh 5.x and later\n # shellcheck disable=2249\n case $ZSH_EVAL_CONTEXT in (*:file*) vars_script_name=\"${(%):-%x}\" ;; esac ;\nelif [ -n \"${KSH_VERSION:-}\" ] ; then # ksh, mksh or lksh\n if [ \"$(set | grep -Fq \"KSH_VERSION=.sh.version\" ; echo $?)\" -eq 0 ] ; then # ksh\n vars_script_name=\"${.sh.file}\" ;\n else # mksh or lksh or [lm]ksh masquerading as ksh or sh\n # force [lm]ksh to issue error msg; which contains this script's path/filename, e.g.:\n # mksh: /home/ubuntu/intel/oneapi/vars.sh[137]: ${.sh.file}: bad substitution\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*sh: \\(.*\\)\\[[0-9]*\\]:')\" ;\n fi\nelif [ -n \"${BASH_VERSION:-}\" ] ; then # bash\n # shellcheck disable=2128\n (return 0 2>/dev/null) && vars_script_name=\"${BASH_SOURCE}\" ;\nelif [ \"dash\" = \"$vars_script_shell\" ] ; then # dash\n # force dash to issue error msg; which contains this script's rel/path/filename, e.g.:\n # dash: 146: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*dash: [0-9]*: \\(.*\\):')\" ;\nelif [ \"sh\" = \"$vars_script_shell\" ] ; then # could be dash masquerading as /bin/sh\n # force a shell error msg; which should contain this script's path/filename\n # sample error msg shown; assume this file is named \"vars.sh\"; as required by setvars.sh\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n if [ \"$(printf \"%s\" \"$vars_script_name\" | grep -Eq \"sh: [0-9]+: .*vars\\.sh: \" ; echo $?)\" -eq 0 ] ; then # dash as sh\n # sh: 155: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*sh: [0-9]*: \\(.*\\):')\" ;\n fi\nelse # unrecognized shell or dash being sourced from within a user's script\n # force a shell error msg; which should contain this script's path/filename\n # sample error msg shown; assume this file is named \"vars.sh\"; as required by setvars.sh\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n if [ \"$(printf \"%s\" \"$vars_script_name\" | grep -Eq \"^.+: [0-9]+: .*vars\\.sh: \" ; echo $?)\" -eq 0 ] ; then # dash\n # .*: 164: intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*: [0-9]*: \\(.*\\):')\" ;\n else\n vars_script_name=\"\" ;\n fi\nfi\n\nif [ \"\" = \"$vars_script_name\" ] ; then\n >&2 echo \":: ERROR: Unable to proceed: possible causes listed below.\"\n >&2 echo \" This script must be sourced. Did you execute or source this script?\" ;\n >&2 echo \" Unrecognized/unsupported shell (supported: bash, zsh, ksh, m/lksh, dash).\" ;\n >&2 echo \" Can be caused by sourcing from ZSH version 4.x or older.\" ;\n return 255 2>/dev/null || exit 255\nfi\n\nTBBROOT=$(get_script_path \"${vars_script_name:-}\")/..\n\nTBB_TARGET_ARCH=\"intel64\"\nTBB_ARCH_SUFFIX=\"\"\n\nif [ -n \"${SETVARS_ARGS:-}\" ]; then\n tbb_arg_ia32=\"$(expr \"${SETVARS_ARGS:-}\" : '^.*\\(ia32\\)')\" || true\n if [ -n \"${tbb_arg_ia32:-}\" ]; then\n TBB_TARGET_ARCH=\"ia32\"\n fi\nelse\n for arg do\n case \"$arg\" in\n (intel64|ia32)\n TBB_TARGET_ARCH=\"${arg}\"\n ;;\n (*) ;;\n esac\n done\nfi\n\nTBB_LIB_NAME=\"libtbb.so.12\"\n\n# Parse layout\nif [ -e \"$TBBROOT/lib/$TBB_TARGET_ARCH\" ]; then\n TBB_LIB_DIR=\"$TBB_TARGET_ARCH/gcc4.8\"\nelse\n if [ \"$TBB_TARGET_ARCH\" = \"ia32\" ] ; then\n TBB_ARCH_SUFFIX=\"32\"\n fi\n TBB_LIB_DIR=\"\"\nfi\n\nif [ -e \"$TBBROOT/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR/$TBB_LIB_NAME\" ]; then\n export TBBROOT\n\n LIBRARY_PATH=$(prepend_path \"${TBBROOT}/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR\" \"${LIBRARY_PATH:-}\") ; export LIBRARY_PATH\n LD_LIBRARY_PATH=$(prepend_path \"${TBBROOT}/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR\" \"${LD_LIBRARY_PATH:-}\") ; export LD_LIBRARY_PATH\n CPATH=$(prepend_path \"${TBBROOT}/include\" \"${CPATH:-}\") ; export CPATH\n CMAKE_PREFIX_PATH=$(prepend_path \"${TBBROOT}\" \"${CMAKE_PREFIX_PATH:-}\") ; export CMAKE_PREFIX_PATH\n PKG_CONFIG_PATH=$(prepend_path \"${TBBROOT}/lib$TBB_ARCH_SUFFIX/pkgconfig\" \"${PKG_CONFIG_PATH:-}\") ; export PKG_CONFIG_PATH\nelse\n >&2 echo \"ERROR: $TBB_LIB_NAME library does not exist in $TBBROOT/lib$TBB_ARCH_SUFFIX/$TBB_LIB_DIR.\"\n return 255 2>/dev/null || exit 255\nfi\n" }, { "path": "src/tbb/integration/linux/env/vars.sh.in", "content": "#!/bin/sh\n#\n# Copyright (c) 2005-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport TBBROOT=@TBBROOT_REPLACEMENT@\n\nLD_LIBRARY_PATH=\"@LIBRARY_PATH_REPLACEMENT@:${LD_LIBRARY_PATH}\"; export LD_LIBRARY_PATH\nLIBRARY_PATH=\"@LIBRARY_PATH_REPLACEMENT@:${LIBRARY_PATH}\"; export LIBRARY_PATH\nCPATH=\"${TBBROOT}/include:${CPATH}\"; export CPATH\nPKG_CONFIG_PATH=\"@LIBRARY_PATH_REPLACEMENT@/pkgconfig:${PKG_CONFIG_PATH}\"; export PKG_CONFIG_PATH\n\n@CMAKE_ENVIRONMENT_SOURCING_STRING@\n" }, { "path": "src/tbb/integration/linux/modulefiles/tbb", "content": "#%Module1.0###################################################################\n#\n# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This modulefile requires Environment Modules 4.1 or later.\n# Type `module --version` to determine the current installed version.\n\n##############################################################################\n\nset min_tcl_ver 8.4\nif { $tcl_version < $min_tcl_ver } {\n puts stderr \" \"\n puts stderr \"ERROR: This modulefile requires tcl $min_tcl_ver or greater.\"\n puts stderr \"Your system reports that tclsh version $tcl_version is installed.\"\n exit 1\n}\n\n# if modulefile script name is a symlink, resolve it to get the fully\n# qualified pathname that points to the actual modulefile script\n# see: https://wiki.tcl-lang.org/page/file+normalize\nset scriptpath \"${ModulesCurrentModulefile}\"\nset scriptpath \"[file dirname [file normalize \"$scriptpath/___\"]]\"\n\n# define componentroot, modulefilepath, modulefilename and modulefilever\nset modulefilename \"[file tail [file dirname \"${scriptpath}\"]]\"\nset modulefilever \"[file tail \"${scriptpath}\"]\"\nset modulefilepath \"${scriptpath}\"\nset componentroot \"[file dirname [file dirname [file dirname [file dirname \"${scriptpath}\"]]]]\"\n\n##############################################################################\n\nmodule-whatis \"Name: Intel(R) oneAPI Threading Building Blocks\"\nmodule-whatis \"Version: $modulefilename/$modulefilever\"\nmodule-whatis \"Description: Flexible threading library for adding parallelism to complex applications across accelerated architectures.\"\nmodule-whatis \"URL: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onetbb.html\"\nmodule-whatis \"Dependencies: none\"\n\nproc ModulesHelp { } {\n global modulefilename\n global modulefilever\n module whatis \"${modulefilename}/${modulefilever}\"\n}\n\n##############################################################################\n\n# Define environment variables needed for an isolated component install.\n\nset tbbroot \"$componentroot\"\nset tbb_target_arch \"intel64\"\n\nsetenv TBBROOT \"$tbbroot\"\n\nprepend-path CPATH \"$tbbroot/include\"\nprepend-path LIBRARY_PATH \"$tbbroot/lib\"\nprepend-path LD_LIBRARY_PATH \"$tbbroot/lib\"\nprepend-path CMAKE_PREFIX_PATH \"$tbbroot\"\nprepend-path PKG_CONFIG_PATH \"$tbbroot/lib/pkgconfig\"\n" }, { "path": "src/tbb/integration/linux/modulefiles/tbb32", "content": "#%Module1.0###################################################################\n#\n# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This modulefile requires Environment Modules 4.1 or later.\n# Type `module --version` to determine the current installed version.\n\n##############################################################################\n\nset min_tcl_ver 8.4\nif { $tcl_version < $min_tcl_ver } {\n puts stderr \" \"\n puts stderr \"ERROR: This modulefile requires tcl $min_tcl_ver or greater.\"\n puts stderr \"Your system reports that tclsh version $tcl_version is installed.\"\n exit 1\n}\n\n# if modulefile script name is a symlink, resolve it to get the fully\n# qualified pathname that points to the actual modulefile script\n# see: https://wiki.tcl-lang.org/page/file+normalize\nset scriptpath \"${ModulesCurrentModulefile}\"\nset scriptpath \"[file dirname [file normalize \"$scriptpath/___\"]]\"\n\n# define componentroot, modulefilepath, modulefilename and modulefilever\nset modulefilename \"[file tail [file dirname \"${scriptpath}\"]]\"\nset modulefilever \"[file tail \"${scriptpath}\"]\"\nset modulefilepath \"${scriptpath}\"\nset componentroot \"[file dirname [file dirname [file dirname [file dirname \"${scriptpath}\"]]]]\"\n\n##############################################################################\n\nmodule-whatis \"Name: Intel(R) oneAPI Threading Building Blocks\"\nmodule-whatis \"Version: $modulefilename/$modulefilever\"\nmodule-whatis \"Description: Flexible threading library for adding parallelism to complex applications across accelerated architectures.\"\nmodule-whatis \"URL: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onetbb.html\"\nmodule-whatis \"Dependencies: none\"\n\nproc ModulesHelp { } {\n global modulefilename\n global modulefilever\n module whatis \"${modulefilename}/${modulefilever}\"\n}\n\n##############################################################################\n\n# Define environment variables needed for an isolated component install.\n\nset tbbroot \"$componentroot\"\nset tbb_target_arch \"ia32\"\n\nsetenv TBBROOT \"$tbbroot\"\n\nprepend-path CPATH \"$tbbroot/include32:$tbbroot/include\"\nprepend-path LIBRARY_PATH \"$tbbroot/lib32\"\nprepend-path LD_LIBRARY_PATH \"$tbbroot/lib32\"\nprepend-path CMAKE_PREFIX_PATH \"$tbbroot\"\nprepend-path PKG_CONFIG_PATH \"$tbbroot/lib32/pkgconfig\"\n" }, { "path": "src/tbb/integration/linux/oneapi/vars.sh", "content": "#!/bin/sh\n# shellcheck shell=sh\n#\n# Copyright (c) 2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"${SETVARS_CALL:-}\" ] ; then\n >&2 echo \" \"\n >&2 echo \":: ERROR: This script must be sourced by setvars.sh.\"\n >&2 echo \" Try 'source /setvars.sh --help' for help.\"\n >&2 echo \" \"\n return 255\nfi\n\nif [ -z \"${ONEAPI_ROOT:-}\" ] ; then\n >&2 echo \" \"\n >&2 echo \":: ERROR: This script requires that the ONEAPI_ROOT env variable is set.\"\n >&2 echo \" Try 'source \\setvars.sh --help' for help.\"\n >&2 echo \" \"\n return 254\nfi\n\nTBBROOT=\"${ONEAPI_ROOT}\"; export TBBROOT\n" }, { "path": "src/tbb/integration/linux/sys_check/sys_check.sh", "content": "#!/bin/sh\n#\n# Copyright (c) 2019-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nLOC=$(realpath $(dirname \"${BASH_SOURCE[0]}\"))\nsource $LOC/../../../common.sh $@\n\nERRORSTATE=0\nreturn $ERRORSTATE\n" }, { "path": "src/tbb/integration/mac/env/vars.sh", "content": "#!/bin/sh\n# shellcheck shell=sh\n#\n# Copyright (c) 2005-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Get absolute path to script. Gets a relative path as argument and outputs an absolute path.\nget_script_path() (\n script_path=\"$1\"\n while [ -L \"$script_path\" ] ; do\n script_dir=$(command dirname -- \"$script_path\")\n script_dir=$(cd \"$script_dir\" && command pwd -P)\n script_path=\"$(readlink \"$script_path\")\"\n case $script_path in\n (/*) ;;\n (*) script_path=\"$script_dir/$script_path\" ;;\n esac\n done\n script_dir=$(command dirname -- \"$script_path\")\n script_dir=$(cd \"$script_dir\" && command pwd -P)\n printf \"%s\" \"$script_dir\"\n)\n\n_vars_get_proc_name() {\n if [ -n \"${ZSH_VERSION:-}\" ] ; then\n script=\"$(ps -p \"$$\" -o comm=)\"\n else\n script=\"$1\"\n while [ -L \"$script\" ] ; do\n script=\"$(readlink \"$script\")\"\n done\n fi\n basename -- \"$script\"\n}\n\n_vars_this_script_name=\"vars.sh\"\nif [ \"$_vars_this_script_name\" = \"$(_vars_get_proc_name \"$0\")\" ] ; then\n echo \":: ERROR: Incorrect usage: this script must be sourced.\"\n echo \" Usage: . path/to/${_vars_this_script_name}\"\n return 255 2>/dev/null || exit 255\nfi\n\n# Prepend path segment(s) to path-like env vars (PATH, CPATH, etc.).\n\n# prepend_path() avoids dangling \":\" that affects some env vars (PATH and CPATH)\n# PATH > https://www.gnu.org/software/libc/manual/html_node/Standard-Environment.html\n\n# Usage:\n# env_var=$(prepend_path \"$prepend_to_var\" \"$existing_env_var\")\n# export env_var\n#\n# Inputs:\n# $1 == path segment to be prepended to $2\n# $2 == value of existing path-like environment variable\n\nprepend_path() (\n path_to_add=\"$1\"\n path_is_now=\"$2\"\n\n if [ \"\" = \"${path_is_now}\" ] ; then # avoid dangling \":\"\n printf \"%s\" \"${path_to_add}\"\n else\n printf \"%s\" \"${path_to_add}:${path_is_now}\"\n fi\n)\n\n# Extract the name and location of this sourced script.\n\n# Generally, \"ps -o comm=\" is limited to a 15 character result, but it works\n# fine for this usage, because we are primarily interested in finding the name\n# of the execution shell, not the name of any calling script.\n\nvars_script_name=\"\"\nvars_script_shell=\"$(ps -p \"$$\" -o comm=)\"\n# ${var:-} needed to pass \"set -eu\" checks\nif [ -n \"${ZSH_VERSION:-}\" ] && [ -n \"${ZSH_EVAL_CONTEXT:-}\" ] ; then # zsh 5.x and later\n # shellcheck disable=2249\n case $ZSH_EVAL_CONTEXT in (*:file*) vars_script_name=\"${(%):-%x}\" ;; esac ;\nelif [ -n \"${KSH_VERSION:-}\" ] ; then # ksh, mksh or lksh\n if [ \"$(set | grep -Fq \"KSH_VERSION=.sh.version\" ; echo $?)\" -eq 0 ] ; then # ksh\n vars_script_name=\"${.sh.file}\" ;\n else # mksh or lksh or [lm]ksh masquerading as ksh or sh\n # force [lm]ksh to issue error msg; which contains this script's path/filename, e.g.:\n # mksh: /home/ubuntu/intel/oneapi/vars.sh[137]: ${.sh.file}: bad substitution\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*sh: \\(.*\\)\\[[0-9]*\\]:')\" ;\n fi\nelif [ -n \"${BASH_VERSION:-}\" ] ; then # bash\n # shellcheck disable=2128\n (return 0 2>/dev/null) && vars_script_name=\"${BASH_SOURCE}\" ;\nelif [ \"dash\" = \"$vars_script_shell\" ] ; then # dash\n # force dash to issue error msg; which contains this script's rel/path/filename, e.g.:\n # dash: 146: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*dash: [0-9]*: \\(.*\\):')\" ;\nelif [ \"sh\" = \"$vars_script_shell\" ] ; then # could be dash masquerading as /bin/sh\n # force a shell error msg; which should contain this script's path/filename\n # sample error msg shown; assume this file is named \"vars.sh\"; as required by setvars.sh\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n if [ \"$(printf \"%s\" \"$vars_script_name\" | grep -Eq \"sh: [0-9]+: .*vars\\.sh: \" ; echo $?)\" -eq 0 ] ; then # dash as sh\n # sh: 155: /home/ubuntu/intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*sh: [0-9]*: \\(.*\\):')\" ;\n fi\nelse # unrecognized shell or dash being sourced from within a user's script\n # force a shell error msg; which should contain this script's path/filename\n # sample error msg shown; assume this file is named \"vars.sh\"; as required by setvars.sh\n vars_script_name=\"$( (echo \"${.sh.file}\") 2>&1 )\" || : ;\n if [ \"$(printf \"%s\" \"$vars_script_name\" | grep -Eq \"^.+: [0-9]+: .*vars\\.sh: \" ; echo $?)\" -eq 0 ] ; then # dash\n # .*: 164: intel/oneapi/vars.sh: Bad substitution\n vars_script_name=\"$(expr \"${vars_script_name:-}\" : '^.*: [0-9]*: \\(.*\\):')\" ;\n else\n vars_script_name=\"\" ;\n fi\nfi\n\nif [ \"\" = \"$vars_script_name\" ] ; then\n >&2 echo \":: ERROR: Unable to proceed: possible causes listed below.\"\n >&2 echo \" This script must be sourced. Did you execute or source this script?\" ;\n >&2 echo \" Unrecognized/unsupported shell (supported: bash, zsh, ksh, m/lksh, dash).\" ;\n >&2 echo \" Can be caused by sourcing from ZSH version 4.x or older.\" ;\n return 255 2>/dev/null || exit 255\nfi\n\nTBBROOT=$(get_script_path \"${vars_script_name:-}\")/..\nLIBTBB_NAME=\"libtbb.dylib\"\n\nif [ -e \"$TBBROOT/lib/$LIBTBB_NAME\" ]; then\n export TBBROOT\n\n LIBRARY_PATH=$(prepend_path \"${TBBROOT}/lib\" \"${LIBRARY_PATH:-}\") ; export LIBRARY_PATH\n DYLD_LIBRARY_PATH=$(prepend_path \"${TBBROOT}/lib\" \"${DYLD_LIBRARY_PATH:-}\") ; export DYLD_LIBRARY_PATH\n CPATH=$(prepend_path \"${TBBROOT}/include\" \"${CPATH:-}\") ; export CPATH\n CMAKE_PREFIX_PATH=$(prepend_path \"${TBBROOT}\" \"${CMAKE_PREFIX_PATH:-}\") ; export CMAKE_PREFIX_PATH\n PKG_CONFIG_PATH=$(prepend_path \"${TBBROOT}/lib/pkgconfig\" \"${PKG_CONFIG_PATH:-}\") ; export PKG_CONFIG_PATH\nelse\n >&2 echo \"ERROR: $LIBTBB_NAME library does not exist in $TBBROOT/lib.\"\n return 255 2>/dev/null || exit 255\nfi\n" }, { "path": "src/tbb/integration/mac/env/vars.sh.in", "content": "#!/bin/sh\n#\n# Copyright (c) 2005-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport TBBROOT=@TBBROOT_REPLACEMENT@\n\nDYLD_LIBRARY_PATH=\"@LIBRARY_PATH_REPLACEMENT@:${DYLD_LIBRARY_PATH}\"; export DYLD_LIBRARY_PATH\nLIBRARY_PATH=\"@LIBRARY_PATH_REPLACEMENT@:${LIBRARY_PATH}\"; export LIBRARY_PATH\nCPATH=\"${TBBROOT}/include:${CPATH}\"; export CPATH\nPKG_CONFIG_PATH=\"@LIBRARY_PATH_REPLACEMENT@/pkgconfig:${PKG_CONFIG_PATH}\"; export PKG_CONFIG_PATH\n\n@CMAKE_ENVIRONMENT_SOURCING_STRING@\n" }, { "path": "src/tbb/integration/pkg-config/tbb.pc.in", "content": "# Copyright (c) 2021-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nprefix=@_prefix_for_pc_file@\nlibdir=@_libdir_for_pc_file@\nincludedir=@_includedir_for_pc_file@\n\nName: oneAPI Threading Building Blocks (oneTBB)\nDescription: C++ library for parallel programming on multi-core processors.\nURL: https://github.com/oneapi-src/oneTBB\nVersion: @TBB_VERSION@\nLibs: -L${libdir} @_tbb_pc_extra_libdir@ -l@_tbb_pc_lib_name@\nCflags: -I${includedir}\n" }, { "path": "src/tbb/integration/windows/env/vars.bat", "content": "@echo off\nREM\nREM Copyright (c) 2005-2023 Intel Corporation\nREM\nREM Licensed under the Apache License, Version 2.0 (the \"License\");\nREM you may not use this file except in compliance with the License.\nREM You may obtain a copy of the License at\nREM\nREM http://www.apache.org/licenses/LICENSE-2.0\nREM\nREM Unless required by applicable law or agreed to in writing, software\nREM distributed under the License is distributed on an \"AS IS\" BASIS,\nREM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nREM See the License for the specific language governing permissions and\nREM limitations under the License.\nREM\n\nREM Syntax:\nREM %SCRIPT_NAME% [^] [^]\nREM ^ should be one of the following\nREM ia32 : Set up for IA-32 architecture\nREM intel64 : Set up for Intel(R) 64 architecture\nREM if ^ is not set Intel(R) 64 architecture will be used\nREM ^ should be one of the following\nREM vs2019 : Set to use with Microsoft Visual Studio 2019 runtime DLLs\nREM vs2022 : Set to use with Microsoft Visual Studio 2022 runtime DLLs\nREM all : Set to use oneTBB statically linked with Microsoft Visual C++ runtime\nREM if ^ is not set oneTBB dynamically linked with Microsoft Visual C++ runtime will be used.\n\nset \"SCRIPT_NAME=%~nx0\"\nset \"TBB_SCRIPT_DIR=%~d0%~p0\"\nset \"TBBROOT=%TBB_SCRIPT_DIR%..\"\n\n:: Set the default arguments\nset TBB_TARGET_ARCH=intel64\nset TBB_ARCH_SUFFIX=\nset TBB_TARGET_VS=vc14\n\n:ParseArgs\n:: Parse the incoming arguments\nif /i \"%1\"==\"\" goto ParseLayout\nif /i \"%1\"==\"ia32\" (set TBB_TARGET_ARCH=ia32) & shift & goto ParseArgs\nif /i \"%1\"==\"intel64\" (set TBB_TARGET_ARCH=intel64) & shift & goto ParseArgs\nif /i \"%1\"==\"vs2019\" (set TBB_TARGET_VS=vc14) & shift & goto ParseArgs\nif /i \"%1\"==\"vs2022\" (set TBB_TARGET_VS=vc14) & shift & goto ParseArgs\nif /i \"%1\"==\"all\" (set TBB_TARGET_VS=vc_mt) & shift & goto ParseArgs\n\n:ParseLayout\nif exist \"%TBBROOT%\\redist\\\" (\n set \"TBB_BIN_DIR=%TBBROOT%\\redist\"\n set \"TBB_SUBDIR=%TBB_TARGET_ARCH%\"\n goto SetEnv\n)\n\nif \"%TBB_TARGET_ARCH%\" == \"ia32\" (\n set TBB_ARCH_SUFFIX=32\n)\nif exist \"%TBBROOT%\\bin%TBB_ARCH_SUFFIX%\" (\n set \"TBB_BIN_DIR=%TBBROOT%\\bin%TBB_ARCH_SUFFIX%\"\n if \"%TBB_TARGET_VS%\" == \"vc14\" (\n set TBB_TARGET_VS=\n )\n goto SetEnv\n)\n:: Couldn't parse TBBROOT/bin, unset variable\nset TBB_ARCH_SUFFIX=\n\nif exist \"%TBBROOT%\\..\\redist\\\" (\n set \"TBB_BIN_DIR=%TBBROOT%\\..\\redist\"\n set \"TBB_SUBDIR=%TBB_TARGET_ARCH%\\tbb\"\n goto SetEnv\n)\n\n:SetEnv\nif exist \"%TBB_BIN_DIR%\\%TBB_SUBDIR%\\%TBB_TARGET_VS%\\tbb12.dll\" (\n set \"TBB_DLL_PATH=%TBB_BIN_DIR%\\%TBB_SUBDIR%\\%TBB_TARGET_VS%\"\n) else (\n echo:\n echo :: ERROR: tbb12.dll library does not exist in \"%TBB_BIN_DIR%\\%TBB_SUBDIR%\\%TBB_TARGET_VS%\\\"\n echo:\n exit /b 255\n)\n\nset \"PATH=%TBB_DLL_PATH%;%PATH%\"\n\nset \"LIB=%TBBROOT%\\lib%TBB_ARCH_SUFFIX%\\%TBB_SUBDIR%\\%TBB_TARGET_VS%;%LIB%\"\nset \"INCLUDE=%TBBROOT%\\include;%INCLUDE%\"\nset \"CPATH=%TBBROOT%\\include;%CPATH%\"\nset \"CMAKE_PREFIX_PATH=%TBBROOT%;%CMAKE_PREFIX_PATH%\"\nset \"PKG_CONFIG_PATH=%TBBROOT%\\lib%TBB_ARCH_SUFFIX%\\pkgconfig;%PKG_CONFIG_PATH%\"\n\n:End\nexit /B 0\n" }, { "path": "src/tbb/integration/windows/env/vars.bat.in", "content": "@echo off\nREM\nREM Copyright (c) 2005-2021 Intel Corporation\nREM\nREM Licensed under the Apache License, Version 2.0 (the \"License\");\nREM you may not use this file except in compliance with the License.\nREM You may obtain a copy of the License at\nREM\nREM http://www.apache.org/licenses/LICENSE-2.0\nREM\nREM Unless required by applicable law or agreed to in writing, software\nREM distributed under the License is distributed on an \"AS IS\" BASIS,\nREM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nREM See the License for the specific language governing permissions and\nREM limitations under the License.\nREM\n\n@echo off\n\nset \"TBBROOT=@TBBROOT_REPLACEMENT@\"\nset \"TBB_DLL_PATH=@BINARY_PATH_REPLACEMENT@\"\n\nset \"INCLUDE=%TBBROOT%\\include;%INCLUDE%\"\nset \"CPATH=%TBBROOT%\\include;%CPATH%\"\nset \"LIB=@LIBRARY_PATH_REPLACEMENT@;%LIB%\"\nset \"PATH=@BINARY_PATH_REPLACEMENT@;%PATH%\"\nset \"PKG_CONFIG_PATH=@LIBRARY_PATH_REPLACEMENT@\\pkgconfig;%PKG_CONFIG_PATH%\"\n\n@CMAKE_ENVIRONMENT_SOURCING_STRING@\n" }, { "path": "src/tbb/integration/windows/nuget/inteltbb.devel.win.targets", "content": "\n\n\n\n \n \n \n $(MSBuildThisFileDirectory)..\\..\\build\\native\\include;%(AdditionalIncludeDirectories)\n TBB_USE_DEBUG;%(PreprocessorDefinitions)\n \n \n\n \n \n \n $(MSBuildThisFileDirectory)..\\..\\build\\native\\win-x86;%(AdditionalLibraryDirectories)\n tbb12.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;%(AdditionalDependencies)\n \n \n \n \n $(MSBuildThisFileDirectory)..\\..\\build\\native\\win-x64;%(AdditionalLibraryDirectories)\n tbb12.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;%(AdditionalDependencies)\n \n \n \n \n $(MSBuildThisFileDirectory)..\\..\\build\\native\\win-x86;%(AdditionalLibraryDirectories)\n tbb12_debug.lib;tbbmalloc_debug.lib;tbbmalloc_proxy_debug.lib;%(AdditionalDependencies)\n \n \n \n \n $(MSBuildThisFileDirectory)..\\..\\build\\native\\win-x64;%(AdditionalLibraryDirectories)\n tbb12_debug.lib;tbbmalloc_debug.lib;tbbmalloc_proxy_debug.lib;%(AdditionalDependencies)\n \n \n\n \n \n \n \n \n \n \n \n\n\n" }, { "path": "src/tbb/integration/windows/nuget/inteltbb.redist.win.targets", "content": "\n\n\n\n\n \n \n \n \n \n \n \n\n\n" }, { "path": "src/tbb/integration/windows/oneapi/vars.bat", "content": "@echo off\nREM\nREM Copyright (c) 2023 Intel Corporation\nREM\nREM Licensed under the Apache License, Version 2.0 (the \"License\");\nREM you may not use this file except in compliance with the License.\nREM You may obtain a copy of the License at\nREM\nREM http://www.apache.org/licenses/LICENSE-2.0\nREM\nREM Unless required by applicable law or agreed to in writing, software\nREM distributed under the License is distributed on an \"AS IS\" BASIS,\nREM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nREM See the License for the specific language governing permissions and\nREM limitations under the License.\nREM\n\nif not defined SETVARS_CALL (\n echo:\n echo :: ERROR: This script must be executed by setvars.bat.\n echo: Try '[install-dir]\\setvars.bat --help' for help.\n echo:\n exit /b 255\n)\n\nif not defined ONEAPI_ROOT (\n echo:\n echo :: ERROR: This script requires that the ONEAPI_ROOT env variable is set.\"\n echo: Try '[install-dir]\\setvars.bat --help' for help.\n echo:\n exit /b 254\n)\n\nset \"TBBROOT=%ONEAPI_ROOT%\"\n\n:: Set the default arguments\nset \"TBB_TARGET_ARCH=%INTEL_TARGET_ARCH%\"\nset TBB_TARGET_VS=\nset ARCH_SUFFIX=\n\n:ParseArgs\n:: Parse the incoming arguments\nif /i \"%1\"==\"\" goto SetEnv\nif /i \"%1\"==\"vs2019\" (set TBB_TARGET_VS= ) & shift & goto ParseArgs\nif /i \"%1\"==\"vs2022\" (set TBB_TARGET_VS= ) & shift & goto ParseArgs\nif /i \"%1\"==\"all\" (set TBB_TARGET_VS=vc_mt) & shift & goto ParseArgs\n\nif \"%TBB_TARGET_ARCH%\"==\"ia32\" set ARCH_SUFFIX=32 \n\n:SetEnv\nif exist \"%TBBROOT%\\bin%ARCH_SUFFIX%\\%TBB_TARGET_VS%\\tbb12.dll\" (\n set \"TBB_DLL_PATH=%TBBROOT%\\bin%ARCH_SUFFIX%\\%TBB_TARGET_VS%\"\n)\n\n:End\nexit /B 0\n" }, { "path": "src/tbb/integration/windows/sys_check/sys_check.bat", "content": "@echo off\nREM\nREM Copyright (c) 2019-2021 Intel Corporation\nREM\nREM Licensed under the Apache License, Version 2.0 (the \"License\");\nREM you may not use this file except in compliance with the License.\nREM You may obtain a copy of the License at\nREM\nREM http://www.apache.org/licenses/LICENSE-2.0\nREM\nREM Unless required by applicable law or agreed to in writing, software\nREM distributed under the License is distributed on an \"AS IS\" BASIS,\nREM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nREM See the License for the specific language governing permissions and\nREM limitations under the License.\nREM\n\nexit /B 0\n" }, { "path": "src/tbb/src/tbb/CMakeLists.txt", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nadd_library(tbb\n address_waiter.cpp\n allocator.cpp\n arena.cpp\n arena_slot.cpp\n concurrent_bounded_queue.cpp\n dynamic_link.cpp\n exception.cpp\n governor.cpp\n global_control.cpp\n itt_notify.cpp\n main.cpp\n market.cpp\n tcm_adaptor.cpp\n misc.cpp\n misc_ex.cpp\n observer_proxy.cpp\n parallel_pipeline.cpp\n private_server.cpp\n profiling.cpp\n rml_tbb.cpp\n rtm_mutex.cpp\n rtm_rw_mutex.cpp\n semaphore.cpp\n small_object_pool.cpp\n task.cpp\n task_dispatcher.cpp\n task_group_context.cpp\n thread_dispatcher.cpp\n thread_request_serializer.cpp\n threading_control.cpp\n version.cpp\n queuing_rw_mutex.cpp)\n\nadd_library(TBB::tbb ALIAS tbb)\n\nif (WIN32)\n target_sources(tbb PRIVATE tbb.rc)\n set_target_properties(tbb PROPERTIES OUTPUT_NAME \"tbb${TBB_BINARY_VERSION}\")\nendif()\n\n# TODO: Add statistics.cpp\n\ntarget_compile_definitions(tbb\n PUBLIC\n $<$:TBB_USE_DEBUG>\n PRIVATE\n __TBB_BUILD\n ${TBB_RESUMABLE_TASKS_USE_THREADS}\n $<$>:__TBB_DYNAMIC_LOAD_ENABLED=0>\n $<$>:__TBB_SOURCE_DIRECTLY_INCLUDED=1>)\n\nif (NOT (\"${CMAKE_SYSTEM_PROCESSOR}\" MATCHES \"(armv7-a|aarch64|mips|arm64|riscv)\" OR\n \"${CMAKE_OSX_ARCHITECTURES}\" MATCHES \"arm64\" OR\n WINDOWS_STORE OR\n TBB_WINDOWS_DRIVER))\n target_compile_definitions(tbb PRIVATE __TBB_USE_ITT_NOTIFY)\nendif()\n\ntarget_include_directories(tbb\n PUBLIC\n $\n $)\n\ntarget_compile_options(tbb\n PRIVATE\n ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC.\n ${TBB_MMD_FLAG}\n ${TBB_DSE_FLAG}\n ${TBB_WARNING_LEVEL}\n ${TBB_WARNING_SUPPRESS}\n ${TBB_LIB_COMPILE_FLAGS}\n ${TBB_COMMON_COMPILE_FLAGS}\n)\n\n# Avoid use of target_link_libraries here as it changes /DEF option to \\DEF on Windows.\nset_target_properties(tbb PROPERTIES\n DEFINE_SYMBOL \"\"\n)\n\ntbb_handle_ipo(tbb)\n\nif (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives\n set_target_properties(tbb PROPERTIES\n LINK_FLAGS \"${TBB_LINK_DEF_FILE_FLAG}\\\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbb.def\\\"\"\n LINK_DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbb.def\"\n )\nendif()\n\n# Prefer using target_link_options instead of target_link_libraries to specify link options because\n# target_link_libraries may incorrectly handle some options (on Windows, for example).\nif (COMMAND target_link_options)\n target_link_options(tbb\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nelse()\n target_link_libraries(tbb\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nendif()\n\ntarget_link_libraries(tbb\n PRIVATE\n Threads::Threads\n ${TBB_LIB_LINK_LIBS}\n ${TBB_COMMON_LINK_LIBS}\n)\n\n# Strip debug symbols into a separate .dbg file\nif(TBB_LINUX_SEPARATE_DBG)\n if(NOT CMAKE_BUILD_TYPE STREQUAL \"release\")\n find_program(OBJCOPY_COMMAND objcopy)\n if(NOT OBJCOPY_COMMAND)\n message(WARNING \"objcopy command not found in the system\")\n else()\n add_custom_command(TARGET tbb POST_BUILD\n COMMAND objcopy --only-keep-debug $ $.dbg\n COMMAND objcopy --strip-debug $\n COMMAND objcopy --add-gnu-debuglink=$.dbg $\n COMMENT \"Creating and associating .dbg file with tbb\"\n )\n endif()\n else()\n message(WARNING \" TBB_LINUX_SEPARATE_DBG flag is not used on release config\")\n endif()\nendif()\n\nif(TBB_BUILD_APPLE_FRAMEWORKS)\n set_target_properties(tbb PROPERTIES\n FRAMEWORK TRUE\n FRAMEWORK_VERSION ${TBB_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbb\n MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbb\n MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBB_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBB_BINARY_VERSION})\nendif()\n\ntbb_install_target(tbb)\n\nif (TBB_INSTALL)\n if (MSVC)\n # Create a copy of target linker file (tbb[_debug].lib) with legacy name (tbb[_debug].lib)\n # to support previous user experience for linkage.\n install(FILES\n $\n DESTINATION lib\n CONFIGURATIONS RelWithDebInfo Release MinSizeRel\n RENAME tbb.lib\n COMPONENT devel\n )\n\n install(FILES\n $\n DESTINATION lib\n CONFIGURATIONS Debug\n RENAME tbb_debug.lib\n COMPONENT devel\n )\n endif()\n if(TBB_LINUX_SEPARATE_DBG)\n install(FILES\n $.dbg\n DESTINATION lib\n COMPONENT devel\n )\n endif()\n set(_tbb_pc_lib_name tbb)\n\n if (WIN32)\n set(_tbb_pc_lib_name ${_tbb_pc_lib_name}${TBB_BINARY_VERSION})\n endif()\n\n if (CMAKE_SIZEOF_VOID_P EQUAL 8)\n set(TBB_PC_NAME tbb)\n else()\n set(TBB_PC_NAME tbb32)\n endif()\n\n set(_prefix_for_pc_file \"${CMAKE_INSTALL_PREFIX}\")\n\n if (IS_ABSOLUTE \"${CMAKE_INSTALL_LIBDIR}\")\n set(_libdir_for_pc_file \"${CMAKE_INSTALL_LIBDIR}\")\n else()\n set(_libdir_for_pc_file \"\\${prefix}/${CMAKE_INSTALL_LIBDIR}\")\n endif()\n\n if (IS_ABSOLUTE \"${CMAKE_INSTALL_INCLUDEDIR}\")\n set(_includedir_for_pc_file \"${CMAKE_INSTALL_INCLUDEDIR}\")\n else()\n set(_includedir_for_pc_file \"\\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}\")\n endif()\n\n configure_file(${PROJECT_SOURCE_DIR}/integration/pkg-config/tbb.pc.in ${CMAKE_CURRENT_BINARY_DIR}/${TBB_PC_NAME}.pc @ONLY)\n install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${TBB_PC_NAME}.pc\n DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig/\n COMPONENT devel)\nendif()\n\nif (COMMAND tbb_gen_vars)\n tbb_gen_vars(tbb)\nendif()\n" }, { "path": "src/tbb/src/tbb/address_waiter.cpp", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"governor.h\"\n#include \"concurrent_monitor.h\"\n#include \"oneapi/tbb/detail/_waitable_atomic.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct address_context {\n address_context() = default;\n\n address_context(void* address, std::uintptr_t context) :\n my_address(address), my_context(context)\n {}\n\n void* my_address{nullptr};\n std::uintptr_t my_context{0};\n};\n\nclass address_waiter : public concurrent_monitor_base {\n using base_type = concurrent_monitor_base;\npublic:\n using base_type::base_type;\n /** per-thread descriptor for concurrent_monitor */\n using thread_context = sleep_node;\n};\n\n// 1024 is a rough estimate based on two assumptions:\n// 1) there are no more than 1000 threads in the application;\n// 2) the mutexes are optimized for short critical sections less than a couple of microseconds,\n// which is less than 1/1000 of a time slice.\n// In the worst case, we have single mutex that is locked and its thread is preempted.\n// Therefore, the probability of a collision while taking unrelated mutex is about 1/size of a table.\nstatic constexpr std::size_t num_address_waiters = 2 << 10;\nstatic_assert(std::is_standard_layout::value,\n \"address_waiter must be with standard layout\");\nstatic address_waiter address_waiter_table[num_address_waiters];\n\nvoid clear_address_waiter_table() {\n for (std::size_t i = 0; i < num_address_waiters; ++i) {\n address_waiter_table[i].destroy();\n }\n}\n\nstatic address_waiter& get_address_waiter(void* address) {\n std::uintptr_t tag = std::uintptr_t(address);\n return address_waiter_table[((tag >> 5) ^ tag) % num_address_waiters];\n}\n\nvoid wait_on_address(void* address, d1::delegate_base& predicate, std::uintptr_t context) {\n address_waiter& waiter = get_address_waiter(address);\n waiter.wait(predicate, address_context{address, context});\n}\n\nvoid notify_by_address(void* address, std::uintptr_t target_context) {\n address_waiter& waiter = get_address_waiter(address);\n\n auto predicate = [address, target_context] (address_context ctx) {\n return ctx.my_address == address && ctx.my_context == target_context;\n };\n\n waiter.notify_relaxed(predicate);\n}\n\nvoid notify_by_address_one(void* address) {\n address_waiter& waiter = get_address_waiter(address);\n\n auto predicate = [address] (address_context ctx) {\n return ctx.my_address == address;\n };\n\n waiter.notify_one_relaxed(predicate);\n}\n\nvoid notify_by_address_all(void* address) {\n address_waiter& waiter = get_address_waiter(address);\n\n auto predicate = [address] (address_context ctx) {\n return ctx.my_address == address;\n };\n\n waiter.notify_relaxed(predicate);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/allocator.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/version.h\"\n\n#include \"oneapi/tbb/detail/_exception.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/tbb_allocator.h\" // Is this OK?\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n\n#include \"dynamic_link.h\"\n#include \"misc.h\"\n\n#include \n\n#ifdef _WIN32\n#include \n#else\n#include \n#endif\n\n#if (!defined(_WIN32) && !defined(_WIN64)) || defined(__CYGWIN__)\n#include // posix_memalign, free\n// With glibc, uClibc and musl on Linux and bionic on Android it is safe to use memalign(), as the allocated memory\n// can be freed with free(). It is also better to use memalign() since posix_memalign() is just a wrapper on top of\n// memalign() and it offers nothing but overhead due to inconvenient interface. This is likely the case with other\n// standard libraries as well, and more libraries can be added to the preprocessor check below. Unfortunately, we\n// can't detect musl, so we simply enable memalign() on Linux and Android in general.\n#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__ANDROID__)\n#include // memalign\n#define __TBB_USE_MEMALIGN\n#else\n#define __TBB_USE_POSIX_MEMALIGN\n#endif\n#elif defined(_MSC_VER) || defined(__MINGW32__)\n#include // _aligned_malloc, _aligned_free\n#define __TBB_USE_MSVC_ALIGNED_MALLOC\n#endif\n\n#if __TBB_WEAK_SYMBOLS_PRESENT\n\n#pragma weak scalable_malloc\n#pragma weak scalable_free\n#pragma weak scalable_aligned_malloc\n#pragma weak scalable_aligned_free\n\nextern \"C\" {\n void* scalable_malloc(std::size_t);\n void scalable_free(void*);\n void* scalable_aligned_malloc(std::size_t, std::size_t);\n void scalable_aligned_free(void*);\n}\n\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Initialization routine used for first indirect call via allocate_handler.\nstatic void* initialize_allocate_handler(std::size_t size);\n\n//! Handler for memory allocation\nusing allocate_handler_type = void* (*)(std::size_t size);\nstatic std::atomic allocate_handler{ &initialize_allocate_handler };\nallocate_handler_type allocate_handler_unsafe = nullptr;\n\n//! Handler for memory deallocation\nstatic void (*deallocate_handler)(void* pointer) = nullptr;\n\n//! Initialization routine used for first indirect call via cache_aligned_allocate_handler.\nstatic void* initialize_cache_aligned_allocate_handler(std::size_t n, std::size_t alignment);\n\n//! Allocates overaligned memory using standard memory allocator. It is used when scalable_allocator is not available.\nstatic void* std_cache_aligned_allocate(std::size_t n, std::size_t alignment);\n\n//! Deallocates overaligned memory using standard memory allocator. It is used when scalable_allocator is not available.\nstatic void std_cache_aligned_deallocate(void* p);\n\n//! Handler for padded memory allocation\nusing cache_aligned_allocate_handler_type = void* (*)(std::size_t n, std::size_t alignment);\nstatic std::atomic cache_aligned_allocate_handler{ &initialize_cache_aligned_allocate_handler };\ncache_aligned_allocate_handler_type cache_aligned_allocate_handler_unsafe = nullptr;\n\n//! Handler for padded memory deallocation\nstatic void (*cache_aligned_deallocate_handler)(void* p) = nullptr;\n\n//! Table describing how to link the handlers.\nstatic const dynamic_link_descriptor MallocLinkTable[] = {\n DLD(scalable_malloc, allocate_handler_unsafe),\n DLD(scalable_free, deallocate_handler),\n DLD(scalable_aligned_malloc, cache_aligned_allocate_handler_unsafe),\n DLD(scalable_aligned_free, cache_aligned_deallocate_handler),\n};\n\n\n#if TBB_USE_DEBUG\n#define DEBUG_SUFFIX \"_debug\"\n#else\n#define DEBUG_SUFFIX\n#endif /* TBB_USE_DEBUG */\n\n// MALLOCLIB_NAME is the name of the oneTBB memory allocator library.\n#if _WIN32||_WIN64\n#define MALLOCLIB_NAME \"tbbmalloc\" DEBUG_SUFFIX \".dll\"\n#elif __APPLE__\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX \".2.dylib\"\n#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX \".so\"\n#elif __unix__ // Note that order of these #elif's is important!\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX \".so.2\"\n#else\n#error Unknown OS\n#endif\n\n//! Initialize the allocation/free handler pointers.\n/** Caller is responsible for ensuring this routine is called exactly once.\n The routine attempts to dynamically link with the TBB memory allocator.\n If that allocator is not found, it links to malloc and free. */\nvoid initialize_handler_pointers() {\n __TBB_ASSERT(allocate_handler == &initialize_allocate_handler, nullptr);\n bool success = dynamic_link(MALLOCLIB_NAME, MallocLinkTable, 4);\n if(!success) {\n // If unsuccessful, set the handlers to the default routines.\n // This must be done now, and not before FillDynamicLinks runs, because if other\n // threads call the handlers, we want them to go through the DoOneTimeInitializations logic,\n // which forces them to wait.\n allocate_handler_unsafe = &std::malloc;\n deallocate_handler = &std::free;\n cache_aligned_allocate_handler_unsafe = &std_cache_aligned_allocate;\n cache_aligned_deallocate_handler = &std_cache_aligned_deallocate;\n }\n\n allocate_handler.store(allocate_handler_unsafe, std::memory_order_release);\n cache_aligned_allocate_handler.store(cache_aligned_allocate_handler_unsafe, std::memory_order_release);\n\n PrintExtraVersionInfo( \"ALLOCATOR\", success?\"scalable_malloc\":\"malloc\" );\n}\n\nstatic std::once_flag initialization_state;\nvoid initialize_cache_aligned_allocator() {\n std::call_once(initialization_state, &initialize_handler_pointers);\n}\n\n//! Executed on very first call through allocate_handler\n/** Only one of initialize_allocate_handler() and initialize_cache_aligned_allocate_handler()\n is called, since each one of them also initializes the other.\n\n In the current implementation of oneTBB library initialization, cache_aligned_allocate() is\n used, which in turn calls initialize_cache_aligned_allocate_handler(). As mentioned above,\n that also initializes the regular allocate_handler.\n\n Therefore, initialize_allocate_handler() is not called in the current library implementation. */\nstatic void* initialize_allocate_handler(std::size_t size) {\n initialize_cache_aligned_allocator();\n __TBB_ASSERT(allocate_handler != &initialize_allocate_handler, nullptr);\n return (*allocate_handler)(size);\n}\n\n//! Executed on very first call through cache_aligned_allocate_handler\nstatic void* initialize_cache_aligned_allocate_handler(std::size_t bytes, std::size_t alignment) {\n initialize_cache_aligned_allocator();\n __TBB_ASSERT(cache_aligned_allocate_handler != &initialize_cache_aligned_allocate_handler, nullptr);\n return (*cache_aligned_allocate_handler)(bytes, alignment);\n}\n\n// TODO: use CPUID to find actual line size, though consider backward compatibility\n// nfs - no false sharing\nstatic constexpr std::size_t nfs_size = 128;\n\nstd::size_t __TBB_EXPORTED_FUNC cache_line_size() {\n return nfs_size;\n}\n\nvoid* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size) {\n const std::size_t cache_line_size = nfs_size;\n __TBB_ASSERT(is_power_of_two(cache_line_size), \"must be power of two\");\n\n // Check for overflow\n if (size + cache_line_size < size) {\n throw_exception(exception_id::bad_alloc);\n }\n // scalable_aligned_malloc considers zero size request an error, and returns nullptr\n if (size == 0) size = 1;\n\n void* result = cache_aligned_allocate_handler.load(std::memory_order_acquire)(size, cache_line_size);\n if (!result) {\n throw_exception(exception_id::bad_alloc);\n }\n __TBB_ASSERT(is_aligned(result, cache_line_size), \"The returned address isn't aligned\");\n return result;\n}\n\nvoid __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p) {\n __TBB_ASSERT(cache_aligned_deallocate_handler, \"Initialization has not been yet.\");\n (*cache_aligned_deallocate_handler)(p);\n}\n\nstatic void* std_cache_aligned_allocate(std::size_t bytes, std::size_t alignment) {\n#if defined(__TBB_USE_MEMALIGN)\n return memalign(alignment, bytes);\n#elif defined(__TBB_USE_POSIX_MEMALIGN)\n void* p = nullptr;\n int res = posix_memalign(&p, alignment, bytes);\n if (res != 0)\n p = nullptr;\n return p;\n#elif defined(__TBB_USE_MSVC_ALIGNED_MALLOC)\n return _aligned_malloc(bytes, alignment);\n#else\n // TODO: make it common with cache_aligned_resource\n std::size_t space = alignment + bytes;\n std::uintptr_t base = reinterpret_cast(std::malloc(space));\n if (!base) {\n return nullptr;\n }\n std::uintptr_t result = (base + nfs_size) & ~(nfs_size - 1);\n // Round up to the next cache line (align the base address)\n __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), \"Cannot store a base pointer to the header\");\n __TBB_ASSERT(space - (result - base) >= bytes, \"Not enough space for the storage\");\n\n // Record where block actually starts.\n (reinterpret_cast(result))[-1] = base;\n return reinterpret_cast(result);\n#endif\n}\n\nstatic void std_cache_aligned_deallocate(void* p) {\n#if defined(__TBB_USE_MEMALIGN) || defined(__TBB_USE_POSIX_MEMALIGN)\n free(p);\n#elif defined(__TBB_USE_MSVC_ALIGNED_MALLOC)\n _aligned_free(p);\n#else\n if (p) {\n __TBB_ASSERT(reinterpret_cast(p) >= 0x4096, \"attempt to free block not obtained from cache_aligned_allocator\");\n // Recover where block actually starts\n std::uintptr_t base = (reinterpret_cast(p))[-1];\n __TBB_ASSERT(((base + nfs_size) & ~(nfs_size - 1)) == reinterpret_cast(p), \"Incorrect alignment or not allocated by std_cache_aligned_deallocate?\");\n std::free(reinterpret_cast(base));\n }\n#endif\n}\n\nvoid* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size) {\n void* result = allocate_handler.load(std::memory_order_acquire)(size);\n if (!result) {\n throw_exception(exception_id::bad_alloc);\n }\n return result;\n}\n\nvoid __TBB_EXPORTED_FUNC deallocate_memory(void* p) {\n if (p) {\n __TBB_ASSERT(deallocate_handler, \"Initialization has not been yet.\");\n (*deallocate_handler)(p);\n }\n}\n\nbool __TBB_EXPORTED_FUNC is_tbbmalloc_used() {\n auto handler_snapshot = allocate_handler.load(std::memory_order_acquire);\n if (handler_snapshot == &initialize_allocate_handler) {\n initialize_cache_aligned_allocator();\n }\n handler_snapshot = allocate_handler.load(std::memory_order_relaxed);\n __TBB_ASSERT(handler_snapshot != &initialize_allocate_handler && deallocate_handler != nullptr, nullptr);\n // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__)\n __TBB_ASSERT((reinterpret_cast(handler_snapshot) == reinterpret_cast(&std::malloc)) == (reinterpret_cast(deallocate_handler) == reinterpret_cast(&std::free)),\n \"Both shim pointers must refer to routines from the same package (either TBB or CRT)\");\n return reinterpret_cast(handler_snapshot) == reinterpret_cast(&std::malloc);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/arena.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"task_dispatcher.h\"\n#include \"governor.h\"\n#include \"threading_control.h\"\n#include \"arena.h\"\n#include \"itt_notify.h\"\n#include \"semaphore.h\"\n#include \"waiters.h\"\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/info.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_ARENA_BINDING\nclass numa_binding_observer : public tbb::task_scheduler_observer {\n binding_handler* my_binding_handler;\npublic:\n numa_binding_observer( d1::task_arena* ta, int num_slots, int numa_id, core_type_id core_type, int max_threads_per_core )\n : task_scheduler_observer(*ta)\n , my_binding_handler(construct_binding_handler(num_slots, numa_id, core_type, max_threads_per_core))\n {}\n\n void on_scheduler_entry( bool ) override {\n apply_affinity_mask(my_binding_handler, this_task_arena::current_thread_index());\n }\n\n void on_scheduler_exit( bool ) override {\n restore_affinity_mask(my_binding_handler, this_task_arena::current_thread_index());\n }\n\n ~numa_binding_observer() override{\n destroy_binding_handler(my_binding_handler);\n }\n};\n\nnuma_binding_observer* construct_binding_observer( d1::task_arena* ta, int num_slots, int numa_id, core_type_id core_type, int max_threads_per_core ) {\n numa_binding_observer* binding_observer = nullptr;\n if ((core_type >= 0 && core_type_count() > 1) || (numa_id >= 0 && numa_node_count() > 1) || max_threads_per_core > 0) {\n binding_observer = new(allocate_memory(sizeof(numa_binding_observer))) numa_binding_observer(ta, num_slots, numa_id, core_type, max_threads_per_core);\n __TBB_ASSERT(binding_observer, \"Failure during NUMA binding observer allocation and construction\");\n }\n return binding_observer;\n}\n\nvoid destroy_binding_observer( numa_binding_observer* binding_observer ) {\n __TBB_ASSERT(binding_observer, \"Trying to deallocate nullptr pointer\");\n binding_observer->observe(false);\n binding_observer->~numa_binding_observer();\n deallocate_memory(binding_observer);\n}\n#endif /*!__TBB_ARENA_BINDING*/\n\nvoid arena::on_thread_leaving(unsigned ref_param) {\n //\n // Implementation of arena destruction synchronization logic contained various\n // bugs/flaws at the different stages of its evolution, so below is a detailed\n // description of the issues taken into consideration in the framework of the\n // current design.\n //\n // In case of using fire-and-forget tasks (scheduled via task::enqueue())\n // external thread is allowed to leave its arena before all its work is executed,\n // and market may temporarily revoke all workers from this arena. Since revoked\n // workers never attempt to reset arena state to EMPTY and cancel its request\n // to RML for threads, the arena object is destroyed only when both the last\n // thread is leaving it and arena's state is EMPTY (that is its external thread\n // left and it does not contain any work).\n // Thus resetting arena to EMPTY state (as earlier TBB versions did) should not\n // be done here (or anywhere else in the external thread to that matter); doing so\n // can result either in arena's premature destruction (at least without\n // additional costly checks in workers) or in unnecessary arena state changes\n // (and ensuing workers migration).\n //\n // A worker that checks for work presence and transitions arena to the EMPTY\n // state (in snapshot taking procedure arena::out_of_work()) updates\n // arena::my_pool_state first and only then arena::my_num_workers_requested.\n // So the check for work absence must be done against the latter field.\n //\n // In a time window between decrementing the active threads count and checking\n // if there is an outstanding request for workers. New worker thread may arrive,\n // finish remaining work, set arena state to empty, and leave decrementing its\n // refcount and destroying. Then the current thread will destroy the arena\n // the second time. To preclude it a local copy of the outstanding request\n // value can be stored before decrementing active threads count.\n //\n // But this technique may cause two other problem. When the stored request is\n // zero, it is possible that arena still has threads and they can generate new\n // tasks and thus re-establish non-zero requests. Then all the threads can be\n // revoked (as described above) leaving this thread the last one, and causing\n // it to destroy non-empty arena.\n //\n // The other problem takes place when the stored request is non-zero. Another\n // thread may complete the work, set arena state to empty, and leave without\n // arena destruction before this thread decrements the refcount. This thread\n // cannot destroy the arena either. Thus the arena may be \"orphaned\".\n //\n // In both cases we cannot dereference arena pointer after the refcount is\n // decremented, as our arena may already be destroyed.\n //\n // If this is the external thread, the market is protected by refcount to it.\n // In case of workers market's liveness is ensured by the RML connection\n // rundown protocol, according to which the client (i.e. the market) lives\n // until RML server notifies it about connection termination, and this\n // notification is fired only after all workers return into RML.\n //\n // Thus if we decremented refcount to zero we ask the market to check arena\n // state (including the fact if it is alive) under the lock.\n //\n\n __TBB_ASSERT(my_references.load(std::memory_order_relaxed) >= ref_param, \"broken arena reference counter\");\n\n // When there is no workers someone must free arena, as\n // without workers, no one calls out_of_work().\n if (ref_param == ref_external && !my_mandatory_concurrency.test()) {\n out_of_work();\n }\n\n threading_control* tc = my_threading_control;\n auto tc_client_snapshot = tc->prepare_client_destruction(my_tc_client);\n // Release our reference to sync with destroy_client\n unsigned remaining_ref = my_references.fetch_sub(ref_param, std::memory_order_release) - ref_param;\n // do not access `this` it might be destroyed already\n if (remaining_ref == 0) {\n if (tc->try_destroy_client(tc_client_snapshot)) {\n // We are requested to destroy ourself\n free_arena();\n }\n }\n}\n\nstd::size_t arena::occupy_free_slot_in_range( thread_data& tls, std::size_t lower, std::size_t upper ) {\n if ( lower >= upper ) return out_of_arena;\n // Start search for an empty slot from the one we occupied the last time\n std::size_t index = tls.my_arena_index;\n if ( index < lower || index >= upper ) index = tls.my_random.get() % (upper - lower) + lower;\n __TBB_ASSERT( index >= lower && index < upper, nullptr);\n // Find a free slot\n for ( std::size_t i = index; i < upper; ++i )\n if (my_slots[i].try_occupy()) return i;\n for ( std::size_t i = lower; i < index; ++i )\n if (my_slots[i].try_occupy()) return i;\n return out_of_arena;\n}\n\ntemplate \nstd::size_t arena::occupy_free_slot(thread_data& tls) {\n // Firstly, external threads try to occupy reserved slots\n std::size_t index = as_worker ? out_of_arena : occupy_free_slot_in_range( tls, 0, my_num_reserved_slots );\n if ( index == out_of_arena ) {\n // Secondly, all threads try to occupy all non-reserved slots\n index = occupy_free_slot_in_range(tls, my_num_reserved_slots, my_num_slots );\n // Likely this arena is already saturated\n if ( index == out_of_arena )\n return out_of_arena;\n }\n\n atomic_update( my_limit, (unsigned)(index + 1), std::less() );\n return index;\n}\n\nstd::uintptr_t arena::calculate_stealing_threshold() {\n stack_anchor_type anchor;\n return r1::calculate_stealing_threshold(reinterpret_cast(&anchor), my_threading_control->worker_stack_size());\n}\n\nvoid arena::process(thread_data& tls) {\n governor::set_thread_data(tls); // TODO: consider moving to create_one_job.\n __TBB_ASSERT( is_alive(my_guard), nullptr);\n __TBB_ASSERT( my_num_slots >= 1, nullptr);\n\n std::size_t index = occupy_free_slot(tls);\n if (index == out_of_arena) {\n on_thread_leaving(ref_worker);\n return;\n }\n\n __TBB_ASSERT( index >= my_num_reserved_slots, \"Workers cannot occupy reserved slots\" );\n tls.attach_arena(*this, index);\n // worker thread enters the dispatch loop to look for a work\n tls.my_inbox.set_is_idle(true);\n if (tls.my_arena_slot->is_task_pool_published()) {\n tls.my_inbox.set_is_idle(false);\n }\n\n task_dispatcher& task_disp = tls.my_arena_slot->default_task_dispatcher();\n tls.enter_task_dispatcher(task_disp, calculate_stealing_threshold());\n __TBB_ASSERT(task_disp.can_steal(), nullptr);\n\n __TBB_ASSERT( !tls.my_last_observer, \"There cannot be notified local observers when entering arena\" );\n my_observers.notify_entry_observers(tls.my_last_observer, tls.my_is_worker);\n\n // Waiting on special object tied to this arena\n outermost_worker_waiter waiter(*this);\n d1::task* t = tls.my_task_dispatcher->local_wait_for_all(nullptr, waiter);\n // For purposes of affinity support, the slot's mailbox is considered idle while no thread is\n // attached to it.\n tls.my_inbox.set_is_idle(true);\n\n __TBB_ASSERT_EX(t == nullptr, \"Outermost worker must not leave dispatch loop with a task\");\n __TBB_ASSERT(governor::is_thread_data_set(&tls), nullptr);\n __TBB_ASSERT(tls.my_task_dispatcher == &task_disp, nullptr);\n\n my_observers.notify_exit_observers(tls.my_last_observer, tls.my_is_worker);\n tls.my_last_observer = nullptr;\n\n tls.leave_task_dispatcher();\n\n // Arena slot detach (arena may be used in market::process)\n // TODO: Consider moving several calls below into a new method(e.g.detach_arena).\n tls.my_arena_slot->release();\n tls.my_arena_slot = nullptr;\n tls.my_inbox.detach();\n __TBB_ASSERT(tls.my_inbox.is_idle_state(true), nullptr);\n __TBB_ASSERT(is_alive(my_guard), nullptr);\n\n // In contrast to earlier versions of TBB (before 3.0 U5) now it is possible\n // that arena may be temporarily left unpopulated by threads. See comments in\n // arena::on_thread_leaving() for more details.\n on_thread_leaving(ref_worker);\n __TBB_ASSERT(tls.my_arena == this, \"my_arena is used as a hint when searching the arena to join\");\n}\n\narena::arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned priority_level) {\n __TBB_ASSERT( !my_guard, \"improperly allocated arena?\" );\n __TBB_ASSERT( sizeof(my_slots[0]) % cache_line_size()==0, \"arena::slot size not multiple of cache line size\" );\n __TBB_ASSERT( is_aligned(this, cache_line_size()), \"arena misaligned\" );\n my_threading_control = control;\n my_limit = 1;\n // Two slots are mandatory: for the external thread, and for 1 worker (required to support starvation resistant tasks).\n my_num_slots = num_arena_slots(num_slots, num_reserved_slots);\n my_num_reserved_slots = num_reserved_slots;\n my_max_num_workers = num_slots-num_reserved_slots;\n my_priority_level = priority_level;\n my_references = ref_external; // accounts for the external thread\n my_observers.my_arena = this;\n my_co_cache.init(4 * num_slots);\n __TBB_ASSERT ( my_max_num_workers <= my_num_slots, nullptr);\n // Initialize the default context. It should be allocated before task_dispatch construction.\n my_default_ctx = new (cache_aligned_allocate(sizeof(d1::task_group_context)))\n d1::task_group_context{ d1::task_group_context::isolated, d1::task_group_context::fp_settings };\n // Construct slots. Mark internal synchronization elements for the tools.\n task_dispatcher* base_td_pointer = reinterpret_cast(my_slots + my_num_slots);\n for( unsigned i = 0; i < my_num_slots; ++i ) {\n // __TBB_ASSERT( !my_slots[i].my_scheduler && !my_slots[i].task_pool, nullptr);\n __TBB_ASSERT( !my_slots[i].task_pool_ptr, nullptr);\n __TBB_ASSERT( !my_slots[i].my_task_pool_size, nullptr);\n mailbox(i).construct();\n my_slots[i].init_task_streams(i);\n my_slots[i].my_default_task_dispatcher = new(base_td_pointer + i) task_dispatcher(this);\n my_slots[i].my_is_occupied.store(false, std::memory_order_relaxed);\n }\n my_fifo_task_stream.initialize(my_num_slots);\n my_resume_task_stream.initialize(my_num_slots);\n#if __TBB_PREVIEW_CRITICAL_TASKS\n my_critical_task_stream.initialize(my_num_slots);\n#endif\n my_mandatory_requests = 0;\n}\n\narena& arena::allocate_arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots,\n unsigned priority_level)\n{\n __TBB_ASSERT( sizeof(base_type) + sizeof(arena_slot) == sizeof(arena), \"All arena data fields must go to arena_base\" );\n __TBB_ASSERT( sizeof(base_type) % cache_line_size() == 0, \"arena slots area misaligned: wrong padding\" );\n __TBB_ASSERT( sizeof(mail_outbox) == max_nfs_size, \"Mailbox padding is wrong\" );\n std::size_t n = allocation_size(num_arena_slots(num_slots, num_reserved_slots));\n unsigned char* storage = (unsigned char*)cache_aligned_allocate(n);\n // Zero all slots to indicate that they are empty\n std::memset( storage, 0, n );\n\n return *new( storage + num_arena_slots(num_slots, num_reserved_slots) * sizeof(mail_outbox) )\n arena(control, num_slots, num_reserved_slots, priority_level);\n}\n\nvoid arena::free_arena () {\n __TBB_ASSERT( is_alive(my_guard), nullptr);\n __TBB_ASSERT( !my_references.load(std::memory_order_relaxed), \"There are threads in the dying arena\" );\n __TBB_ASSERT( !my_total_num_workers_requested && !my_num_workers_allotted, \"Dying arena requests workers\" );\n __TBB_ASSERT( is_empty(), \"Inconsistent state of a dying arena\" );\n#if __TBB_ARENA_BINDING\n if (my_numa_binding_observer != nullptr) {\n destroy_binding_observer(my_numa_binding_observer);\n my_numa_binding_observer = nullptr;\n }\n#endif /*__TBB_ARENA_BINDING*/\n poison_value( my_guard );\n for ( unsigned i = 0; i < my_num_slots; ++i ) {\n // __TBB_ASSERT( !my_slots[i].my_scheduler, \"arena slot is not empty\" );\n // TODO: understand the assertion and modify\n // __TBB_ASSERT( my_slots[i].task_pool == EmptyTaskPool, nullptr);\n __TBB_ASSERT( my_slots[i].head == my_slots[i].tail, nullptr); // TODO: replace by is_quiescent_local_task_pool_empty\n my_slots[i].free_task_pool();\n mailbox(i).drain();\n my_slots[i].my_default_task_dispatcher->~task_dispatcher();\n }\n __TBB_ASSERT(my_fifo_task_stream.empty(), \"Not all enqueued tasks were executed\");\n __TBB_ASSERT(my_resume_task_stream.empty(), \"Not all enqueued tasks were executed\");\n // Cleanup coroutines/schedulers cache\n my_co_cache.cleanup();\n my_default_ctx->~task_group_context();\n cache_aligned_deallocate(my_default_ctx);\n#if __TBB_PREVIEW_CRITICAL_TASKS\n __TBB_ASSERT( my_critical_task_stream.empty(), \"Not all critical tasks were executed\");\n#endif\n // Clear enfources synchronization with observe(false)\n my_observers.clear();\n\n void* storage = &mailbox(my_num_slots-1);\n __TBB_ASSERT( my_references.load(std::memory_order_relaxed) == 0, nullptr);\n this->~arena();\n#if TBB_USE_ASSERT > 1\n std::memset( storage, 0, allocation_size(my_num_slots) );\n#endif /* TBB_USE_ASSERT */\n cache_aligned_deallocate( storage );\n}\n\nbool arena::has_enqueued_tasks() {\n return !my_fifo_task_stream.empty();\n}\n\nvoid arena::request_workers(int mandatory_delta, int workers_delta, bool wakeup_threads) {\n my_threading_control->adjust_demand(my_tc_client, mandatory_delta, workers_delta);\n\n if (wakeup_threads) {\n // Notify all sleeping threads that work has appeared in the arena.\n get_waiting_threads_monitor().notify([&] (market_context context) {\n return this == context.my_arena_addr;\n });\n }\n}\n\nbool arena::has_tasks() {\n // TODO: rework it to return at least a hint about where a task was found; better if the task itself.\n std::size_t n = my_limit.load(std::memory_order_acquire);\n bool tasks_are_available = false;\n for (std::size_t k = 0; k < n && !tasks_are_available; ++k) {\n tasks_are_available = !my_slots[k].is_empty();\n }\n tasks_are_available = tasks_are_available || has_enqueued_tasks() || !my_resume_task_stream.empty();\n#if __TBB_PREVIEW_CRITICAL_TASKS\n tasks_are_available = tasks_are_available || !my_critical_task_stream.empty();\n#endif\n return tasks_are_available;\n}\n\nvoid arena::out_of_work() {\n // We should try unset my_pool_state first due to keep arena invariants in consistent state\n // Otherwise, we might have my_pool_state = false and my_mandatory_concurrency = true that is broken invariant\n bool disable_mandatory = my_mandatory_concurrency.try_clear_if([this] { return !has_enqueued_tasks(); });\n bool release_workers = my_pool_state.try_clear_if([this] { return !has_tasks(); });\n\n if (disable_mandatory || release_workers) {\n int mandatory_delta = disable_mandatory ? -1 : 0;\n int workers_delta = release_workers ? -(int)my_max_num_workers : 0;\n\n if (disable_mandatory && is_arena_workerless()) {\n // We had set workers_delta to 1 when enabled mandatory concurrency, so revert it now\n workers_delta = -1;\n }\n request_workers(mandatory_delta, workers_delta);\n }\n}\n\nvoid arena::set_top_priority(bool is_top_priority) {\n my_is_top_priority.store(is_top_priority, std::memory_order_relaxed);\n}\n\nbool arena::is_top_priority() const {\n return my_is_top_priority.load(std::memory_order_relaxed);\n}\n\nbool arena::try_join() {\n if (is_joinable()) {\n my_references += arena::ref_worker;\n return true;\n }\n return false;\n}\n\nvoid arena::set_allotment(unsigned allotment) {\n if (my_num_workers_allotted.load(std::memory_order_relaxed) != allotment) {\n my_num_workers_allotted.store(allotment, std::memory_order_relaxed);\n }\n}\n\nint arena::update_concurrency(unsigned allotment) {\n int delta = allotment - my_num_workers_allotted.load(std::memory_order_relaxed);\n if (delta != 0) {\n my_num_workers_allotted.store(allotment, std::memory_order_relaxed);\n }\n return delta;\n}\n\nstd::pair arena::update_request(int mandatory_delta, int workers_delta) {\n __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr);\n\n int min_workers_request = 0;\n int max_workers_request = 0;\n\n // Calculate min request\n my_mandatory_requests += mandatory_delta;\n min_workers_request = my_mandatory_requests > 0 ? 1 : 0;\n\n // Calculate max request\n my_total_num_workers_requested += workers_delta;\n // Clamp worker request into interval [0, my_max_num_workers]\n max_workers_request = clamp(my_total_num_workers_requested, 0,\n min_workers_request > 0 && is_arena_workerless() ? 1 : (int)my_max_num_workers);\n\n return { min_workers_request, max_workers_request };\n}\n\nthread_control_monitor& arena::get_waiting_threads_monitor() {\n return my_threading_control->get_waiting_threads_monitor();\n}\n\nvoid arena::enqueue_task(d1::task& t, d1::task_group_context& ctx, thread_data& td) {\n task_group_context_impl::bind_to(ctx, &td);\n task_accessor::context(t) = &ctx;\n task_accessor::isolation(t) = no_isolation;\n my_fifo_task_stream.push( &t, random_lane_selector(td.my_random) );\n advertise_new_work();\n}\n\narena& arena::create(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned arena_priority_level, d1::constraints constraints) {\n __TBB_ASSERT(num_slots > 0, NULL);\n __TBB_ASSERT(num_reserved_slots <= num_slots, NULL);\n // Add public market reference for an external thread/task_arena (that adds an internal reference in exchange).\n arena& a = arena::allocate_arena(control, num_slots, num_reserved_slots, arena_priority_level);\n a.my_tc_client = control->create_client(a);\n // We should not publish arena until all fields are initialized\n control->publish_client(a.my_tc_client, constraints);\n return a;\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n// Enable task_arena.h\n#include \"oneapi/tbb/task_arena.h\" // task_arena_base\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if TBB_USE_ASSERT\nvoid assert_arena_priority_valid( tbb::task_arena::priority a_priority ) {\n bool is_arena_priority_correct =\n a_priority == tbb::task_arena::priority::high ||\n a_priority == tbb::task_arena::priority::normal ||\n a_priority == tbb::task_arena::priority::low;\n __TBB_ASSERT( is_arena_priority_correct,\n \"Task arena priority should be equal to one of the predefined values.\" );\n}\n#else\nvoid assert_arena_priority_valid( tbb::task_arena::priority ) {}\n#endif\n\nunsigned arena_priority_level( tbb::task_arena::priority a_priority ) {\n assert_arena_priority_valid( a_priority );\n return d1::num_priority_levels - unsigned(int(a_priority) / d1::priority_stride);\n}\n\ntbb::task_arena::priority arena_priority( unsigned priority_level ) {\n auto priority = tbb::task_arena::priority(\n (d1::num_priority_levels - priority_level) * d1::priority_stride\n );\n assert_arena_priority_valid( priority );\n return priority;\n}\n\nstruct task_arena_impl {\n static void initialize(d1::task_arena_base&);\n static void terminate(d1::task_arena_base&);\n static bool attach(d1::task_arena_base&);\n static void execute(d1::task_arena_base&, d1::delegate_base&);\n static void wait(d1::task_arena_base&);\n static int max_concurrency(const d1::task_arena_base*);\n static void enqueue(d1::task&, d1::task_group_context*, d1::task_arena_base*);\n static d1::slot_id execution_slot(const d1::task_arena_base&);\n};\n\nvoid __TBB_EXPORTED_FUNC initialize(d1::task_arena_base& ta) {\n task_arena_impl::initialize(ta);\n}\nvoid __TBB_EXPORTED_FUNC terminate(d1::task_arena_base& ta) {\n task_arena_impl::terminate(ta);\n}\nbool __TBB_EXPORTED_FUNC attach(d1::task_arena_base& ta) {\n return task_arena_impl::attach(ta);\n}\nvoid __TBB_EXPORTED_FUNC execute(d1::task_arena_base& ta, d1::delegate_base& d) {\n task_arena_impl::execute(ta, d);\n}\nvoid __TBB_EXPORTED_FUNC wait(d1::task_arena_base& ta) {\n task_arena_impl::wait(ta);\n}\n\nint __TBB_EXPORTED_FUNC max_concurrency(const d1::task_arena_base* ta) {\n return task_arena_impl::max_concurrency(ta);\n}\n\nvoid __TBB_EXPORTED_FUNC enqueue(d1::task& t, d1::task_arena_base* ta) {\n task_arena_impl::enqueue(t, nullptr, ta);\n}\n\nvoid __TBB_EXPORTED_FUNC enqueue(d1::task& t, d1::task_group_context& ctx, d1::task_arena_base* ta) {\n task_arena_impl::enqueue(t, &ctx, ta);\n}\n\nd1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::task_arena_base& arena) {\n return task_arena_impl::execution_slot(arena);\n}\n\nvoid task_arena_impl::initialize(d1::task_arena_base& ta) {\n // Enforce global market initialization to properly initialize soft limit\n (void)governor::get_thread_data();\n d1::constraints arena_constraints;\n\n#if __TBB_ARENA_BINDING\n arena_constraints = d1::constraints{}\n .set_core_type(ta.core_type())\n .set_max_threads_per_core(ta.max_threads_per_core())\n .set_numa_id(ta.my_numa_id);\n#endif /*__TBB_ARENA_BINDING*/\n\n if (ta.my_max_concurrency < 1) {\n#if __TBB_ARENA_BINDING\n ta.my_max_concurrency = (int)default_concurrency(arena_constraints);\n#else /*!__TBB_ARENA_BINDING*/\n ta.my_max_concurrency = (int)governor::default_num_threads();\n#endif /*!__TBB_ARENA_BINDING*/\n }\n\n#if __TBB_CPUBIND_PRESENT\n numa_binding_observer* observer = construct_binding_observer(\n static_cast(&ta), arena::num_arena_slots(ta.my_max_concurrency, ta.my_num_reserved_slots),\n ta.my_numa_id, ta.core_type(), ta.max_threads_per_core());\n if (observer) {\n // TODO: Consider lazy initialization for internal arena so\n // the direct calls to observer might be omitted until actual initialization.\n observer->on_scheduler_entry(true);\n }\n#endif /*__TBB_CPUBIND_PRESENT*/\n\n __TBB_ASSERT(ta.my_arena.load(std::memory_order_relaxed) == nullptr, \"Arena already initialized\");\n unsigned priority_level = arena_priority_level(ta.my_priority);\n threading_control* thr_control = threading_control::register_public_reference();\n arena& a = arena::create(thr_control, unsigned(ta.my_max_concurrency), ta.my_num_reserved_slots, priority_level, arena_constraints);\n\n ta.my_arena.store(&a, std::memory_order_release);\n#if __TBB_CPUBIND_PRESENT\n a.my_numa_binding_observer = observer;\n if (observer) {\n observer->on_scheduler_exit(true);\n observer->observe(true);\n }\n#endif /*__TBB_CPUBIND_PRESENT*/\n}\n\nvoid task_arena_impl::terminate(d1::task_arena_base& ta) {\n arena* a = ta.my_arena.load(std::memory_order_relaxed);\n assert_pointer_valid(a);\n threading_control::unregister_public_reference(/*blocking_terminate=*/false);\n a->on_thread_leaving(arena::ref_external);\n ta.my_arena.store(nullptr, std::memory_order_relaxed);\n}\n\nbool task_arena_impl::attach(d1::task_arena_base& ta) {\n __TBB_ASSERT(!ta.my_arena.load(std::memory_order_relaxed), nullptr);\n thread_data* td = governor::get_thread_data_if_initialized();\n if( td && td->my_arena ) {\n arena* a = td->my_arena;\n // There is an active arena to attach to.\n // It's still used by s, so won't be destroyed right away.\n __TBB_ASSERT(a->my_references > 0, nullptr);\n a->my_references += arena::ref_external;\n ta.my_num_reserved_slots = a->my_num_reserved_slots;\n ta.my_priority = arena_priority(a->my_priority_level);\n ta.my_max_concurrency = ta.my_num_reserved_slots + a->my_max_num_workers;\n __TBB_ASSERT(arena::num_arena_slots(ta.my_max_concurrency, ta.my_num_reserved_slots) == a->my_num_slots, nullptr);\n ta.my_arena.store(a, std::memory_order_release);\n // increases threading_control's ref count for task_arena\n threading_control::register_public_reference();\n return true;\n }\n return false;\n}\n\nvoid task_arena_impl::enqueue(d1::task& t, d1::task_group_context* c, d1::task_arena_base* ta) {\n thread_data* td = governor::get_thread_data(); // thread data is only needed for FastRandom instance\n assert_pointer_valid(td, \"thread_data pointer should not be null\");\n arena* a = ta ?\n ta->my_arena.load(std::memory_order_relaxed)\n : td->my_arena\n ;\n assert_pointer_valid(a, \"arena pointer should not be null\");\n auto* ctx = c ? c : a->my_default_ctx;\n assert_pointer_valid(ctx, \"context pointer should not be null\");\n // Is there a better place for checking the state of ctx?\n __TBB_ASSERT(!a->my_default_ctx->is_group_execution_cancelled(),\n \"The task will not be executed because its task_group_context is cancelled.\");\n a->enqueue_task(t, *ctx, *td);\n}\n\nd1::slot_id task_arena_impl::execution_slot(const d1::task_arena_base& ta) {\n thread_data* td = governor::get_thread_data_if_initialized();\n if (td && (td->is_attached_to(ta.my_arena.load(std::memory_order_relaxed)))) {\n return td->my_arena_index;\n }\n return d1::slot_id(-1);\n}\n\nclass nested_arena_context : no_copy {\npublic:\n nested_arena_context(thread_data& td, arena& nested_arena, std::size_t slot_index)\n : m_orig_execute_data_ext(td.my_task_dispatcher->m_execute_data_ext)\n {\n if (td.my_arena != &nested_arena) {\n m_orig_arena = td.my_arena;\n m_orig_slot_index = td.my_arena_index;\n m_orig_last_observer = td.my_last_observer;\n m_orig_is_thread_registered = td.my_is_registered;\n\n td.detach_task_dispatcher();\n td.attach_arena(nested_arena, slot_index);\n td.my_is_registered = false;\n if (td.my_inbox.is_idle_state(true))\n td.my_inbox.set_is_idle(false);\n task_dispatcher& task_disp = td.my_arena_slot->default_task_dispatcher();\n td.enter_task_dispatcher(task_disp, m_orig_execute_data_ext.task_disp->m_stealing_threshold);\n\n // If the calling thread occupies the slots out of external thread reserve we need to notify the\n // market that this arena requires one worker less.\n if (td.my_arena_index >= td.my_arena->my_num_reserved_slots) {\n td.my_arena->request_workers(/* mandatory_delta = */ 0, /* workers_delta = */ -1);\n }\n\n td.my_last_observer = nullptr;\n // The task_arena::execute method considers each calling thread as an external thread.\n td.my_arena->my_observers.notify_entry_observers(td.my_last_observer, /* worker*/false);\n }\n\n m_task_dispatcher = td.my_task_dispatcher;\n m_orig_fifo_tasks_allowed = m_task_dispatcher->allow_fifo_task(true);\n m_orig_critical_task_allowed = m_task_dispatcher->m_properties.critical_task_allowed;\n m_task_dispatcher->m_properties.critical_task_allowed = true;\n\n execution_data_ext& ed_ext = td.my_task_dispatcher->m_execute_data_ext;\n ed_ext.context = td.my_arena->my_default_ctx;\n ed_ext.original_slot = td.my_arena_index;\n ed_ext.affinity_slot = d1::no_slot;\n ed_ext.task_disp = td.my_task_dispatcher;\n ed_ext.isolation = no_isolation;\n\n __TBB_ASSERT(td.my_arena_slot, nullptr);\n __TBB_ASSERT(td.my_arena_slot->is_occupied(), nullptr);\n __TBB_ASSERT(td.my_task_dispatcher, nullptr);\n }\n ~nested_arena_context() {\n thread_data& td = *m_task_dispatcher->m_thread_data;\n __TBB_ASSERT(governor::is_thread_data_set(&td), nullptr);\n m_task_dispatcher->allow_fifo_task(m_orig_fifo_tasks_allowed);\n m_task_dispatcher->m_properties.critical_task_allowed = m_orig_critical_task_allowed;\n if (m_orig_arena) {\n td.my_arena->my_observers.notify_exit_observers(td.my_last_observer, /*worker*/ false);\n td.my_last_observer = m_orig_last_observer;\n\n // Notify the market that this thread releasing a one slot\n // that can be used by a worker thread.\n if (td.my_arena_index >= td.my_arena->my_num_reserved_slots) {\n td.my_arena->request_workers(/* mandatory_delta = */ 0, /* workers_delta = */ 1);\n }\n\n td.leave_task_dispatcher();\n td.my_arena_slot->release();\n td.my_arena->my_exit_monitors.notify_one(); // do not relax!\n td.my_is_registered = m_orig_is_thread_registered;\n td.attach_arena(*m_orig_arena, m_orig_slot_index);\n td.attach_task_dispatcher(*m_orig_execute_data_ext.task_disp);\n __TBB_ASSERT(td.my_inbox.is_idle_state(false), nullptr);\n }\n td.my_task_dispatcher->m_execute_data_ext = m_orig_execute_data_ext;\n }\n\nprivate:\n execution_data_ext m_orig_execute_data_ext{};\n arena* m_orig_arena{ nullptr };\n observer_proxy* m_orig_last_observer{ nullptr };\n task_dispatcher* m_task_dispatcher{ nullptr };\n unsigned m_orig_slot_index{};\n bool m_orig_fifo_tasks_allowed{};\n bool m_orig_critical_task_allowed{};\n bool m_orig_is_thread_registered{};\n};\n\nclass delegated_task : public d1::task {\n d1::delegate_base& m_delegate;\n concurrent_monitor& m_monitor;\n d1::wait_context& m_wait_ctx;\n std::atomic m_completed;\n d1::task* execute(d1::execution_data& ed) override {\n const execution_data_ext& ed_ext = static_cast(ed);\n execution_data_ext orig_execute_data_ext = ed_ext.task_disp->m_execute_data_ext;\n __TBB_ASSERT(&ed_ext.task_disp->m_execute_data_ext == &ed,\n \"The execute data shall point to the current task dispatcher execute data\");\n __TBB_ASSERT(ed_ext.task_disp->m_execute_data_ext.isolation == no_isolation, nullptr);\n\n ed_ext.task_disp->m_execute_data_ext.context = ed_ext.task_disp->get_thread_data().my_arena->my_default_ctx;\n bool fifo_task_allowed = ed_ext.task_disp->allow_fifo_task(true);\n try_call([&] {\n m_delegate();\n }).on_completion([&] {\n ed_ext.task_disp->m_execute_data_ext = orig_execute_data_ext;\n ed_ext.task_disp->allow_fifo_task(fifo_task_allowed);\n });\n\n finalize();\n return nullptr;\n }\n d1::task* cancel(d1::execution_data&) override {\n finalize();\n return nullptr;\n }\n void finalize() {\n m_wait_ctx.release(); // must precede the wakeup\n m_monitor.notify([this] (std::uintptr_t ctx) {\n return ctx == std::uintptr_t(&m_delegate);\n }); // do not relax, it needs a fence!\n m_completed.store(true, std::memory_order_release);\n }\npublic:\n delegated_task(d1::delegate_base& d, concurrent_monitor& s, d1::wait_context& wo)\n : m_delegate(d), m_monitor(s), m_wait_ctx(wo), m_completed{ false }{}\n ~delegated_task() override {\n // The destructor can be called earlier than the m_monitor is notified\n // because the waiting thread can be released after m_wait_ctx.release_wait.\n // To close that race we wait for the m_completed signal.\n spin_wait_until_eq(m_completed, true);\n }\n};\n\nvoid task_arena_impl::execute(d1::task_arena_base& ta, d1::delegate_base& d) {\n arena* a = ta.my_arena.load(std::memory_order_relaxed);\n __TBB_ASSERT(a != nullptr, nullptr);\n thread_data* td = governor::get_thread_data();\n\n bool same_arena = td->my_arena == a;\n std::size_t index1 = td->my_arena_index;\n if (!same_arena) {\n index1 = a->occupy_free_slot(*td);\n if (index1 == arena::out_of_arena) {\n concurrent_monitor::thread_context waiter((std::uintptr_t)&d);\n d1::wait_context wo(1);\n d1::task_group_context exec_context(d1::task_group_context::isolated);\n task_group_context_impl::copy_fp_settings(exec_context, *a->my_default_ctx);\n\n delegated_task dt(d, a->my_exit_monitors, wo);\n a->enqueue_task( dt, exec_context, *td);\n size_t index2 = arena::out_of_arena;\n do {\n a->my_exit_monitors.prepare_wait(waiter);\n if (!wo.continue_execution()) {\n a->my_exit_monitors.cancel_wait(waiter);\n break;\n }\n index2 = a->occupy_free_slot(*td);\n if (index2 != arena::out_of_arena) {\n a->my_exit_monitors.cancel_wait(waiter);\n nested_arena_context scope(*td, *a, index2 );\n r1::wait(wo, exec_context);\n __TBB_ASSERT(!exec_context.my_exception.load(std::memory_order_relaxed), nullptr); // exception can be thrown above, not deferred\n break;\n }\n a->my_exit_monitors.commit_wait(waiter);\n } while (wo.continue_execution());\n if (index2 == arena::out_of_arena) {\n // notify a waiting thread even if this thread did not enter arena,\n // in case it was woken by a leaving thread but did not need to enter\n a->my_exit_monitors.notify_one(); // do not relax!\n }\n // process possible exception\n auto exception = exec_context.my_exception.load(std::memory_order_acquire);\n if (exception) {\n __TBB_ASSERT(exec_context.is_group_execution_cancelled(), \"The task group context with an exception should be canceled.\");\n exception->throw_self();\n }\n __TBB_ASSERT(governor::is_thread_data_set(td), nullptr);\n return;\n } // if (index1 == arena::out_of_arena)\n } // if (!same_arena)\n\n context_guard_helper context_guard;\n context_guard.set_ctx(a->my_default_ctx);\n nested_arena_context scope(*td, *a, index1);\n#if _WIN64\n try {\n#endif\n d();\n __TBB_ASSERT(same_arena || governor::is_thread_data_set(td), nullptr);\n#if _WIN64\n } catch (...) {\n context_guard.restore_default();\n throw;\n }\n#endif\n}\n\nvoid task_arena_impl::wait(d1::task_arena_base& ta) {\n arena* a = ta.my_arena.load(std::memory_order_relaxed);\n __TBB_ASSERT(a != nullptr, nullptr);\n thread_data* td = governor::get_thread_data();\n __TBB_ASSERT_EX(td, \"Scheduler is not initialized\");\n __TBB_ASSERT(td->my_arena != a || td->my_arena_index == 0, \"internal_wait is not supported within a worker context\" );\n if (a->my_max_num_workers != 0) {\n while (a->num_workers_active() || !a->is_empty()) {\n yield();\n }\n }\n}\n\nint task_arena_impl::max_concurrency(const d1::task_arena_base *ta) {\n arena* a = nullptr;\n if( ta ) // for special cases of ta->max_concurrency()\n a = ta->my_arena.load(std::memory_order_relaxed);\n else if( thread_data* td = governor::get_thread_data_if_initialized() )\n a = td->my_arena; // the current arena if any\n\n if( a ) { // Get parameters from the arena\n __TBB_ASSERT( !ta || ta->my_max_concurrency==1, nullptr);\n int mandatory_worker = 0;\n if (a->is_arena_workerless() && a->my_num_reserved_slots == 1) {\n mandatory_worker = a->my_mandatory_concurrency.test() ? 1 : 0;\n }\n return a->my_num_reserved_slots + a->my_max_num_workers + mandatory_worker;\n }\n\n if (ta && ta->my_max_concurrency == 1) {\n return 1;\n }\n\n#if __TBB_ARENA_BINDING\n if (ta) {\n d1::constraints arena_constraints = d1::constraints{}\n .set_numa_id(ta->my_numa_id)\n .set_core_type(ta->core_type())\n .set_max_threads_per_core(ta->max_threads_per_core());\n return (int)default_concurrency(arena_constraints);\n }\n#endif /*!__TBB_ARENA_BINDING*/\n\n __TBB_ASSERT(!ta || ta->my_max_concurrency==d1::task_arena_base::automatic, nullptr);\n return int(governor::default_num_threads());\n}\n\nvoid isolate_within_arena(d1::delegate_base& d, std::intptr_t isolation) {\n // TODO: Decide what to do if the scheduler is not initialized. Is there a use case for it?\n thread_data* tls = governor::get_thread_data();\n assert_pointers_valid(tls, tls->my_task_dispatcher);\n task_dispatcher* dispatcher = tls->my_task_dispatcher;\n isolation_type previous_isolation = dispatcher->m_execute_data_ext.isolation;\n try_call([&] {\n // We temporarily change the isolation tag of the currently running task. It will be restored in the destructor of the guard.\n isolation_type current_isolation = isolation ? isolation : reinterpret_cast(&d);\n // Save the current isolation value and set new one\n previous_isolation = dispatcher->set_isolation(current_isolation);\n // Isolation within this callable\n d();\n }).on_completion([&] {\n __TBB_ASSERT(governor::get_thread_data()->my_task_dispatcher == dispatcher, nullptr);\n dispatcher->set_isolation(previous_isolation);\n });\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/arena.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_arena_H\n#define _TBB_arena_H\n\n#include \n#include \n\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/spin_mutex.h\"\n\n#include \"scheduler_common.h\"\n#include \"intrusive_list.h\"\n#include \"task_stream.h\"\n#include \"arena_slot.h\"\n#include \"rml_tbb.h\"\n#include \"mailbox.h\"\n#include \"governor.h\"\n#include \"concurrent_monitor.h\"\n#include \"observer_proxy.h\"\n#include \"thread_control_monitor.h\"\n#include \"threading_control_client.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass task_dispatcher;\nclass task_group_context;\nclass threading_control;\nclass allocate_root_with_context_proxy;\n\n#if __TBB_ARENA_BINDING\nclass numa_binding_observer;\n#endif /*__TBB_ARENA_BINDING*/\n\n//! Bounded coroutines cache LIFO ring buffer\nclass arena_co_cache {\n //! Ring buffer storage\n task_dispatcher** my_co_scheduler_cache;\n //! Current cache index\n unsigned my_head;\n //! Cache capacity for arena\n unsigned my_max_index;\n //! Accessor lock for modification operations\n tbb::spin_mutex my_co_cache_mutex;\n\n unsigned next_index() {\n return ( my_head == my_max_index ) ? 0 : my_head + 1;\n }\n\n unsigned prev_index() {\n return ( my_head == 0 ) ? my_max_index : my_head - 1;\n }\n\n bool internal_empty() {\n return my_co_scheduler_cache[prev_index()] == nullptr;\n }\n\n void internal_task_dispatcher_cleanup(task_dispatcher* to_cleanup) {\n to_cleanup->~task_dispatcher();\n cache_aligned_deallocate(to_cleanup);\n }\n\npublic:\n void init(unsigned cache_capacity) {\n std::size_t alloc_size = cache_capacity * sizeof(task_dispatcher*);\n my_co_scheduler_cache = (task_dispatcher**)cache_aligned_allocate(alloc_size);\n std::memset( my_co_scheduler_cache, 0, alloc_size );\n my_head = 0;\n my_max_index = cache_capacity - 1;\n }\n\n void cleanup() {\n while (task_dispatcher* to_cleanup = pop()) {\n internal_task_dispatcher_cleanup(to_cleanup);\n }\n cache_aligned_deallocate(my_co_scheduler_cache);\n }\n\n //! Insert scheduler to the current available place.\n //! Replace an old value, if necessary.\n void push(task_dispatcher* s) {\n task_dispatcher* to_cleanup = nullptr;\n {\n tbb::spin_mutex::scoped_lock lock(my_co_cache_mutex);\n // Check if we are replacing some existing buffer entrance\n if (my_co_scheduler_cache[my_head] != nullptr) {\n to_cleanup = my_co_scheduler_cache[my_head];\n }\n // Store the cached value\n my_co_scheduler_cache[my_head] = s;\n // Move head index to the next slot\n my_head = next_index();\n }\n // Cleanup replaced buffer if any\n if (to_cleanup) {\n internal_task_dispatcher_cleanup(to_cleanup);\n }\n }\n\n //! Get a cached scheduler if any\n task_dispatcher* pop() {\n tbb::spin_mutex::scoped_lock lock(my_co_cache_mutex);\n // No cached coroutine\n if (internal_empty()) {\n return nullptr;\n }\n // Move head index to the currently available value\n my_head = prev_index();\n // Retrieve the value from the buffer\n task_dispatcher* to_return = my_co_scheduler_cache[my_head];\n // Clear the previous entrance value\n my_co_scheduler_cache[my_head] = nullptr;\n return to_return;\n }\n};\n\nstruct stack_anchor_type {\n stack_anchor_type() = default;\n stack_anchor_type(const stack_anchor_type&) = delete;\n};\n\nclass atomic_flag {\n static const std::uintptr_t SET = 1;\n static const std::uintptr_t UNSET = 0;\n std::atomic my_state{UNSET};\npublic:\n bool test_and_set() {\n std::uintptr_t state = my_state.load(std::memory_order_acquire);\n switch (state) {\n case SET:\n return false;\n default: /* busy */\n if (my_state.compare_exchange_strong(state, SET)) {\n // We interrupted clear transaction\n return false;\n }\n if (state != UNSET) {\n // We lost our epoch\n return false;\n }\n // We are too late but still in the same epoch\n __TBB_fallthrough;\n case UNSET:\n return my_state.compare_exchange_strong(state, SET);\n }\n }\n template \n bool try_clear_if(Pred&& pred) {\n std::uintptr_t busy = std::uintptr_t(&busy);\n std::uintptr_t state = my_state.load(std::memory_order_acquire);\n if (state == SET && my_state.compare_exchange_strong(state, busy)) {\n if (pred()) {\n return my_state.compare_exchange_strong(busy, UNSET);\n }\n // The result of the next operation is discarded, always false should be returned.\n my_state.compare_exchange_strong(busy, SET);\n }\n return false;\n }\n bool test(std::memory_order order = std::memory_order_acquire) {\n return my_state.load(order) != UNSET;\n }\n};\n\n//! The structure of an arena, except the array of slots.\n/** Separated in order to simplify padding.\n Intrusive list node base class is used by market to form a list of arenas. **/\n// TODO: Analyze arena_base cache lines placement\nstruct arena_base : padded {\n //! The number of workers that have been marked out by the resource manager to service the arena.\n std::atomic my_num_workers_allotted; // heavy use in stealing loop\n\n //! Reference counter for the arena.\n /** Worker and external thread references are counted separately: first several bits are for references\n from external thread threads or explicit task_arenas (see arena::ref_external_bits below);\n the rest counts the number of workers servicing the arena. */\n std::atomic my_references; // heavy use in stealing loop\n\n //! The maximal number of currently busy slots.\n std::atomic my_limit; // heavy use in stealing loop\n\n //! Task pool for the tasks scheduled via task::enqueue() method.\n /** Such scheduling guarantees eventual execution even if\n - new tasks are constantly coming (by extracting scheduled tasks in\n relaxed FIFO order);\n - the enqueuing thread does not call any of wait_for_all methods. **/\n task_stream my_fifo_task_stream; // heavy use in stealing loop\n\n //! Task pool for the tasks scheduled via tbb::resume() function.\n task_stream my_resume_task_stream; // heavy use in stealing loop\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\n //! Task pool for the tasks with critical property set.\n /** Critical tasks are scheduled for execution ahead of other sources (including local task pool\n and even bypassed tasks) unless the thread already executes a critical task in an outer\n dispatch loop **/\n // used on the hot path of the task dispatch loop\n task_stream my_critical_task_stream;\n#endif\n\n //! The total number of workers that are requested from the resource manager.\n int my_total_num_workers_requested;\n\n //! The index in the array of per priority lists of arenas this object is in.\n /*const*/ unsigned my_priority_level;\n\n //! The max priority level of arena in permit manager.\n std::atomic my_is_top_priority{false};\n\n //! Current task pool state and estimate of available tasks amount.\n atomic_flag my_pool_state;\n\n //! The list of local observers attached to this arena.\n observer_list my_observers;\n\n#if __TBB_ARENA_BINDING\n //! Pointer to internal observer that allows to bind threads in arena to certain NUMA node.\n numa_binding_observer* my_numa_binding_observer{nullptr};\n#endif /*__TBB_ARENA_BINDING*/\n\n // Below are rarely modified members\n\n threading_control* my_threading_control;\n\n //! Default task group context.\n d1::task_group_context* my_default_ctx;\n\n //! Waiting object for external threads that cannot join the arena.\n concurrent_monitor my_exit_monitors;\n\n //! Coroutines (task_dispathers) cache buffer\n arena_co_cache my_co_cache;\n\n // arena needs an extra worker despite the arena limit\n atomic_flag my_mandatory_concurrency;\n // the number of local mandatory concurrency requests\n int my_mandatory_requests;\n\n //! The number of slots in the arena.\n unsigned my_num_slots;\n //! The number of reserved slots (can be occupied only by external threads).\n unsigned my_num_reserved_slots;\n //! The number of workers requested by the external thread owning the arena.\n unsigned my_max_num_workers;\n\n threading_control_client my_tc_client;\n\n#if TBB_USE_ASSERT\n //! Used to trap accesses to the object after its destruction.\n std::uintptr_t my_guard;\n#endif /* TBB_USE_ASSERT */\n}; // struct arena_base\n\nclass arena: public padded\n{\npublic:\n using base_type = padded;\n\n //! Types of work advertised by advertise_new_work()\n enum new_work_type {\n work_spawned,\n wakeup,\n work_enqueued\n };\n\n //! Constructor\n arena(threading_control* control, unsigned max_num_workers, unsigned num_reserved_slots, unsigned priority_level);\n\n //! Allocate an instance of arena.\n static arena& allocate_arena(threading_control* control, unsigned num_slots, unsigned num_reserved_slots,\n unsigned priority_level);\n\n static arena& create(threading_control* control, unsigned num_slots, unsigned num_reserved_slots, unsigned arena_priority_level, d1::constraints constraints = d1::constraints{});\n\n static int unsigned num_arena_slots ( unsigned num_slots, unsigned num_reserved_slots ) {\n return num_reserved_slots == 0 ? num_slots : max(2u, num_slots);\n }\n\n static int allocation_size( unsigned num_slots ) {\n return sizeof(base_type) + num_slots * (sizeof(mail_outbox) + sizeof(arena_slot) + sizeof(task_dispatcher));\n }\n\n //! Get reference to mailbox corresponding to given slot_id\n mail_outbox& mailbox( d1::slot_id slot ) {\n __TBB_ASSERT( slot != d1::no_slot, \"affinity should be specified\" );\n\n return reinterpret_cast(this)[-(int)(slot+1)]; // cast to 'int' is redundant but left for readability\n }\n\n //! Completes arena shutdown, destructs and deallocates it.\n void free_arena();\n\n //! The number of least significant bits for external references\n static const unsigned ref_external_bits = 12; // up to 4095 external and 1M workers\n\n //! Reference increment values for externals and workers\n static const unsigned ref_external = 1;\n static const unsigned ref_worker = 1 << ref_external_bits;\n\n //! The number of workers active in the arena.\n unsigned num_workers_active() const {\n return my_references.load(std::memory_order_acquire) >> ref_external_bits;\n }\n\n //! Check if the recall is requested by the market.\n bool is_recall_requested() const {\n return num_workers_active() > my_num_workers_allotted.load(std::memory_order_relaxed);\n }\n\n void request_workers(int mandatory_delta, int workers_delta, bool wakeup_threads = false);\n\n //! If necessary, raise a flag that there is new job in arena.\n template void advertise_new_work();\n\n //! Attempts to steal a task from a randomly chosen arena slot\n d1::task* steal_task(unsigned arena_index, FastRandom& frnd, execution_data_ext& ed, isolation_type isolation);\n\n //! Get a task from a global starvation resistant queue\n template\n d1::task* get_stream_task(task_stream& stream, unsigned& hint);\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\n //! Tries to find a critical task in global critical task stream\n d1::task* get_critical_task(unsigned& hint, isolation_type isolation);\n#endif\n\n //! Check if there is job anywhere in arena.\n void out_of_work();\n\n //! enqueue a task into starvation-resistance queue\n void enqueue_task(d1::task&, d1::task_group_context&, thread_data&);\n\n //! Registers the worker with the arena and enters TBB scheduler dispatch loop\n void process(thread_data&);\n\n //! Notification that the thread leaves its arena\n\n void on_thread_leaving(unsigned ref_param);\n\n //! Check for the presence of enqueued tasks\n bool has_enqueued_tasks();\n\n //! Check for the presence of any tasks\n bool has_tasks();\n\n bool is_empty() { return my_pool_state.test() == /* EMPTY */ false; }\n\n thread_control_monitor& get_waiting_threads_monitor();\n\n static const std::size_t out_of_arena = ~size_t(0);\n //! Tries to occupy a slot in the arena. On success, returns the slot index; if no slot is available, returns out_of_arena.\n template \n std::size_t occupy_free_slot(thread_data&);\n //! Tries to occupy a slot in the specified range.\n std::size_t occupy_free_slot_in_range(thread_data& tls, std::size_t lower, std::size_t upper);\n\n std::uintptr_t calculate_stealing_threshold();\n\n unsigned priority_level() { return my_priority_level; }\n\n bool has_request() { return my_total_num_workers_requested; }\n\n unsigned references() const { return my_references.load(std::memory_order_acquire); }\n\n bool is_arena_workerless() const { return my_max_num_workers == 0; }\n\n void set_top_priority(bool);\n\n bool is_top_priority() const;\n\n bool is_joinable() const {\n return num_workers_active() < my_num_workers_allotted.load(std::memory_order_relaxed);\n }\n\n bool try_join();\n\n void set_allotment(unsigned allotment);\n\n int update_concurrency(unsigned concurrency);\n\n std::pair update_request(int mandatory_delta, int workers_delta);\n\n /** Must be the last data field */\n arena_slot my_slots[1];\n}; // class arena\n\ntemplate \nvoid arena::advertise_new_work() {\n bool is_mandatory_needed = false;\n bool are_workers_needed = false;\n\n if (work_type != work_spawned) {\n // Local memory fence here and below is required to avoid missed wakeups; see the comment below.\n // Starvation resistant tasks require concurrency, so missed wakeups are unacceptable.\n atomic_fence_seq_cst();\n }\n\n if (work_type == work_enqueued && my_num_slots > my_num_reserved_slots) {\n is_mandatory_needed = my_mandatory_concurrency.test_and_set();\n }\n\n // Double-check idiom that, in case of spawning, is deliberately sloppy about memory fences.\n // Technically, to avoid missed wakeups, there should be a full memory fence between the point we\n // released the task pool (i.e. spawned task) and read the arena's state. However, adding such a\n // fence might hurt overall performance more than it helps, because the fence would be executed\n // on every task pool release, even when stealing does not occur. Since TBB allows parallelism,\n // but never promises parallelism, the missed wakeup is not a correctness problem.\n are_workers_needed = my_pool_state.test_and_set();\n\n if (is_mandatory_needed || are_workers_needed) {\n int mandatory_delta = is_mandatory_needed ? 1 : 0;\n int workers_delta = are_workers_needed ? my_max_num_workers : 0;\n\n if (is_mandatory_needed && is_arena_workerless()) {\n // Set workers_delta to 1 to keep arena invariants consistent\n workers_delta = 1;\n }\n\n request_workers(mandatory_delta, workers_delta, /* wakeup_threads = */ true);\n }\n}\n\ninline d1::task* arena::steal_task(unsigned arena_index, FastRandom& frnd, execution_data_ext& ed, isolation_type isolation) {\n auto slot_num_limit = my_limit.load(std::memory_order_relaxed);\n if (slot_num_limit == 1) {\n // No slots to steal from\n return nullptr;\n }\n // Try to steal a task from a random victim.\n std::size_t k = frnd.get() % (slot_num_limit - 1);\n // The following condition excludes the external thread that might have\n // already taken our previous place in the arena from the list .\n // of potential victims. But since such a situation can take\n // place only in case of significant oversubscription, keeping\n // the checks simple seems to be preferable to complicating the code.\n if (k >= arena_index) {\n ++k; // Adjusts random distribution to exclude self\n }\n arena_slot* victim = &my_slots[k];\n d1::task **pool = victim->task_pool.load(std::memory_order_relaxed);\n d1::task *t = nullptr;\n if (pool == EmptyTaskPool || !(t = victim->steal_task(*this, isolation, k))) {\n return nullptr;\n }\n if (task_accessor::is_proxy_task(*t)) {\n task_proxy &tp = *(task_proxy*)t;\n d1::slot_id slot = tp.slot;\n t = tp.extract_task();\n if (!t) {\n // Proxy was empty, so it's our responsibility to free it\n tp.allocator.delete_object(&tp, ed);\n return nullptr;\n }\n // Note affinity is called for any stolen task (proxy or general)\n ed.affinity_slot = slot;\n } else {\n // Note affinity is called for any stolen task (proxy or general)\n ed.affinity_slot = d1::any_slot;\n }\n // Update task owner thread id to identify stealing\n ed.original_slot = k;\n return t;\n}\n\ntemplate\ninline d1::task* arena::get_stream_task(task_stream& stream, unsigned& hint) {\n if (stream.empty())\n return nullptr;\n return stream.pop(subsequent_lane_selector(hint));\n}\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\n// Retrieves critical task respecting isolation level, if provided. The rule is:\n// 1) If no outer critical task and no isolation => take any critical task\n// 2) If working on an outer critical task and no isolation => cannot take any critical task\n// 3) If no outer critical task but isolated => respect isolation\n// 4) If working on an outer critical task and isolated => respect isolation\n// Hint is used to keep some LIFO-ness, start search with the lane that was used during push operation.\ninline d1::task* arena::get_critical_task(unsigned& hint, isolation_type isolation) {\n if (my_critical_task_stream.empty())\n return nullptr;\n\n if ( isolation != no_isolation ) {\n return my_critical_task_stream.pop_specific( hint, isolation );\n } else {\n return my_critical_task_stream.pop(preceding_lane_selector(hint));\n }\n}\n#endif // __TBB_PREVIEW_CRITICAL_TASKS\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_arena_H */\n" }, { "path": "src/tbb/src/tbb/arena_slot.cpp", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"arena_slot.h\"\n#include \"arena.h\"\n#include \"thread_data.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// Arena Slot\n//------------------------------------------------------------------------\nd1::task* arena_slot::get_task_impl(size_t T, execution_data_ext& ed, bool& tasks_omitted, isolation_type isolation) {\n __TBB_ASSERT(tail.load(std::memory_order_relaxed) <= T || is_local_task_pool_quiescent(),\n \"Is it safe to get a task at position T?\");\n\n d1::task* result = task_pool_ptr[T];\n __TBB_ASSERT(!is_poisoned( result ), \"The poisoned task is going to be processed\");\n\n if (!result) {\n return nullptr;\n }\n bool omit = isolation != no_isolation && isolation != task_accessor::isolation(*result);\n if (!omit && !task_accessor::is_proxy_task(*result)) {\n return result;\n } else if (omit) {\n tasks_omitted = true;\n return nullptr;\n }\n\n task_proxy& tp = static_cast(*result);\n d1::slot_id aff_id = tp.slot;\n if ( d1::task *t = tp.extract_task() ) {\n ed.affinity_slot = aff_id;\n return t;\n }\n // Proxy was empty, so it's our responsibility to free it\n tp.allocator.delete_object(&tp, ed);\n\n if ( tasks_omitted ) {\n task_pool_ptr[T] = nullptr;\n }\n return nullptr;\n}\n\nd1::task* arena_slot::get_task(execution_data_ext& ed, isolation_type isolation) {\n __TBB_ASSERT(is_task_pool_published(), nullptr);\n // The current task position in the task pool.\n std::size_t T0 = tail.load(std::memory_order_relaxed);\n // The bounds of available tasks in the task pool. H0 is only used when the head bound is reached.\n std::size_t H0 = (std::size_t)-1, T = T0;\n d1::task* result = nullptr;\n bool task_pool_empty = false;\n bool tasks_omitted = false;\n do {\n __TBB_ASSERT( !result, nullptr );\n // The full fence is required to sync the store of `tail` with the load of `head` (write-read barrier)\n T = --tail;\n // The acquire load of head is required to guarantee consistency of our task pool\n // when a thief rolls back the head.\n if ( (std::intptr_t)( head.load(std::memory_order_acquire) ) > (std::intptr_t)T ) {\n acquire_task_pool();\n H0 = head.load(std::memory_order_relaxed);\n if ( (std::intptr_t)H0 > (std::intptr_t)T ) {\n // The thief has not backed off - nothing to grab.\n __TBB_ASSERT( H0 == head.load(std::memory_order_relaxed)\n && T == tail.load(std::memory_order_relaxed)\n && H0 == T + 1, \"victim/thief arbitration algorithm failure\" );\n reset_task_pool_and_leave();\n // No tasks in the task pool.\n task_pool_empty = true;\n break;\n } else if ( H0 == T ) {\n // There is only one task in the task pool.\n reset_task_pool_and_leave();\n task_pool_empty = true;\n } else {\n // Release task pool if there are still some tasks.\n // After the release, the tail will be less than T, thus a thief\n // will not attempt to get a task at position T.\n release_task_pool();\n }\n }\n result = get_task_impl( T, ed, tasks_omitted, isolation );\n if ( result ) {\n poison_pointer( task_pool_ptr[T] );\n break;\n } else if ( !tasks_omitted ) {\n poison_pointer( task_pool_ptr[T] );\n __TBB_ASSERT( T0 == T+1, nullptr );\n T0 = T;\n }\n } while ( !result && !task_pool_empty );\n\n if ( tasks_omitted ) {\n if ( task_pool_empty ) {\n // All tasks have been checked. The task pool should be in reset state.\n // We just restore the bounds for the available tasks.\n // TODO: Does it have sense to move them to the beginning of the task pool?\n __TBB_ASSERT( is_quiescent_local_task_pool_reset(), nullptr );\n if ( result ) {\n // If we have a task, it should be at H0 position.\n __TBB_ASSERT( H0 == T, nullptr );\n ++H0;\n }\n __TBB_ASSERT( H0 <= T0, nullptr );\n if ( H0 < T0 ) {\n // Restore the task pool if there are some tasks.\n head.store(H0, std::memory_order_relaxed);\n tail.store(T0, std::memory_order_relaxed);\n // The release fence is used in publish_task_pool.\n publish_task_pool();\n // Synchronize with snapshot as we published some tasks.\n ed.task_disp->m_thread_data->my_arena->advertise_new_work();\n }\n } else {\n // A task has been obtained. We need to make a hole in position T.\n __TBB_ASSERT( is_task_pool_published(), nullptr );\n __TBB_ASSERT( result, nullptr );\n task_pool_ptr[T] = nullptr;\n tail.store(T0, std::memory_order_release);\n // Synchronize with snapshot as we published some tasks.\n // TODO: consider some approach not to call wakeup for each time. E.g. check if the tail reached the head.\n ed.task_disp->m_thread_data->my_arena->advertise_new_work();\n }\n }\n\n __TBB_ASSERT( (std::intptr_t)tail.load(std::memory_order_relaxed) >= 0, nullptr );\n __TBB_ASSERT( result || tasks_omitted || is_quiescent_local_task_pool_reset(), nullptr );\n return result;\n}\n\nd1::task* arena_slot::steal_task(arena& a, isolation_type isolation, std::size_t slot_index) {\n d1::task** victim_pool = lock_task_pool();\n if (!victim_pool) {\n return nullptr;\n }\n d1::task* result = nullptr;\n std::size_t H = head.load(std::memory_order_relaxed); // mirror\n std::size_t H0 = H;\n bool tasks_omitted = false;\n do {\n // The full fence is required to sync the store of `head` with the load of `tail` (write-read barrier)\n H = ++head;\n // The acquire load of tail is required to guarantee consistency of victim_pool\n // because the owner synchronizes task spawning via tail.\n if ((std::intptr_t)H > (std::intptr_t)(tail.load(std::memory_order_acquire))) {\n // Stealing attempt failed, deque contents has not been changed by us\n head.store( /*dead: H = */ H0, std::memory_order_relaxed );\n __TBB_ASSERT( !result, nullptr );\n goto unlock;\n }\n result = victim_pool[H-1];\n __TBB_ASSERT( !is_poisoned( result ), nullptr );\n\n if (result) {\n if (isolation == no_isolation || isolation == task_accessor::isolation(*result)) {\n if (!task_accessor::is_proxy_task(*result)) {\n break;\n }\n task_proxy& tp = *static_cast(result);\n // If mailed task is likely to be grabbed by its destination thread, skip it.\n if (!task_proxy::is_shared(tp.task_and_tag) || !tp.outbox->recipient_is_idle() || a.mailbox(slot_index).recipient_is_idle()) {\n break;\n }\n }\n // The task cannot be executed either due to isolation or proxy constraints.\n result = nullptr;\n tasks_omitted = true;\n } else if (!tasks_omitted) {\n // Cleanup the task pool from holes until a task is skipped.\n __TBB_ASSERT( H0 == H-1, nullptr );\n poison_pointer( victim_pool[H0] );\n H0 = H;\n }\n } while (!result);\n __TBB_ASSERT( result, nullptr );\n\n // emit \"task was consumed\" signal\n poison_pointer( victim_pool[H-1] );\n if (tasks_omitted) {\n // Some proxies in the task pool have been omitted. Set the stolen task to nullptr.\n victim_pool[H-1] = nullptr;\n // The release store synchronizes the victim_pool update(the store of nullptr).\n head.store( /*dead: H = */ H0, std::memory_order_release );\n }\nunlock:\n unlock_task_pool(victim_pool);\n\n#if __TBB_PREFETCHING\n __TBB_cl_evict(&victim_slot.head);\n __TBB_cl_evict(&victim_slot.tail);\n#endif\n if (tasks_omitted) {\n // Synchronize with snapshot as the head and tail can be bumped which can falsely trigger EMPTY state\n a.advertise_new_work();\n }\n return result;\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/arena_slot.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_arena_slot_H\n#define _TBB_arena_slot_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n#include \"oneapi/tbb/detail/_task.h\"\n\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n\n#include \"misc.h\"\n#include \"mailbox.h\"\n#include \"scheduler_common.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass arena;\nclass task_group_context;\n\n//--------------------------------------------------------------------------------------------------------\n// Arena Slot\n//--------------------------------------------------------------------------------------------------------\n\nstatic d1::task** const EmptyTaskPool = nullptr;\nstatic d1::task** const LockedTaskPool = reinterpret_cast(~std::intptr_t(0));\n\nstruct alignas(max_nfs_size) arena_slot_shared_state {\n //! Scheduler of the thread attached to the slot\n /** Marks the slot as busy, and is used to iterate through the schedulers belonging to this arena **/\n std::atomic my_is_occupied;\n\n // Synchronization of access to Task pool\n /** Also is used to specify if the slot is empty or locked:\n 0 - empty\n -1 - locked **/\n std::atomic task_pool;\n\n //! Index of the first ready task in the deque.\n /** Modified by thieves, and by the owner during compaction/reallocation **/\n std::atomic head;\n};\n\nstruct alignas(max_nfs_size) arena_slot_private_state {\n //! Hint provided for operations with the container of starvation-resistant tasks.\n /** Modified by the owner thread (during these operations). **/\n unsigned hint_for_fifo_stream;\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\n //! Similar to 'hint_for_fifo_stream' but for critical tasks.\n unsigned hint_for_critical_stream;\n#endif\n\n //! Similar to 'hint_for_fifo_stream' but for the resume tasks.\n unsigned hint_for_resume_stream;\n\n //! Index of the element following the last ready task in the deque.\n /** Modified by the owner thread. **/\n std::atomic tail;\n\n //! Capacity of the primary task pool (number of elements - pointers to task).\n std::size_t my_task_pool_size;\n\n //! Task pool of the scheduler that owns this slot\n // TODO: previously was task**__TBB_atomic, but seems like not accessed on other thread\n d1::task** task_pool_ptr;\n};\n\nclass arena_slot : private arena_slot_shared_state, private arena_slot_private_state {\n friend class arena;\n friend class outermost_worker_waiter;\n friend class task_dispatcher;\n friend class thread_data;\n friend class nested_arena_context;\n\n //! The original task dispather associated with this slot\n task_dispatcher* my_default_task_dispatcher;\n\n#if TBB_USE_ASSERT\n void fill_with_canary_pattern ( std::size_t first, std::size_t last ) {\n for ( std::size_t i = first; i < last; ++i )\n poison_pointer(task_pool_ptr[i]);\n }\n#else\n void fill_with_canary_pattern ( size_t, std::size_t ) {}\n#endif /* TBB_USE_ASSERT */\n\n static constexpr std::size_t min_task_pool_size = 64;\n\n void allocate_task_pool( std::size_t n ) {\n std::size_t byte_size = ((n * sizeof(d1::task*) + max_nfs_size - 1) / max_nfs_size) * max_nfs_size;\n my_task_pool_size = byte_size / sizeof(d1::task*);\n task_pool_ptr = (d1::task**)cache_aligned_allocate(byte_size);\n // No need to clear the fresh deque since valid items are designated by the head and tail members.\n // But fill it with a canary pattern in the high vigilance debug mode.\n fill_with_canary_pattern( 0, my_task_pool_size );\n }\n\npublic:\n //! Deallocate task pool that was allocated by means of allocate_task_pool.\n void free_task_pool( ) {\n // TODO: understand the assertion and modify\n // __TBB_ASSERT( !task_pool /* TODO: == EmptyTaskPool */, nullptr);\n if( task_pool_ptr ) {\n __TBB_ASSERT( my_task_pool_size, nullptr);\n cache_aligned_deallocate( task_pool_ptr );\n task_pool_ptr = nullptr;\n my_task_pool_size = 0;\n }\n }\n\n //! Get a task from the local pool.\n /** Called only by the pool owner.\n Returns the pointer to the task or nullptr if a suitable task is not found.\n Resets the pool if it is empty. **/\n d1::task* get_task(execution_data_ext&, isolation_type);\n\n //! Steal task from slot's ready pool\n d1::task* steal_task(arena&, isolation_type, std::size_t);\n\n //! Some thread is now the owner of this slot\n void occupy() {\n __TBB_ASSERT(!my_is_occupied.load(std::memory_order_relaxed), nullptr);\n my_is_occupied.store(true, std::memory_order_release);\n }\n\n //! Try to occupy the slot\n bool try_occupy() {\n return !is_occupied() && my_is_occupied.exchange(true) == false;\n }\n\n //! Some thread is now the owner of this slot\n void release() {\n __TBB_ASSERT(my_is_occupied.load(std::memory_order_relaxed), nullptr);\n my_is_occupied.store(false, std::memory_order_release);\n }\n\n //! Spawn newly created tasks\n void spawn(d1::task& t) {\n std::size_t T = prepare_task_pool(1);\n __TBB_ASSERT(is_poisoned(task_pool_ptr[T]), nullptr);\n task_pool_ptr[T] = &t;\n commit_spawned_tasks(T + 1);\n if (!is_task_pool_published()) {\n publish_task_pool();\n }\n }\n\n bool is_task_pool_published() const {\n return task_pool.load(std::memory_order_relaxed) != EmptyTaskPool;\n }\n\n bool is_empty() const {\n return task_pool.load(std::memory_order_relaxed) == EmptyTaskPool ||\n head.load(std::memory_order_relaxed) >= tail.load(std::memory_order_relaxed);\n }\n\n bool is_occupied() const {\n return my_is_occupied.load(std::memory_order_relaxed);\n }\n\n task_dispatcher& default_task_dispatcher() {\n __TBB_ASSERT(my_default_task_dispatcher != nullptr, nullptr);\n return *my_default_task_dispatcher;\n }\n\n void init_task_streams(unsigned h) {\n hint_for_fifo_stream = h;\n#if __TBB_RESUMABLE_TASKS\n hint_for_resume_stream = h;\n#endif\n#if __TBB_PREVIEW_CRITICAL_TASKS\n hint_for_critical_stream = h;\n#endif\n }\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\n unsigned& critical_hint() {\n return hint_for_critical_stream;\n }\n#endif\nprivate:\n //! Get a task from the local pool at specified location T.\n /** Returns the pointer to the task or nullptr if the task cannot be executed,\n e.g. proxy has been deallocated or isolation constraint is not met.\n tasks_omitted tells if some tasks have been omitted.\n Called only by the pool owner. The caller should guarantee that the\n position T is not available for a thief. **/\n d1::task* get_task_impl(size_t T, execution_data_ext& ed, bool& tasks_omitted, isolation_type isolation);\n\n //! Makes sure that the task pool can accommodate at least n more elements\n /** If necessary relocates existing task pointers or grows the ready task deque.\n * Returns (possible updated) tail index (not accounting for n). **/\n std::size_t prepare_task_pool(std::size_t num_tasks) {\n std::size_t T = tail.load(std::memory_order_relaxed); // mirror\n if ( T + num_tasks <= my_task_pool_size ) {\n return T;\n }\n\n std::size_t new_size = num_tasks;\n if ( !my_task_pool_size ) {\n __TBB_ASSERT( !is_task_pool_published() && is_quiescent_local_task_pool_reset(), nullptr);\n __TBB_ASSERT( !task_pool_ptr, nullptr);\n if ( num_tasks < min_task_pool_size ) new_size = min_task_pool_size;\n allocate_task_pool( new_size );\n return 0;\n }\n acquire_task_pool();\n std::size_t H = head.load(std::memory_order_relaxed); // mirror\n d1::task** new_task_pool = task_pool_ptr;\n __TBB_ASSERT( my_task_pool_size >= min_task_pool_size, nullptr);\n // Count not skipped tasks. Consider using std::count_if.\n for ( std::size_t i = H; i < T; ++i )\n if ( new_task_pool[i] ) ++new_size;\n // If the free space at the beginning of the task pool is too short, we\n // are likely facing a pathological single-producer-multiple-consumers\n // scenario, and thus it's better to expand the task pool\n bool allocate = new_size > my_task_pool_size - min_task_pool_size/4;\n if ( allocate ) {\n // Grow task pool. As this operation is rare, and its cost is asymptotically\n // amortizable, we can tolerate new task pool allocation done under the lock.\n if ( new_size < 2 * my_task_pool_size )\n new_size = 2 * my_task_pool_size;\n allocate_task_pool( new_size ); // updates my_task_pool_size\n }\n // Filter out skipped tasks. Consider using std::copy_if.\n std::size_t T1 = 0;\n for ( std::size_t i = H; i < T; ++i ) {\n if ( new_task_pool[i] ) {\n task_pool_ptr[T1++] = new_task_pool[i];\n }\n }\n // Deallocate the previous task pool if a new one has been allocated.\n if ( allocate )\n cache_aligned_deallocate( new_task_pool );\n else\n fill_with_canary_pattern( T1, tail );\n // Publish the new state.\n commit_relocated_tasks( T1 );\n // assert_task_pool_valid();\n return T1;\n }\n\n //! Makes newly spawned tasks visible to thieves\n void commit_spawned_tasks(std::size_t new_tail) {\n __TBB_ASSERT (new_tail <= my_task_pool_size, \"task deque end was overwritten\");\n // emit \"task was released\" signal\n // Release fence is necessary to make sure that previously stored task pointers\n // are visible to thieves.\n tail.store(new_tail, std::memory_order_release);\n }\n\n //! Used by workers to enter the task pool\n /** Does not lock the task pool in case if arena slot has been successfully grabbed. **/\n void publish_task_pool() {\n __TBB_ASSERT ( task_pool == EmptyTaskPool, \"someone else grabbed my arena slot?\" );\n __TBB_ASSERT ( head.load(std::memory_order_relaxed) < tail.load(std::memory_order_relaxed),\n \"entering arena without tasks to share\" );\n // Release signal on behalf of previously spawned tasks (when this thread was not in arena yet)\n task_pool.store(task_pool_ptr, std::memory_order_release );\n }\n\n //! Locks the local task pool\n /** Garbles task_pool for the duration of the lock. Requires correctly set task_pool_ptr.\n ATTENTION: This method is mostly the same as generic_scheduler::lock_task_pool(), with\n a little different logic of slot state checks (slot is either locked or points\n to our task pool). Thus if either of them is changed, consider changing the counterpart as well. **/\n void acquire_task_pool() {\n if (!is_task_pool_published()) {\n return; // we are not in arena - nothing to lock\n }\n bool sync_prepare_done = false;\n for( atomic_backoff b;;b.pause() ) {\n#if TBB_USE_ASSERT\n // Local copy of the arena slot task pool pointer is necessary for the next\n // assertion to work correctly to exclude asynchronous state transition effect.\n d1::task** tp = task_pool.load(std::memory_order_relaxed);\n __TBB_ASSERT( tp == LockedTaskPool || tp == task_pool_ptr, \"slot ownership corrupt?\" );\n#endif\n d1::task** expected = task_pool_ptr;\n if( task_pool.load(std::memory_order_relaxed) != LockedTaskPool &&\n task_pool.compare_exchange_strong(expected, LockedTaskPool ) ) {\n // We acquired our own slot\n break;\n } else if( !sync_prepare_done ) {\n // Start waiting\n sync_prepare_done = true;\n }\n // Someone else acquired a lock, so pause and do exponential backoff.\n }\n __TBB_ASSERT( task_pool.load(std::memory_order_relaxed) == LockedTaskPool, \"not really acquired task pool\" );\n }\n\n //! Unlocks the local task pool\n /** Restores task_pool munged by acquire_task_pool. Requires\n correctly set task_pool_ptr. **/\n void release_task_pool() {\n if ( !(task_pool.load(std::memory_order_relaxed) != EmptyTaskPool) )\n return; // we are not in arena - nothing to unlock\n __TBB_ASSERT( task_pool.load(std::memory_order_relaxed) == LockedTaskPool, \"arena slot is not locked\" );\n task_pool.store( task_pool_ptr, std::memory_order_release );\n }\n\n //! Locks victim's task pool, and returns pointer to it. The pointer can be nullptr.\n /** Garbles victim_arena_slot->task_pool for the duration of the lock. **/\n d1::task** lock_task_pool() {\n d1::task** victim_task_pool;\n for ( atomic_backoff backoff;; /*backoff pause embedded in the loop*/) {\n victim_task_pool = task_pool.load(std::memory_order_relaxed);\n // Microbenchmarks demonstrated that aborting stealing attempt when the\n // victim's task pool is locked degrade performance.\n // NOTE: Do not use comparison of head and tail indices to check for\n // the presence of work in the victim's task pool, as they may give\n // incorrect indication because of task pool relocations and resizes.\n if (victim_task_pool == EmptyTaskPool) {\n break;\n }\n d1::task** expected = victim_task_pool;\n if (victim_task_pool != LockedTaskPool && task_pool.compare_exchange_strong(expected, LockedTaskPool) ) {\n // We've locked victim's task pool\n break;\n } \n // Someone else acquired a lock, so pause and do exponential backoff.\n backoff.pause();\n }\n __TBB_ASSERT(victim_task_pool == EmptyTaskPool ||\n (task_pool.load(std::memory_order_relaxed) == LockedTaskPool &&\n victim_task_pool != LockedTaskPool), \"not really locked victim's task pool?\");\n return victim_task_pool;\n }\n\n //! Unlocks victim's task pool\n /** Restores victim_arena_slot->task_pool munged by lock_task_pool. **/\n void unlock_task_pool(d1::task** victim_task_pool) {\n __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, \"victim arena slot is not locked\");\n __TBB_ASSERT(victim_task_pool != LockedTaskPool, nullptr);\n task_pool.store(victim_task_pool, std::memory_order_release);\n }\n\n#if TBB_USE_ASSERT\n bool is_local_task_pool_quiescent() const {\n d1::task** tp = task_pool.load(std::memory_order_relaxed);\n return tp == EmptyTaskPool || tp == LockedTaskPool;\n }\n\n bool is_quiescent_local_task_pool_empty() const {\n __TBB_ASSERT(is_local_task_pool_quiescent(), \"Task pool is not quiescent\");\n return head.load(std::memory_order_relaxed) == tail.load(std::memory_order_relaxed);\n }\n\n bool is_quiescent_local_task_pool_reset() const {\n __TBB_ASSERT(is_local_task_pool_quiescent(), \"Task pool is not quiescent\");\n return head.load(std::memory_order_relaxed) == 0 && tail.load(std::memory_order_relaxed) == 0;\n }\n#endif // TBB_USE_ASSERT\n\n //! Leave the task pool\n /** Leaving task pool automatically releases the task pool if it is locked. **/\n void leave_task_pool() {\n __TBB_ASSERT(is_task_pool_published(), \"Not in arena\");\n // Do not reset my_arena_index. It will be used to (attempt to) re-acquire the slot next time\n __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, \"Task pool must be locked when leaving arena\");\n __TBB_ASSERT(is_quiescent_local_task_pool_empty(), \"Cannot leave arena when the task pool is not empty\");\n // No release fence is necessary here as this assignment precludes external\n // accesses to the local task pool when becomes visible. Thus it is harmless\n // if it gets hoisted above preceding local bookkeeping manipulations.\n task_pool.store(EmptyTaskPool, std::memory_order_relaxed);\n }\n\n //! Resets head and tail indices to 0, and leaves task pool\n /** The task pool must be locked by the owner (via acquire_task_pool).**/\n void reset_task_pool_and_leave() {\n __TBB_ASSERT(task_pool.load(std::memory_order_relaxed) == LockedTaskPool, \"Task pool must be locked when resetting task pool\");\n tail.store(0, std::memory_order_relaxed);\n head.store(0, std::memory_order_relaxed);\n leave_task_pool();\n }\n\n //! Makes relocated tasks visible to thieves and releases the local task pool.\n /** Obviously, the task pool must be locked when calling this method. **/\n void commit_relocated_tasks(std::size_t new_tail) {\n __TBB_ASSERT(is_local_task_pool_quiescent(), \"Task pool must be locked when calling commit_relocated_tasks()\");\n head.store(0, std::memory_order_relaxed);\n // Tail is updated last to minimize probability of a thread making arena\n // snapshot being misguided into thinking that this task pool is empty.\n tail.store(new_tail, std::memory_order_release);\n release_task_pool();\n }\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_arena_slot_H\n" }, { "path": "src/tbb/src/tbb/assert_impl.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_assert_impl_H\n#define __TBB_assert_impl_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n\n#include \n#include \n#include \n#include \n#if _MSC_VER && _DEBUG\n#include \n#endif\n\n#include \n\n#if __TBBMALLOC_BUILD\nnamespace rml { namespace internal {\n#else\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n#endif\n// TODO: consider extension for formatted error description string\nstatic void assertion_failure_impl(const char* location, int line, const char* expression, const char* comment) {\n\n std::fprintf(stderr, \"Assertion %s failed (located in the %s function, line in file: %d)\\n\",\n expression, location, line);\n\n if (comment) {\n std::fprintf(stderr, \"Detailed description: %s\\n\", comment);\n }\n#if _MSC_VER && _DEBUG\n if (1 == _CrtDbgReport(_CRT_ASSERT, location, line, \"tbb_debug.dll\", \"%s\\r\\n%s\", expression, comment?comment:\"\")) {\n _CrtDbgBreak();\n } else\n#endif\n {\n std::fflush(stderr);\n std::abort();\n }\n}\n\n// Do not move the definition into the assertion_failure function because it will require \"magic statics\".\n// It will bring a dependency on C++ runtime on some platforms while assert_impl.h is reused in tbbmalloc \n// that should not depend on C++ runtime\nstatic std::atomic assertion_state;\n\nvoid __TBB_EXPORTED_FUNC assertion_failure(const char* location, int line, const char* expression, const char* comment) {\n#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED\n // Workaround for erroneous \"unreachable code\" during assertion throwing using call_once\n // #pragma warning (push)\n // #pragma warning (disable: 4702)\n#endif\n // We cannot use std::call_once because it brings a dependency on C++ runtime on some platforms \n // while assert_impl.h is reused in tbbmalloc that should not depend on C++ runtime\n atomic_do_once([&](){ assertion_failure_impl(location, line, expression, comment); }, assertion_state);\n#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED\n // #pragma warning (pop)\n#endif\n}\n\n//! Report a runtime warning.\nvoid runtime_warning( const char* format, ... ) {\n char str[1024]; std::memset(str, 0, 1024);\n va_list args; va_start(args, format);\n vsnprintf( str, 1024-1, format, args);\n va_end(args);\n fprintf(stderr, \"TBB Warning: %s\\n\", str);\n}\n\n#if __TBBMALLOC_BUILD\n}} // namespaces rml::internal\n#else\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n#endif\n\n#endif // __TBB_assert_impl_H\n\n" }, { "path": "src/tbb/src/tbb/cancellation_disseminator.h", "content": "/*\n Copyright (c) 2022-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_cancellation_disseminator_H\n#define _TBB_cancellation_disseminator_H\n\n#include \"oneapi/tbb/mutex.h\"\n#include \"oneapi/tbb/task_group.h\"\n\n#include \"intrusive_list.h\"\n#include \"thread_data.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass cancellation_disseminator {\npublic:\n //! Finds all contexts affected by the state change and propagates the new state to them.\n /* The propagation is relayed to the cancellation_disseminator because tasks created by one\n external thread can be passed to and executed by other external threads. This means\n that context trees can span several arenas at once and thus state change\n propagation cannot be generally localized to one arena only.\n */\n bool propagate_task_group_state(std::atomic d1::task_group_context::*mptr_state, d1::task_group_context& src, uint32_t new_state) {\n if (src.my_may_have_children.load(std::memory_order_relaxed) != d1::task_group_context::may_have_children) {\n return true;\n }\n\n // The whole propagation algorithm is under the lock in order to ensure correctness\n // in case of concurrent state changes at the different levels of the context tree.\n threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex);\n // TODO: consider to use double-check idiom\n if ((src.*mptr_state).load(std::memory_order_relaxed) != new_state) {\n // Another thread has concurrently changed the state. Back down.\n return false;\n }\n\n // Advance global state propagation epoch\n ++the_context_state_propagation_epoch;\n // Propagate to all workers and external threads and sync up their local epochs with the global one\n // The whole propagation sequence is locked, thus no contention is expected\n for (auto& thr_data : my_threads_list) {\n thr_data.propagate_task_group_state(mptr_state, src, new_state);\n }\n\n return true;\n }\n\n void register_thread(thread_data& td) {\n threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex);\n my_threads_list.push_front(td);\n }\n\n void unregister_thread(thread_data& td) {\n threads_list_mutex_type::scoped_lock lock(my_threads_list_mutex);\n my_threads_list.remove(td);\n }\n\nprivate:\n using thread_data_list_type = intrusive_list;\n using threads_list_mutex_type = d1::mutex;\n\n threads_list_mutex_type my_threads_list_mutex;\n thread_data_list_type my_threads_list;\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_cancellation_disseminator_H\n" }, { "path": "src/tbb/src/tbb/co_context.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_co_context_H\n#define _TBB_co_context_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n\n#if __TBB_RESUMABLE_TASKS\n\n#include \n#include \n\n#if __TBB_RESUMABLE_TASKS_USE_THREADS\n\n#if _WIN32 || _WIN64\n#include \n#else\n#include \n#endif\n\n#include \n#include \"governor.h\"\n\n#elif _WIN32 || _WIN64\n#include \n#else\n// ucontext.h API is deprecated since macOS 10.6\n#if __APPLE__\n #if __INTEL_COMPILER\n // #pragma warning(push)\n // #pragma warning(disable:1478)\n #elif __clang__\n // #pragma clang diagnostic push\n // #pragma clang diagnostic ignored \"-Wdeprecated-declarations\"\n #endif\n#endif // __APPLE__\n\n#include \n#include // mprotect\n\n#include \"governor.h\" // default_page_size()\n\n#ifndef MAP_STACK\n// macOS* does not define MAP_STACK\n#define MAP_STACK 0\n#endif\n#ifndef MAP_ANONYMOUS\n// macOS* defines MAP_ANON, which is deprecated in Linux*.\n#define MAP_ANONYMOUS MAP_ANON\n#endif\n#endif // _WIN32 || _WIN64\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_RESUMABLE_TASKS_USE_THREADS\n struct coroutine_type {\n#if _WIN32 || _WIN64\n using handle_type = HANDLE;\n#else\n using handle_type = pthread_t;\n#endif\n\n handle_type my_thread;\n std::condition_variable my_condvar;\n std::mutex my_mutex;\n thread_data* my_thread_data{ nullptr };\n bool my_is_active{ true };\n };\n#elif _WIN32 || _WIN64\n typedef LPVOID coroutine_type;\n#else\n struct coroutine_type {\n coroutine_type() : my_context(), my_stack(), my_stack_size() {}\n ucontext_t my_context;\n void* my_stack;\n std::size_t my_stack_size;\n };\n#endif\n\n // Forward declaration of the coroutine API.\n void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg);\n void current_coroutine(coroutine_type& c);\n void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine);\n void destroy_coroutine(coroutine_type& c);\n\nclass co_context {\n enum co_state {\n co_invalid,\n co_suspended,\n co_executing,\n co_destroyed\n };\n coroutine_type my_coroutine;\n co_state my_state;\n\npublic:\n co_context(std::size_t stack_size, void* arg)\n : my_state(stack_size ? co_suspended : co_executing)\n {\n if (stack_size) {\n __TBB_ASSERT(arg != nullptr, nullptr);\n create_coroutine(my_coroutine, stack_size, arg);\n } else {\n current_coroutine(my_coroutine);\n }\n }\n\n ~co_context() {\n __TBB_ASSERT(1 << my_state & (1 << co_suspended | 1 << co_executing), nullptr);\n if (my_state == co_suspended) {\n#if __TBB_RESUMABLE_TASKS_USE_THREADS\n my_state = co_executing;\n#endif\n destroy_coroutine(my_coroutine);\n }\n my_state = co_destroyed;\n }\n\n void resume(co_context& target) {\n // Do not create non-trivial objects on the stack of this function. They might never be destroyed.\n __TBB_ASSERT(my_state == co_executing, nullptr);\n __TBB_ASSERT(target.my_state == co_suspended, nullptr);\n\n my_state = co_suspended;\n target.my_state = co_executing;\n\n // 'target' can reference an invalid object after swap_coroutine. Do not access it.\n swap_coroutine(my_coroutine, target.my_coroutine);\n\n __TBB_ASSERT(my_state == co_executing, nullptr);\n }\n};\n\n#if _WIN32 || _WIN64\n/* [[noreturn]] */ void __stdcall co_local_wait_for_all(void* arg) noexcept;\n#else\n/* [[noreturn]] */ void co_local_wait_for_all(unsigned hi, unsigned lo) noexcept;\n#endif\n\n#if __TBB_RESUMABLE_TASKS_USE_THREADS\nvoid handle_perror(int error_code, const char* what);\n\ninline void check(int error_code, const char* routine) {\n if (error_code) {\n handle_perror(error_code, routine);\n }\n}\n\nusing thread_data_t = std::pair;\n\n#if _WIN32 || _WIN64\ninline unsigned WINAPI coroutine_thread_func(void* d)\n#else\ninline void* coroutine_thread_func(void* d)\n#endif\n{\n thread_data_t& data = *static_cast(d);\n coroutine_type& c = data.first;\n void* arg = data.second;\n {\n std::unique_lock lock(c.my_mutex);\n __TBB_ASSERT(c.my_thread_data == nullptr, nullptr);\n c.my_is_active = false;\n\n // We read the data notify the waiting thread\n data.second = nullptr;\n c.my_condvar.notify_one();\n\n c.my_condvar.wait(lock, [&c] { return c.my_is_active == true; });\n }\n __TBB_ASSERT(c.my_thread_data != nullptr, nullptr);\n governor::set_thread_data(*c.my_thread_data);\n\n#if _WIN32 || _WIN64\n co_local_wait_for_all(arg);\n\n return 0;\n#else\n std::uintptr_t addr = std::uintptr_t(arg);\n unsigned lo = unsigned(addr);\n unsigned hi = unsigned(std::uint64_t(addr) >> 32);\n __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr);\n\n co_local_wait_for_all(hi, lo);\n\n return nullptr;\n#endif\n};\n\ninline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) {\n thread_data_t data{ c, arg };\n\n#if _WIN32 || _WIN64\n c.my_thread = (HANDLE)_beginthreadex(nullptr, unsigned(stack_size), coroutine_thread_func, &data, STACK_SIZE_PARAM_IS_A_RESERVATION, nullptr);\n if (!c.my_thread) {\n handle_perror(0, \"create_coroutine: _beginthreadex failed\\n\");\n }\n#else\n pthread_attr_t s;\n check(pthread_attr_init(&s), \"pthread_attr_init has failed\");\n if (stack_size > 0) {\n check(pthread_attr_setstacksize(&s, stack_size), \"pthread_attr_setstack_size has failed\");\n }\n check(pthread_create(&c.my_thread, &s, coroutine_thread_func, &data), \"pthread_create has failed\");\n check(pthread_attr_destroy(&s), \"pthread_attr_destroy has failed\");\n#endif\n\n // Wait for the just created thread to read the data\n std::unique_lock lock(c.my_mutex);\n c.my_condvar.wait(lock, [&arg] { return arg == nullptr; });\n}\n\ninline void current_coroutine(coroutine_type& c) {\n#if _WIN32 || _WIN64\n c.my_thread = GetCurrentThread();\n#else\n c.my_thread = pthread_self();\n#endif\n}\n\ninline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) {\n thread_data* td = governor::get_thread_data();\n __TBB_ASSERT(prev_coroutine.my_is_active == true, \"The current thread should be active\");\n\n // Detach our state before notification other thread\n // (because we might be notified just after other thread notification)\n prev_coroutine.my_thread_data = nullptr;\n prev_coroutine.my_is_active = false;\n governor::clear_thread_data();\n\n {\n std::unique_lock lock(new_coroutine.my_mutex);\n __TBB_ASSERT(new_coroutine.my_is_active == false, \"The sleeping thread should not be active\");\n __TBB_ASSERT(new_coroutine.my_thread_data == nullptr, \"The sleeping thread should not be active\");\n\n new_coroutine.my_thread_data = td;\n new_coroutine.my_is_active = true;\n new_coroutine.my_condvar.notify_one();\n }\n\n std::unique_lock lock(prev_coroutine.my_mutex);\n prev_coroutine.my_condvar.wait(lock, [&prev_coroutine] { return prev_coroutine.my_is_active == true; });\n __TBB_ASSERT(governor::get_thread_data() != nullptr, nullptr);\n governor::set_thread_data(*prev_coroutine.my_thread_data);\n}\n\ninline void destroy_coroutine(coroutine_type& c) {\n {\n std::unique_lock lock(c.my_mutex);\n __TBB_ASSERT(c.my_thread_data == nullptr, \"The sleeping thread should not be active\");\n __TBB_ASSERT(c.my_is_active == false, \"The sleeping thread should not be active\");\n c.my_is_active = true;\n c.my_condvar.notify_one();\n }\n#if _WIN32 || _WIN64\n WaitForSingleObject(c.my_thread, INFINITE);\n CloseHandle(c.my_thread);\n#else\n check(pthread_join(c.my_thread, nullptr), \"pthread_join has failed\");\n#endif\n}\n#elif _WIN32 || _WIN64\ninline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) {\n __TBB_ASSERT(arg, nullptr);\n c = CreateFiber(stack_size, co_local_wait_for_all, arg);\n __TBB_ASSERT(c, nullptr);\n}\n\ninline void current_coroutine(coroutine_type& c) {\n c = IsThreadAFiber() ? GetCurrentFiber() :\n ConvertThreadToFiberEx(nullptr, FIBER_FLAG_FLOAT_SWITCH);\n __TBB_ASSERT(c, nullptr);\n}\n\ninline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) {\n if (!IsThreadAFiber()) {\n ConvertThreadToFiberEx(nullptr, FIBER_FLAG_FLOAT_SWITCH);\n }\n __TBB_ASSERT(new_coroutine, nullptr);\n prev_coroutine = GetCurrentFiber();\n __TBB_ASSERT(prev_coroutine, nullptr);\n SwitchToFiber(new_coroutine);\n}\n\ninline void destroy_coroutine(coroutine_type& c) {\n __TBB_ASSERT(c, nullptr);\n DeleteFiber(c);\n}\n#else // !(_WIN32 || _WIN64)\n\ninline void create_coroutine(coroutine_type& c, std::size_t stack_size, void* arg) {\n const std::size_t REG_PAGE_SIZE = governor::default_page_size();\n const std::size_t page_aligned_stack_size = (stack_size + (REG_PAGE_SIZE - 1)) & ~(REG_PAGE_SIZE - 1);\n const std::size_t protected_stack_size = page_aligned_stack_size + 2 * REG_PAGE_SIZE;\n\n // Allocate the stack with protection property\n std::uintptr_t stack_ptr = (std::uintptr_t)mmap(nullptr, protected_stack_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);\n __TBB_ASSERT((void*)stack_ptr != MAP_FAILED, nullptr);\n\n // Allow read write on our stack (guarded pages are still protected)\n int err = mprotect((void*)(stack_ptr + REG_PAGE_SIZE), page_aligned_stack_size, PROT_READ | PROT_WRITE);\n __TBB_ASSERT_EX(!err, nullptr);\n\n // Remember the stack state\n c.my_stack = (void*)(stack_ptr + REG_PAGE_SIZE);\n c.my_stack_size = page_aligned_stack_size;\n\n err = getcontext(&c.my_context);\n __TBB_ASSERT_EX(!err, nullptr);\n\n c.my_context.uc_link = nullptr;\n // cast to char* to disable FreeBSD clang-3.4.1 'incompatible type' error\n c.my_context.uc_stack.ss_sp = (char*)c.my_stack;\n c.my_context.uc_stack.ss_size = c.my_stack_size;\n c.my_context.uc_stack.ss_flags = 0;\n\n typedef void(*coroutine_func_t)();\n\n std::uintptr_t addr = std::uintptr_t(arg);\n unsigned lo = unsigned(addr);\n unsigned hi = unsigned(std::uint64_t(addr) >> 32);\n __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr);\n\n makecontext(&c.my_context, (coroutine_func_t)co_local_wait_for_all, 2, hi, lo);\n}\n\ninline void current_coroutine(coroutine_type& c) {\n int err = getcontext(&c.my_context);\n __TBB_ASSERT_EX(!err, nullptr);\n}\n\ninline void swap_coroutine(coroutine_type& prev_coroutine, coroutine_type& new_coroutine) {\n int err = swapcontext(&prev_coroutine.my_context, &new_coroutine.my_context);\n __TBB_ASSERT_EX(!err, nullptr);\n}\n\ninline void destroy_coroutine(coroutine_type& c) {\n const std::size_t REG_PAGE_SIZE = governor::default_page_size();\n // Free stack memory with guarded pages\n munmap((void*)((std::uintptr_t)c.my_stack - REG_PAGE_SIZE), c.my_stack_size + 2 * REG_PAGE_SIZE);\n // Clear the stack state afterwards\n c.my_stack = nullptr;\n c.my_stack_size = 0;\n}\n\n#if __APPLE__\n #if __INTEL_COMPILER\n // #pragma warning(pop) // 1478 warning\n #elif __clang__\n // #pragma clang diagnostic pop // \"-Wdeprecated-declarations\"\n #endif\n#endif\n\n#endif // _WIN32 || _WIN64\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_RESUMABLE_TASKS */\n\n#endif /* _TBB_co_context_H */\n" }, { "path": "src/tbb/src/tbb/concurrent_bounded_queue.cpp", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/concurrent_queue.h\"\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"concurrent_monitor.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstatic constexpr std::size_t monitors_number = 2;\n\nstd::uint8_t* __TBB_EXPORTED_FUNC allocate_bounded_queue_rep( std::size_t queue_rep_size )\n{\n std::size_t monitors_mem_size = sizeof(concurrent_monitor) * monitors_number;\n std::uint8_t* mem = static_cast(cache_aligned_allocate(queue_rep_size + monitors_mem_size));\n\n concurrent_monitor* monitors = reinterpret_cast(mem + queue_rep_size);\n for (std::size_t i = 0; i < monitors_number; ++i) {\n new (monitors + i) concurrent_monitor();\n }\n\n return mem;\n}\n\nvoid __TBB_EXPORTED_FUNC deallocate_bounded_queue_rep( std::uint8_t* mem, std::size_t queue_rep_size )\n{\n concurrent_monitor* monitors = reinterpret_cast(mem + queue_rep_size);\n for (std::size_t i = 0; i < monitors_number; ++i) {\n monitors[i].~concurrent_monitor();\n }\n\n cache_aligned_deallocate(mem);\n}\n\nvoid __TBB_EXPORTED_FUNC wait_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag,\n std::ptrdiff_t target, d1::delegate_base& predicate )\n{\n __TBB_ASSERT(monitor_tag < monitors_number, nullptr);\n concurrent_monitor& monitor = monitors[monitor_tag];\n\n monitor.wait([&] { return !predicate(); }, std::uintptr_t(target));\n}\n\nvoid __TBB_EXPORTED_FUNC abort_bounded_queue_monitors( concurrent_monitor* monitors ) {\n concurrent_monitor& items_avail = monitors[d2::cbq_items_avail_tag];\n concurrent_monitor& slots_avail = monitors[d2::cbq_slots_avail_tag];\n\n items_avail.abort_all();\n slots_avail.abort_all();\n}\n\nstruct predicate_leq {\n std::size_t my_ticket;\n predicate_leq( std::size_t ticket ) : my_ticket(ticket) {}\n bool operator() ( std::uintptr_t ticket ) const { return static_cast(ticket) <= my_ticket; }\n};\n\nvoid __TBB_EXPORTED_FUNC notify_bounded_queue_monitor( concurrent_monitor* monitors,\n std::size_t monitor_tag, std::size_t ticket)\n{\n __TBB_ASSERT(monitor_tag < monitors_number, nullptr);\n concurrent_monitor& monitor = monitors[monitor_tag];\n monitor.notify(predicate_leq(ticket));\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/concurrent_monitor.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_concurrent_monitor_H\n#define __TBB_concurrent_monitor_H\n\n#include \"oneapi/tbb/spin_mutex.h\"\n#include \"oneapi/tbb/detail/_exception.h\"\n#include \"oneapi/tbb/detail/_aligned_space.h\"\n#include \"concurrent_monitor_mutex.h\"\n#include \"semaphore.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Circular doubly-linked list with sentinel\n/** head.next points to the front and head.prev points to the back */\nclass circular_doubly_linked_list_with_sentinel {\npublic:\n struct base_node {\n base_node* next;\n base_node* prev;\n\n constexpr base_node(base_node* n, base_node* p) : next(n), prev(p) {}\n explicit base_node() : next((base_node*)(uintptr_t)0xcdcdcdcd), prev((base_node*)(uintptr_t)0xcdcdcdcd) {}\n };\n\n // ctor\n constexpr circular_doubly_linked_list_with_sentinel() : count(0), head(&head, &head) {}\n\n circular_doubly_linked_list_with_sentinel(const circular_doubly_linked_list_with_sentinel&) = delete;\n circular_doubly_linked_list_with_sentinel& operator=(const circular_doubly_linked_list_with_sentinel&) = delete;\n\n inline std::size_t size() const { return count.load(std::memory_order_relaxed); }\n inline bool empty() const { return size() == 0; }\n inline base_node* front() const { return head.next; }\n inline base_node* last() const { return head.prev; }\n inline const base_node* end() const { return &head; }\n\n //! add to the back of the list\n inline void add( base_node* n ) {\n count.store(count.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n n->prev = head.prev;\n n->next = &head;\n head.prev->next = n;\n head.prev = n;\n }\n\n //! remove node 'n'\n inline void remove( base_node& n ) {\n __TBB_ASSERT(count.load(std::memory_order_relaxed) > 0, \"attempt to remove an item from an empty list\");\n count.store(count.load( std::memory_order_relaxed ) - 1, std::memory_order_relaxed);\n n.prev->next = n.next;\n n.next->prev = n.prev;\n }\n\n //! move all elements to 'lst' and initialize the 'this' list\n inline void flush_to( circular_doubly_linked_list_with_sentinel& lst ) {\n const std::size_t l_count = size();\n if (l_count > 0) {\n lst.count.store(l_count, std::memory_order_relaxed);\n lst.head.next = head.next;\n lst.head.prev = head.prev;\n head.next->prev = &lst.head;\n head.prev->next = &lst.head;\n clear();\n }\n }\n\n void clear() {\n head.next = &head;\n head.prev = &head;\n count.store(0, std::memory_order_relaxed);\n }\nprivate:\n std::atomic count;\n base_node head;\n};\n\nusing base_list = circular_doubly_linked_list_with_sentinel;\nusing base_node = circular_doubly_linked_list_with_sentinel::base_node;\n\ntemplate \nclass concurrent_monitor_base;\n\ntemplate \nclass wait_node : public base_node {\npublic:\n\n#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900\n wait_node(Context ctx) : my_context(ctx), my_is_in_list(false) {}\n#else\n wait_node(Context ctx) : my_context(ctx) {}\n#endif\n\n virtual ~wait_node() = default;\n\n virtual void init() {\n __TBB_ASSERT(!my_initialized, nullptr);\n my_initialized = true;\n }\n\n virtual void wait() = 0;\n\n virtual void reset() {\n __TBB_ASSERT(my_skipped_wakeup, nullptr);\n my_skipped_wakeup = false;\n }\n\n virtual void notify() = 0;\n\nprotected:\n friend class concurrent_monitor_base;\n friend class thread_data;\n\n Context my_context{};\n#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900\n std::atomic my_is_in_list;\n#else\n std::atomic my_is_in_list{false};\n#endif\n\n bool my_initialized{false};\n bool my_skipped_wakeup{false};\n bool my_aborted{false};\n unsigned my_epoch{0};\n};\n\ntemplate \nclass sleep_node : public wait_node {\n using base_type = wait_node;\npublic:\n using base_type::base_type;\n\n ~sleep_node() override {\n if (this->my_initialized) {\n if (this->my_skipped_wakeup) semaphore().P();\n semaphore().~binary_semaphore();\n }\n }\n\n binary_semaphore& semaphore() { return *sema.begin(); }\n\n void init() override {\n if (!this->my_initialized) {\n new (sema.begin()) binary_semaphore;\n base_type::init();\n }\n }\n\n void wait() override {\n __TBB_ASSERT(this->my_initialized,\n \"Use of commit_wait() without prior prepare_wait()\");\n semaphore().P();\n __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), \"Still in the queue?\");\n if (this->my_aborted)\n throw_exception(exception_id::user_abort);\n }\n\n void reset() override {\n base_type::reset();\n semaphore().P();\n }\n\n void notify() override {\n semaphore().V();\n }\n\nprivate:\n tbb::detail::aligned_space sema;\n};\n\n//! concurrent_monitor\n/** fine-grained concurrent_monitor implementation */\ntemplate \nclass concurrent_monitor_base {\npublic:\n //! ctor\n constexpr concurrent_monitor_base() {}\n //! dtor\n ~concurrent_monitor_base() = default;\n\n concurrent_monitor_base(const concurrent_monitor_base&) = delete;\n concurrent_monitor_base& operator=(const concurrent_monitor_base&) = delete;\n\n //! prepare wait by inserting 'thr' into the wait queue\n void prepare_wait( wait_node& node) {\n // TODO: consider making even more lazy instantiation of the semaphore, that is only when it is actually needed, e.g. move it in node::wait()\n if (!node.my_initialized) {\n node.init();\n }\n // this is good place to pump previous skipped wakeup\n else if (node.my_skipped_wakeup) {\n node.reset();\n }\n\n node.my_is_in_list.store(true, std::memory_order_relaxed);\n\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n node.my_epoch = my_epoch.load(std::memory_order_relaxed);\n my_waitset.add(&node);\n }\n\n // Prepare wait guarantees Write Read memory barrier.\n // In C++ only full fence covers this type of barrier.\n atomic_fence_seq_cst();\n }\n\n //! Commit wait if event count has not changed; otherwise, cancel wait.\n /** Returns true if committed, false if canceled. */\n inline bool commit_wait( wait_node& node ) {\n const bool do_it = node.my_epoch == my_epoch.load(std::memory_order_relaxed);\n // this check is just an optimization\n if (do_it) {\n node.wait();\n } else {\n cancel_wait( node );\n }\n return do_it;\n }\n\n //! Cancel the wait. Removes the thread from the wait queue if not removed yet.\n void cancel_wait( wait_node& node ) {\n // possible skipped wakeup will be pumped in the following prepare_wait()\n node.my_skipped_wakeup = true;\n // try to remove node from waitset\n // Cancel wait guarantees acquire memory barrier.\n bool in_list = node.my_is_in_list.load(std::memory_order_acquire);\n if (in_list) {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n if (node.my_is_in_list.load(std::memory_order_relaxed)) {\n my_waitset.remove(node);\n // node is removed from waitset, so there will be no wakeup\n node.my_is_in_list.store(false, std::memory_order_relaxed);\n node.my_skipped_wakeup = false;\n }\n }\n }\n\n //! Wait for a condition to be satisfied with waiting-on my_context\n template \n bool wait(Pred&& pred, NodeType&& node) {\n prepare_wait(node);\n while (!guarded_call(std::forward(pred), node)) {\n if (commit_wait(node)) {\n return true;\n }\n\n prepare_wait(node);\n }\n\n cancel_wait(node);\n return false;\n }\n\n //! Notify one thread about the event\n void notify_one() {\n atomic_fence_seq_cst();\n notify_one_relaxed();\n }\n\n //! Notify one thread about the event. Relaxed version.\n void notify_one_relaxed() {\n if (my_waitset.empty()) {\n return;\n }\n\n base_node* n;\n const base_node* end = my_waitset.end();\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n n = my_waitset.front();\n if (n != end) {\n my_waitset.remove(*n);\n\n// GCC 12.x-13.x issues a warning here that to_wait_node(n)->my_is_in_list might have size 0, since n is\n// a base_node pointer. (This cannot happen, because only wait_node pointers are added to my_waitset.)\n#if (__TBB_GCC_VERSION >= 120100 && __TBB_GCC_VERSION < 140000 ) && !__clang__ && !__INTEL_COMPILER\n// #pragma GCC diagnostic push\n// #pragma GCC diagnostic ignored \"-Wstringop-overflow\"\n#endif\n to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed);\n#if (__TBB_GCC_VERSION >= 120100 && __TBB_GCC_VERSION < 140000 ) && !__clang__ && !__INTEL_COMPILER\n// #pragma GCC diagnostic pop\n#endif\n }\n }\n\n if (n != end) {\n to_wait_node(n)->notify();\n }\n }\n\n //! Notify all waiting threads of the event\n void notify_all() {\n atomic_fence_seq_cst();\n notify_all_relaxed();\n }\n\n // ! Notify all waiting threads of the event; Relaxed version\n void notify_all_relaxed() {\n if (my_waitset.empty()) {\n return;\n }\n\n base_list temp;\n const base_node* end;\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n // TODO: Possible optimization, don't change node state under lock, just do flush\n my_waitset.flush_to(temp);\n end = temp.end();\n for (base_node* n = temp.front(); n != end; n = n->next) {\n to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed);\n }\n }\n\n base_node* nxt;\n for (base_node* n = temp.front(); n != end; n=nxt) {\n nxt = n->next;\n to_wait_node(n)->notify();\n }\n#if TBB_USE_ASSERT\n temp.clear();\n#endif\n }\n\n //! Notify waiting threads of the event that satisfies the given predicate\n template \n void notify( const P& predicate ) {\n atomic_fence_seq_cst();\n notify_relaxed( predicate );\n }\n\n //! Notify waiting threads of the event that satisfies the given predicate;\n //! the predicate is called under the lock. Relaxed version.\n template\n void notify_relaxed( const P& predicate ) {\n if (my_waitset.empty()) {\n return;\n }\n\n base_list temp;\n base_node* nxt;\n const base_node* end = my_waitset.end();\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n my_epoch.store(my_epoch.load( std::memory_order_relaxed ) + 1, std::memory_order_relaxed);\n for (base_node* n = my_waitset.last(); n != end; n = nxt) {\n nxt = n->prev;\n auto* node = static_cast*>(n);\n if (predicate(node->my_context)) {\n my_waitset.remove(*n);\n node->my_is_in_list.store(false, std::memory_order_relaxed);\n temp.add(n);\n }\n }\n }\n\n end = temp.end();\n for (base_node* n=temp.front(); n != end; n = nxt) {\n nxt = n->next;\n to_wait_node(n)->notify();\n }\n#if TBB_USE_ASSERT\n temp.clear();\n#endif\n }\n\n //! Notify waiting threads of the event that satisfies the given predicate;\n //! the predicate is called under the lock. Relaxed version.\n template\n void notify_one_relaxed( const P& predicate ) {\n if (my_waitset.empty()) {\n return;\n }\n\n base_node* tmp = nullptr;\n base_node* next{};\n const base_node* end = my_waitset.end();\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n my_epoch.store(my_epoch.load( std::memory_order_relaxed ) + 1, std::memory_order_relaxed);\n for (base_node* n = my_waitset.last(); n != end; n = next) {\n next = n->prev;\n auto* node = static_cast*>(n);\n if (predicate(node->my_context)) {\n my_waitset.remove(*n);\n node->my_is_in_list.store(false, std::memory_order_relaxed);\n tmp = n;\n break;\n }\n }\n }\n\n if (tmp) {\n to_wait_node(tmp)->notify();\n }\n }\n\n //! Abort any sleeping threads at the time of the call\n void abort_all() {\n atomic_fence_seq_cst();\n abort_all_relaxed();\n }\n\n //! Abort any sleeping threads at the time of the call; Relaxed version\n void abort_all_relaxed() {\n if (my_waitset.empty()) {\n return;\n }\n\n base_list temp;\n const base_node* end;\n {\n concurrent_monitor_mutex::scoped_lock l(my_mutex);\n my_epoch.store(my_epoch.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n my_waitset.flush_to(temp);\n end = temp.end();\n for (base_node* n = temp.front(); n != end; n = n->next) {\n to_wait_node(n)->my_is_in_list.store(false, std::memory_order_relaxed);\n }\n }\n\n base_node* nxt;\n for (base_node* n = temp.front(); n != end; n = nxt) {\n nxt = n->next;\n to_wait_node(n)->my_aborted = true;\n to_wait_node(n)->notify();\n }\n#if TBB_USE_ASSERT\n temp.clear();\n#endif\n }\n\n void destroy() {\n this->abort_all();\n my_mutex.destroy();\n __TBB_ASSERT(this->my_waitset.empty(), \"waitset not empty?\");\n }\n\nprivate:\n template \n bool guarded_call(Pred&& predicate, NodeType& node) {\n bool res = false;\n tbb::detail::d0::try_call( [&] {\n res = std::forward(predicate)();\n }).on_exception( [&] {\n cancel_wait(node);\n });\n\n return res;\n }\n\n concurrent_monitor_mutex my_mutex{};\n base_list my_waitset{};\n std::atomic my_epoch{};\n\n wait_node* to_wait_node( base_node* node ) { return static_cast*>(node); }\n};\n\nclass concurrent_monitor : public concurrent_monitor_base {\n using base_type = concurrent_monitor_base;\npublic:\n using base_type::base_type;\n\n ~concurrent_monitor() {\n destroy();\n }\n\n /** per-thread descriptor for concurrent_monitor */\n using thread_context = sleep_node;\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_concurrent_monitor_H */\n" }, { "path": "src/tbb/src/tbb/concurrent_monitor_mutex.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_monitor_mutex_H\n#define __TBB_monitor_mutex_H\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_aligned_space.h\"\n#include \"semaphore.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass concurrent_monitor_mutex {\npublic:\n using scoped_lock = std::lock_guard;\n\n constexpr concurrent_monitor_mutex() {}\n\n ~concurrent_monitor_mutex() = default;\n\n void destroy() {\n#if !__TBB_USE_FUTEX\n if (my_init_flag.load(std::memory_order_relaxed)) {\n get_semaphore().~semaphore();\n }\n#endif\n }\n\n void lock() {\n auto wakeup_condition = [&] {\n return my_flag.load(std::memory_order_relaxed) == 0;\n };\n\n while (my_flag.exchange(1)) {\n if (!timed_spin_wait_until(wakeup_condition)) {\n ++my_waiters;\n while (!wakeup_condition()) {\n wait();\n }\n --my_waiters;\n }\n }\n }\n\n void unlock() {\n my_flag.exchange(0); // full fence, so the next load is relaxed\n if (my_waiters.load(std::memory_order_relaxed)) {\n wakeup();\n }\n }\n\nprivate:\n void wait() {\n#if __TBB_USE_FUTEX\n futex_wait(&my_flag, 1);\n#else\n get_semaphore().P();\n#endif\n }\n\n void wakeup() {\n#if __TBB_USE_FUTEX\n futex_wakeup_one(&my_flag);\n#else\n get_semaphore().V();\n#endif\n }\n\n // The flag should be int for the futex operations\n std::atomic my_flag{0};\n std::atomic my_waiters{0};\n\n#if !__TBB_USE_FUTEX\n semaphore& get_semaphore() {\n if (!my_init_flag.load(std::memory_order_acquire)) {\n std::lock_guard lock(my_init_mutex);\n if (!my_init_flag.load(std::memory_order_relaxed)) {\n new (my_semaphore.begin()) semaphore();\n my_init_flag.store(true, std::memory_order_release);\n }\n }\n\n return *my_semaphore.begin();\n }\n\n static std::mutex my_init_mutex;\n std::atomic my_init_flag{false};\n aligned_space my_semaphore{};\n#endif\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_monitor_mutex_H\n" }, { "path": "src/tbb/src/tbb/def/lin32-tbb.def", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n\n/* Needed for backwards compatibility */\n_ZNSt13runtime_errorD1Ev;\n_ZTISt13runtime_error;\n_ZTSSt13runtime_error;\n_ZNSt16invalid_argumentD1Ev;\n_ZTISt16invalid_argument;\n_ZTSSt16invalid_argument;\n_ZNSt11range_errorD1Ev;\n_ZTISt11range_error;\n_ZTSSt11range_error;\n_ZNSt12length_errorD1Ev;\n_ZTISt12length_error;\n_ZTSSt12length_error;\n_ZNSt12out_of_rangeD1Ev;\n_ZTISt12out_of_range;\n_ZTSSt12out_of_range;\n\n/* Needed by rstan */\n_ZN3tbb8internal26task_scheduler_observer_v37observeEb;\n\n/* Assertions (assert.cpp) */\n_ZN3tbb6detail2r117assertion_failureEPKciS3_S3_;\n\n/* ITT (profiling.cpp) */\n_ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE;\n_ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy;\n_ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r115call_itt_notifyEiPv;\n_ZN3tbb6detail2r115create_itt_syncEPvPKcS4_;\n_ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y;\n_ZN3tbb6detail2r117itt_set_sync_nameEPvPKc;\n_ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc;\n_ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_;\n\n/* Allocators (allocator.cpp) */\n_ZN3tbb6detail2r115allocate_memoryEj;\n_ZN3tbb6detail2r117deallocate_memoryEPv;\n_ZN3tbb6detail2r122cache_aligned_allocateEj;\n_ZN3tbb6detail2r124cache_aligned_deallocateEPv;\n_ZN3tbb6detail2r115cache_line_sizeEv;\n_ZN3tbb6detail2r117is_tbbmalloc_usedEv;\n\n/* Small object pool (small_object_pool.cpp) */\n_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEj;\n_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEjRKNS2_14execution_dataE;\n_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvj;\n_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvjRKNS2_14execution_dataE;\n\n/* Error handling (exception.cpp) */\n_ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE;\n_ZTIN3tbb6detail2r114bad_last_allocE;\n_ZTVN3tbb6detail2r114bad_last_allocE;\n_ZTIN3tbb6detail2r112missing_waitE;\n_ZTVN3tbb6detail2r112missing_waitE;\n_ZTIN3tbb6detail2r110user_abortE;\n_ZTVN3tbb6detail2r110user_abortE;\n_ZTIN3tbb6detail2r111unsafe_waitE;\n_ZTVN3tbb6detail2r111unsafe_waitE;\n\n/* RTM Mutex (rtm_mutex.cpp) */\n_ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE;\n_ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE;\n\n/* RTM RW Mutex (rtm_rw_mutex.cpp) */\n_ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE;\n_ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE;\n_ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE;\n\n/* Tasks and partitioners (task.cpp) */\n_ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_;\n_ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE;\n_ZN3tbb6detail2r121current_suspend_pointEv;\n_ZN3tbb6detail2r114notify_waitersEj;\n_ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE;\n\n/* Task dispatcher (task_dispatcher.cpp) */\n_ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE;\n_ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE;\n_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE;\n_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt;\n_ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_;\n_ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEj;\n_ZN3tbb6detail2r115current_contextEv;\n\n/* Task group context (task_group_context.cpp) */\n_ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE;\n\n/* Task arena (arena.cpp) */\n_ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE;\n_ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEi;\n_ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE;\n_ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE;\n_ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE;\n\n/* System topology parsing and threads pinning (governor.cpp) */\n_ZN3tbb6detail2r115numa_node_countEv;\n_ZN3tbb6detail2r117fill_numa_indicesEPi;\n_ZN3tbb6detail2r115core_type_countEi;\n_ZN3tbb6detail2r122fill_core_type_indicesEPii;\n_ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEi;\n_ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEi;\n_ZN3tbb6detail2r124numa_default_concurrencyEi;\n\n/* Observer (observer_proxy.cpp) */\n_ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb;\n\n/* Queuing RW Mutex (queuing_rw_mutex.cpp) */\n_ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE;\n_ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE;\n\n/* Global control (global_control.cpp) */\n_ZN3tbb6detail2r16createERNS0_2d114global_controlE;\n_ZN3tbb6detail2r17destroyERNS0_2d114global_controlE;\n_ZN3tbb6detail2r127global_control_active_valueEi;\n_ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEi;\n_ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE;\n\n/* Parallel pipeline (parallel_pipeline.cpp) */\n_ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEjRKNS2_11filter_nodeE;\n_ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE;\n\n/* Concurrent bounded queue (concurrent_bounded_queue.cpp) */\n_ZN3tbb6detail2r126allocate_bounded_queue_repEj;\n_ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEjiRNS0_2d113delegate_baseE;\n_ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE;\n_ZN3tbb6detail2r128deallocate_bounded_queue_repEPhj;\n_ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEjj;\n\n/* Concurrent monitor (address_waiter.cpp) */\n_ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEj;\n_ZN3tbb6detail2r117notify_by_addressEPvj;\n_ZN3tbb6detail2r121notify_by_address_oneEPv;\n_ZN3tbb6detail2r121notify_by_address_allEPv;\n\n/* Versioning (version.cpp) */\nTBB_runtime_interface_version;\nTBB_runtime_version;\n\nlocal:\n/* TODO: fill more precisely */\n*;\n};\n" }, { "path": "src/tbb/src/tbb/def/lin64-tbb.def", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n\n/* Needed for backwards compatibility */\n_ZNSt13runtime_errorD1Ev;\n_ZTISt13runtime_error;\n_ZTSSt13runtime_error;\n_ZNSt16invalid_argumentD1Ev;\n_ZTISt16invalid_argument;\n_ZTSSt16invalid_argument;\n_ZNSt11range_errorD1Ev;\n_ZTISt11range_error;\n_ZTSSt11range_error;\n_ZNSt12length_errorD1Ev;\n_ZTISt12length_error;\n_ZTSSt12length_error;\n_ZNSt12out_of_rangeD1Ev;\n_ZTISt12out_of_range;\n_ZTSSt12out_of_range;\n\n/* Needed by rstan */\n_ZN3tbb8internal26task_scheduler_observer_v37observeEb;\n\n/* Assertions (assert.cpp) */\n_ZN3tbb6detail2r117assertion_failureEPKciS3_S3_;\n\n/* ITT (profiling.cpp) */\n_ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE;\n_ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy;\n_ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r115call_itt_notifyEiPv;\n_ZN3tbb6detail2r115create_itt_syncEPvPKcS4_;\n_ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y;\n_ZN3tbb6detail2r117itt_set_sync_nameEPvPKc;\n_ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE;\n_ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc;\n_ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_;\n\n/* Allocators (allocator.cpp) */\n_ZN3tbb6detail2r115allocate_memoryEm;\n_ZN3tbb6detail2r117deallocate_memoryEPv;\n_ZN3tbb6detail2r122cache_aligned_allocateEm;\n_ZN3tbb6detail2r124cache_aligned_deallocateEPv;\n_ZN3tbb6detail2r115cache_line_sizeEv;\n_ZN3tbb6detail2r117is_tbbmalloc_usedEv;\n\n/* Small object pool (small_object_pool.cpp) */\n_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEm;\n_ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEmRKNS2_14execution_dataE;\n_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvm;\n_ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvmRKNS2_14execution_dataE;\n\n/* Error handling (exception.cpp) */\n_ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE;\n_ZTIN3tbb6detail2r114bad_last_allocE;\n_ZTVN3tbb6detail2r114bad_last_allocE;\n_ZTIN3tbb6detail2r112missing_waitE;\n_ZTVN3tbb6detail2r112missing_waitE;\n_ZTIN3tbb6detail2r110user_abortE;\n_ZTVN3tbb6detail2r110user_abortE;\n_ZTIN3tbb6detail2r111unsafe_waitE;\n_ZTVN3tbb6detail2r111unsafe_waitE;\n\n/* RTM Mutex (rtm_mutex.cpp) */\n_ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE;\n_ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE;\n\n/* RTM RW Mutex (rtm_rw_mutex.cpp) */\n_ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE;\n_ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE;\n_ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE;\n\n/* Tasks and partitioners (task.cpp) */\n_ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_;\n_ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE;\n_ZN3tbb6detail2r121current_suspend_pointEv;\n_ZN3tbb6detail2r114notify_waitersEm;\n_ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE;\n\n/* Task dispatcher (task_dispatcher.cpp) */\n_ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE;\n_ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE;\n_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE;\n_ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt;\n_ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_;\n_ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEm;\n_ZN3tbb6detail2r115current_contextEv;\n\n/* Task group context (task_group_context.cpp) */\n_ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE;\n_ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE;\n\n/* Task arena (arena.cpp) */\n_ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE;\n_ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEl;\n_ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE;\n_ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE;\n_ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE;\n_ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE;\n\n/* System topology parsing and threads pinning (governor.cpp) */\n_ZN3tbb6detail2r115numa_node_countEv;\n_ZN3tbb6detail2r117fill_numa_indicesEPi;\n_ZN3tbb6detail2r115core_type_countEl;\n_ZN3tbb6detail2r122fill_core_type_indicesEPil;\n_ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEl;\n_ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEl;\n_ZN3tbb6detail2r124numa_default_concurrencyEi;\n\n/* Observer (observer_proxy.cpp) */\n_ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb;\n\n/* Queuing RW Mutex (queuing_rw_mutex.cpp) */\n_ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb;\n_ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE;\n_ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE;\n_ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE;\n\n/* Global control (global_control.cpp) */\n_ZN3tbb6detail2r16createERNS0_2d114global_controlE;\n_ZN3tbb6detail2r17destroyERNS0_2d114global_controlE;\n_ZN3tbb6detail2r127global_control_active_valueEi;\n_ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEl;\n_ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE;\n\n/* Parallel pipeline (parallel_pipeline.cpp) */\n_ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEmRKNS2_11filter_nodeE;\n_ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE;\n\n/* Concurrent bounded queue (concurrent_bounded_queue.cpp) */\n_ZN3tbb6detail2r126allocate_bounded_queue_repEm;\n_ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEmlRNS0_2d113delegate_baseE;\n_ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE;\n_ZN3tbb6detail2r128deallocate_bounded_queue_repEPhm;\n_ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEmm;\n\n/* Concurrent monitor (address_waiter.cpp) */\n_ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEm;\n_ZN3tbb6detail2r117notify_by_addressEPvm;\n_ZN3tbb6detail2r121notify_by_address_oneEPv;\n_ZN3tbb6detail2r121notify_by_address_allEPv;\n\n/* Versioning (version.cpp) */\nTBB_runtime_interface_version;\nTBB_runtime_version;\n\nlocal:\n/* TODO: fill more precisely */\n*;\n};\n" }, { "path": "src/tbb/src/tbb/def/mac64-tbb.def", "content": "# Copyright (c) 2005-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# TODO: check the legacy comment below, currently use extra leading underscore everywhere.\n# Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not\n# (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD\n# be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when\n# necessary, depending on the intended usage.\n\n# Needed for backwards compatibility\n__ZNSt13runtime_errorD1Ev\n__ZTISt13runtime_error\n__ZTSSt13runtime_error\n__ZNSt16invalid_argumentD1Ev\n__ZTISt16invalid_argument\n__ZTSSt16invalid_argument\n__ZNSt11range_errorD1Ev\n__ZTISt11range_error\n__ZTSSt11range_error\n__ZNSt12length_errorD1Ev\n__ZTISt12length_error\n__ZTSSt12length_error\n__ZNSt12out_of_rangeD1Ev\n__ZTISt12out_of_range\n__ZTSSt12out_of_range\n\n# Needed by rstan\n__ZN3tbb8internal26task_scheduler_observer_v37observeEb\n\n# Assertions (assert.cpp)\n__ZN3tbb6detail2r117assertion_failureEPKciS3_S3_\n\n# ITT (profiling.cpp)\n__ZN3tbb6detail2r112itt_task_endENS0_2d115itt_domain_enumE\n__ZN3tbb6detail2r114itt_region_endENS0_2d115itt_domain_enumEPvy\n__ZN3tbb6detail2r114itt_task_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE\n__ZN3tbb6detail2r115call_itt_notifyEiPv\n__ZN3tbb6detail2r115create_itt_syncEPvPKcS4_\n__ZN3tbb6detail2r116itt_region_beginENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE\n__ZN3tbb6detail2r116itt_relation_addENS0_2d115itt_domain_enumEPvyNS0_2d012itt_relationES4_y\n__ZN3tbb6detail2r117itt_set_sync_nameEPvPKc\n__ZN3tbb6detail2r119itt_make_task_groupENS0_2d115itt_domain_enumEPvyS4_yNS0_2d021string_resource_indexE\n__ZN3tbb6detail2r120itt_metadata_str_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexEPKc\n__ZN3tbb6detail2r120itt_metadata_ptr_addENS0_2d115itt_domain_enumEPvyNS0_2d021string_resource_indexES4_\n\n# Allocators (allocator.cpp)\n__ZN3tbb6detail2r115allocate_memoryEm\n__ZN3tbb6detail2r117deallocate_memoryEPv\n__ZN3tbb6detail2r122cache_aligned_allocateEm\n__ZN3tbb6detail2r124cache_aligned_deallocateEPv\n__ZN3tbb6detail2r115cache_line_sizeEv\n__ZN3tbb6detail2r117is_tbbmalloc_usedEv\n\n# Small object pool (small_object_pool.cpp)\n__ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEm\n__ZN3tbb6detail2r18allocateERPNS0_2d117small_object_poolEmRKNS2_14execution_dataE\n__ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvm\n__ZN3tbb6detail2r110deallocateERNS0_2d117small_object_poolEPvmRKNS2_14execution_dataE\n\n# Error handling (exception.cpp)\n__ZN3tbb6detail2r115throw_exceptionENS0_2d012exception_idE\n__ZTIN3tbb6detail2r114bad_last_allocE\n__ZTVN3tbb6detail2r114bad_last_allocE\n__ZTIN3tbb6detail2r112missing_waitE\n__ZTVN3tbb6detail2r112missing_waitE\n__ZTIN3tbb6detail2r110user_abortE\n__ZTVN3tbb6detail2r110user_abortE\n__ZTIN3tbb6detail2r111unsafe_waitE\n__ZTVN3tbb6detail2r111unsafe_waitE\n\n# RTM Mutex (rtm_mutex.cpp)\n__ZN3tbb6detail2r17acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockEb\n__ZN3tbb6detail2r17releaseERNS0_2d19rtm_mutex11scoped_lockE\n__ZN3tbb6detail2r111try_acquireERNS0_2d19rtm_mutexERNS3_11scoped_lockE\n\n# RTM RW Mutex (rtm_rw_mutex.cpp)\n__ZN3tbb6detail2r114acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb\n__ZN3tbb6detail2r114acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockEb\n__ZN3tbb6detail2r118try_acquire_readerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE\n__ZN3tbb6detail2r118try_acquire_writerERNS0_2d112rtm_rw_mutexERNS3_11scoped_lockE\n__ZN3tbb6detail2r17releaseERNS0_2d112rtm_rw_mutex11scoped_lockE\n__ZN3tbb6detail2r17upgradeERNS0_2d112rtm_rw_mutex11scoped_lockE\n__ZN3tbb6detail2r19downgradeERNS0_2d112rtm_rw_mutex11scoped_lockE\n\n# Tasks and partitioners (task.cpp)\n__ZN3tbb6detail2r17suspendEPFvPvPNS1_18suspend_point_typeEES2_\n__ZN3tbb6detail2r16resumeEPNS1_18suspend_point_typeE\n__ZN3tbb6detail2r121current_suspend_pointEv\n__ZN3tbb6detail2r114notify_waitersEm\n__ZN3tbb6detail2r127get_thread_reference_vertexEPNS0_2d126wait_tree_vertex_interfaceE\n\n# Task dispatcher (task_dispatcher.cpp)\n__ZN3tbb6detail2r114execution_slotEPKNS0_2d114execution_dataE\n__ZN3tbb6detail2r14waitERNS0_2d112wait_contextERNS2_18task_group_contextE\n__ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextE\n__ZN3tbb6detail2r15spawnERNS0_2d14taskERNS2_18task_group_contextEt\n__ZN3tbb6detail2r116execute_and_waitERNS0_2d14taskERNS2_18task_group_contextERNS2_12wait_contextES6_\n__ZN3tbb6detail2r16submitERNS0_2d14taskERNS2_18task_group_contextEPNS1_5arenaEm\n__ZN3tbb6detail2r115current_contextEv\n\n# Task group context (task_group_context.cpp)\n__ZN3tbb6detail2r110initializeERNS0_2d118task_group_contextE\n__ZN3tbb6detail2r122cancel_group_executionERNS0_2d118task_group_contextE\n__ZN3tbb6detail2r128is_group_execution_cancelledERNS0_2d118task_group_contextE\n__ZN3tbb6detail2r15resetERNS0_2d118task_group_contextE\n__ZN3tbb6detail2r17destroyERNS0_2d118task_group_contextE\n__ZN3tbb6detail2r119capture_fp_settingsERNS0_2d118task_group_contextE\n\n# Task arena (arena.cpp)\n__ZN3tbb6detail2r115max_concurrencyEPKNS0_2d115task_arena_baseE\n__ZN3tbb6detail2r110initializeERNS0_2d115task_arena_baseE\n__ZN3tbb6detail2r16attachERNS0_2d115task_arena_baseE\n__ZN3tbb6detail2r17executeERNS0_2d115task_arena_baseERNS2_13delegate_baseE\n__ZN3tbb6detail2r19terminateERNS0_2d115task_arena_baseE\n__ZN3tbb6detail2r120isolate_within_arenaERNS0_2d113delegate_baseEl\n__ZN3tbb6detail2r17enqueueERNS0_2d14taskEPNS2_15task_arena_baseE\n__ZN3tbb6detail2r17enqueueERNS0_2d14taskERNS2_18task_group_contextEPNS2_15task_arena_baseE\n__ZN3tbb6detail2r14waitERNS0_2d115task_arena_baseE\n__ZN3tbb6detail2r114execution_slotERKNS0_2d115task_arena_baseE\n\n# System topology parsing and threads pinning (governor.cpp)\n__ZN3tbb6detail2r115numa_node_countEv\n__ZN3tbb6detail2r117fill_numa_indicesEPi\n__ZN3tbb6detail2r115core_type_countEl\n__ZN3tbb6detail2r122fill_core_type_indicesEPil\n__ZN3tbb6detail2r131constraints_default_concurrencyERKNS0_2d111constraintsEl\n__ZN3tbb6detail2r128constraints_threads_per_coreERKNS0_2d111constraintsEl\n__ZN3tbb6detail2r124numa_default_concurrencyEi\n\n# Observer (observer_proxy.cpp)\n__ZN3tbb6detail2r17observeERNS0_2d123task_scheduler_observerEb\n\n# Queuing RW Mutex (queuing_rw_mutex.cpp)\n__ZN3tbb6detail2r111try_acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb\n__ZN3tbb6detail2r117upgrade_to_writerERNS0_2d116queuing_rw_mutex11scoped_lockE\n__ZN3tbb6detail2r119downgrade_to_readerERNS0_2d116queuing_rw_mutex11scoped_lockE\n__ZN3tbb6detail2r17acquireERNS0_2d116queuing_rw_mutexERNS3_11scoped_lockEb\n__ZN3tbb6detail2r17releaseERNS0_2d116queuing_rw_mutex11scoped_lockE\n__ZN3tbb6detail2r19constructERNS0_2d116queuing_rw_mutexE\n__ZN3tbb6detail2r19is_writerERKNS0_2d116queuing_rw_mutex11scoped_lockE\n\n# Global control (global_control.cpp)\n__ZN3tbb6detail2r16createERNS0_2d114global_controlE\n__ZN3tbb6detail2r17destroyERNS0_2d114global_controlE\n__ZN3tbb6detail2r127global_control_active_valueEi\n__ZN3tbb6detail2r18finalizeERNS0_2d121task_scheduler_handleEl\n__ZN3tbb6detail2r13getERNS0_2d121task_scheduler_handleE\n\n# Parallel pipeline (parallel_pipeline.cpp)\n__ZN3tbb6detail2r117parallel_pipelineERNS0_2d118task_group_contextEmRKNS2_11filter_nodeE\n__ZN3tbb6detail2r116set_end_of_inputERNS0_2d111base_filterE\n\n# Concurrent bounded queue (concurrent_bounded_queue.cpp)\n__ZN3tbb6detail2r126allocate_bounded_queue_repEm\n__ZN3tbb6detail2r126wait_bounded_queue_monitorEPNS1_18concurrent_monitorEmlRNS0_2d113delegate_baseE\n__ZN3tbb6detail2r128abort_bounded_queue_monitorsEPNS1_18concurrent_monitorE\n__ZN3tbb6detail2r128deallocate_bounded_queue_repEPhm\n__ZN3tbb6detail2r128notify_bounded_queue_monitorEPNS1_18concurrent_monitorEmm\n\n# Concurrent monitor (address_waiter.cpp)\n__ZN3tbb6detail2r115wait_on_addressEPvRNS0_2d113delegate_baseEm\n__ZN3tbb6detail2r117notify_by_addressEPvm\n__ZN3tbb6detail2r121notify_by_address_oneEPv\n__ZN3tbb6detail2r121notify_by_address_allEPv\n\n# Versioning (version.cpp)\n_TBB_runtime_interface_version\n_TBB_runtime_version\n" }, { "path": "src/tbb/src/tbb/def/win32-tbb.def", "content": "; Copyright (c) 2005-2024 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\n; This file is organized with a section for each .cpp file.\n\nEXPORTS\n\n; Assertions (assert.cpp)\n?assertion_failure@r1@detail@tbb@@YAXPBDH00@Z\n\n; ITT (tbb_profiling.cpp)\n?call_itt_notify@r1@detail@tbb@@YAXHPAX@Z\n?create_itt_sync@r1@detail@tbb@@YAXPAXPB_W1@Z\n?itt_make_task_group@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z\n?itt_task_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z\n?itt_task_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@@Z\n?itt_metadata_str_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4string_resource_index@d0@23@PBD@Z\n?itt_relation_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4itt_relation@d0@23@12@Z\n?itt_region_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K12W4string_resource_index@d0@23@@Z\n?itt_region_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_K@Z\n?itt_set_sync_name@r1@detail@tbb@@YAXPAXPB_W@Z\n?itt_metadata_ptr_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PAX_KW4string_resource_index@d0@23@1@Z\n\n; Allocators (tbb_allocator.cpp)\n?cache_aligned_allocate@r1@detail@tbb@@YAPAXI@Z\n?cache_aligned_deallocate@r1@detail@tbb@@YAXPAX@Z\n?cache_line_size@r1@detail@tbb@@YAIXZ\n?allocate_memory@r1@detail@tbb@@YAPAXI@Z\n?deallocate_memory@r1@detail@tbb@@YAXPAX@Z\n?is_tbbmalloc_used@r1@detail@tbb@@YA_NXZ\n\n; Small object pool (small_object_pool.cpp)\n?allocate@r1@detail@tbb@@YAPAXAAPAVsmall_object_pool@d1@23@IABUexecution_data@523@@Z\n?allocate@r1@detail@tbb@@YAPAXAAPAVsmall_object_pool@d1@23@I@Z\n?deallocate@r1@detail@tbb@@YAXAAVsmall_object_pool@d1@23@PAXIABUexecution_data@523@@Z\n?deallocate@r1@detail@tbb@@YAXAAVsmall_object_pool@d1@23@PAXI@Z\n\n; Error handling (exception.cpp)\n?throw_exception@r1@detail@tbb@@YAXW4exception_id@d0@23@@Z\n?what@bad_last_alloc@r1@detail@tbb@@UBEPBDXZ\n?what@user_abort@r1@detail@tbb@@UBEPBDXZ\n?what@missing_wait@r1@detail@tbb@@UBEPBDXZ\n\n; RTM Mutex (rtm_mutex.cpp)\n?acquire@r1@detail@tbb@@YAXAAVrtm_mutex@d1@23@AAVscoped_lock@4523@_N@Z\n?release@r1@detail@tbb@@YAXAAVscoped_lock@rtm_mutex@d1@23@@Z\n?try_acquire@r1@detail@tbb@@YA_NAAVrtm_mutex@d1@23@AAVscoped_lock@4523@@Z\n\n; RTM RW Mutex (rtm_rw_mutex.cpp)\n?acquire_reader@r1@detail@tbb@@YAXAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z\n?acquire_writer@r1@detail@tbb@@YAXAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z\n?downgrade@r1@detail@tbb@@YA_NAAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n?release@r1@detail@tbb@@YAXAAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n?try_acquire_reader@r1@detail@tbb@@YA_NAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@@Z\n?try_acquire_writer@r1@detail@tbb@@YA_NAAVrtm_rw_mutex@d1@23@AAVscoped_lock@4523@@Z\n?upgrade@r1@detail@tbb@@YA_NAAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n\n; Tasks and partitioners (task.cpp)\n?current_suspend_point@r1@detail@tbb@@YAPAUsuspend_point_type@123@XZ\n?resume@r1@detail@tbb@@YAXPAUsuspend_point_type@123@@Z\n?suspend@r1@detail@tbb@@YAXP6AXPAXPAUsuspend_point_type@123@@Z0@Z\n?notify_waiters@r1@detail@tbb@@YAXI@Z\n?get_thread_reference_vertex@r1@detail@tbb@@YAPAVwait_tree_vertex_interface@d1@23@PAV4523@@Z\n\n; Task dispatcher (task_dispatcher.cpp)\n?spawn@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@G@Z\n?spawn@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@@Z\n?execute_and_wait@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@AAVwait_context@523@1@Z\n?execution_slot@r1@detail@tbb@@YAGPBUexecution_data@d1@23@@Z\n?wait@r1@detail@tbb@@YAXAAVwait_context@d1@23@AAVtask_group_context@523@@Z\n?submit@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@PAVarena@123@I@Z\n?current_context@r1@detail@tbb@@YAPAVtask_group_context@d1@23@XZ\n\n; Task group context (task_group_context.cpp)\n?cancel_group_execution@r1@detail@tbb@@YA_NAAVtask_group_context@d1@23@@Z\n?capture_fp_settings@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z\n?destroy@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z\n?initialize@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z\n?is_group_execution_cancelled@r1@detail@tbb@@YA_NAAVtask_group_context@d1@23@@Z\n?reset@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@@Z\n\n; Task arena (arena.cpp)\n?attach@r1@detail@tbb@@YA_NAAVtask_arena_base@d1@23@@Z\n?enqueue@r1@detail@tbb@@YAXAAVtask@d1@23@PAVtask_arena_base@523@@Z\n?execute@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@AAVdelegate_base@523@@Z\n?initialize@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z\n?isolate_within_arena@r1@detail@tbb@@YAXAAVdelegate_base@d1@23@H@Z\n?max_concurrency@r1@detail@tbb@@YAHPBVtask_arena_base@d1@23@@Z\n?terminate@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z\n?wait@r1@detail@tbb@@YAXAAVtask_arena_base@d1@23@@Z\n?enqueue@r1@detail@tbb@@YAXAAVtask@d1@23@AAVtask_group_context@523@PAVtask_arena_base@523@@Z\n?execution_slot@r1@detail@tbb@@YAGABVtask_arena_base@d1@23@@Z\n\n; System topology parsing and threads pinning (governor.cpp)\n?numa_node_count@r1@detail@tbb@@YAIXZ\n?fill_numa_indices@r1@detail@tbb@@YAXPAH@Z\n?core_type_count@r1@detail@tbb@@YAIH@Z\n?fill_core_type_indices@r1@detail@tbb@@YAXPAHH@Z\n?numa_default_concurrency@r1@detail@tbb@@YAHH@Z\n?constraints_default_concurrency@r1@detail@tbb@@YAHABUconstraints@d1@23@H@Z\n?constraints_threads_per_core@r1@detail@tbb@@YAHABUconstraints@d1@23@H@Z\n\n; Observer (observer_proxy.cpp)\n?observe@r1@detail@tbb@@YAXAAVtask_scheduler_observer@d1@23@_N@Z\n\n; Queuing RW Mutex (queuing_rw_mutex.cpp)\n?acquire@r1@detail@tbb@@YAXAAVqueuing_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z\n?construct@r1@detail@tbb@@YAXAAVqueuing_rw_mutex@d1@23@@Z\n?downgrade_to_reader@r1@detail@tbb@@YA_NAAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?release@r1@detail@tbb@@YAXAAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?try_acquire@r1@detail@tbb@@YA_NAAVqueuing_rw_mutex@d1@23@AAVscoped_lock@4523@_N@Z\n?upgrade_to_writer@r1@detail@tbb@@YA_NAAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?is_writer@r1@detail@tbb@@YA_NABVscoped_lock@queuing_rw_mutex@d1@23@@Z\n\n; Global control (global_control.cpp)\n?create@r1@detail@tbb@@YAXAAVglobal_control@d1@23@@Z\n?destroy@r1@detail@tbb@@YAXAAVglobal_control@d1@23@@Z\n?global_control_active_value@r1@detail@tbb@@YAIH@Z\n?get@r1@detail@tbb@@YAXAAVtask_scheduler_handle@d1@23@@Z\n?finalize@r1@detail@tbb@@YA_NAAVtask_scheduler_handle@d1@23@H@Z\n\n; Parallel pipeline (parallel_pipeline.cpp)\n?parallel_pipeline@r1@detail@tbb@@YAXAAVtask_group_context@d1@23@IABVfilter_node@523@@Z\n?set_end_of_input@r1@detail@tbb@@YAXAAVbase_filter@d1@23@@Z\n\n; Concurrent bounded queue (concurrent_bounded_queue.cpp)\n?abort_bounded_queue_monitors@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@@Z\n?allocate_bounded_queue_rep@r1@detail@tbb@@YAPAEI@Z\n?deallocate_bounded_queue_rep@r1@detail@tbb@@YAXPAEI@Z\n?notify_bounded_queue_monitor@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@II@Z\n?wait_bounded_queue_monitor@r1@detail@tbb@@YAXPAVconcurrent_monitor@123@IHAAVdelegate_base@d1@23@@Z\n\n; Concurrent monitor (address_waiter.cpp)\n?wait_on_address@r1@detail@tbb@@YAXPAXAAVdelegate_base@d1@23@I@Z\n?notify_by_address@r1@detail@tbb@@YAXPAXI@Z\n?notify_by_address_one@r1@detail@tbb@@YAXPAX@Z\n?notify_by_address_all@r1@detail@tbb@@YAXPAX@Z\n\n;; Versioning (version.cpp)\nTBB_runtime_interface_version\nTBB_runtime_version\n" }, { "path": "src/tbb/src/tbb/def/win64-tbb.def", "content": "; Copyright (c) 2005-2024 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\n; This file is organized with a section for each .cpp file.\n\nEXPORTS\n\n; Assertions (assert.cpp)\n?assertion_failure@r1@detail@tbb@@YAXPEBDH00@Z\n\n; ITT (tbb_profiling.cpp)\n?call_itt_notify@r1@detail@tbb@@YAXHPEAX@Z\n?create_itt_sync@r1@detail@tbb@@YAXPEAXPEB_W1@Z\n?itt_make_task_group@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z\n?itt_task_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z\n?itt_task_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@@Z\n?itt_set_sync_name@r1@detail@tbb@@YAXPEAXPEB_W@Z\n?itt_metadata_str_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@PEBD@Z\n?itt_relation_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4itt_relation@d0@23@12@Z\n?itt_region_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z\n?itt_region_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K@Z\n?itt_metadata_ptr_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@1@Z\n\n; Allocators (tbb_allocator.cpp)\n?cache_aligned_allocate@r1@detail@tbb@@YAPEAX_K@Z\n?cache_aligned_deallocate@r1@detail@tbb@@YAXPEAX@Z\n?cache_line_size@r1@detail@tbb@@YA_KXZ\n?allocate_memory@r1@detail@tbb@@YAPEAX_K@Z\n?deallocate_memory@r1@detail@tbb@@YAXPEAX@Z\n?is_tbbmalloc_used@r1@detail@tbb@@YA_NXZ\n\n; Small object pool (small_object_pool.cpp)\n?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_KAEBUexecution_data@523@@Z\n?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_K@Z\n?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_KAEBUexecution_data@523@@Z\n?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_K@Z\n\n; Error handling (exception.cpp)\n?throw_exception@r1@detail@tbb@@YAXW4exception_id@d0@23@@Z\n?what@bad_last_alloc@r1@detail@tbb@@UEBAPEBDXZ\n?what@user_abort@r1@detail@tbb@@UEBAPEBDXZ\n?what@missing_wait@r1@detail@tbb@@UEBAPEBDXZ\n\n; RTM Mutex (rtm_mutex.cpp)\n?try_acquire@r1@detail@tbb@@YA_NAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@@Z\n?acquire@r1@detail@tbb@@YAXAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@_N@Z\n?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_mutex@d1@23@@Z\n\n; RTM RW Mutex (rtm_rw_mutex.cpp)\n?acquire_writer@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z\n?acquire_reader@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z\n?upgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n?downgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n?try_acquire_writer@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z\n?try_acquire_reader@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z\n?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z\n\n; Tasks and partitioners (task.cpp)\n?suspend@r1@detail@tbb@@YAXP6AXPEAXPEAUsuspend_point_type@123@@Z0@Z\n?resume@r1@detail@tbb@@YAXPEAUsuspend_point_type@123@@Z\n?current_suspend_point@r1@detail@tbb@@YAPEAUsuspend_point_type@123@XZ\n?notify_waiters@r1@detail@tbb@@YAX_K@Z\n?get_thread_reference_vertex@r1@detail@tbb@@YAPEAVwait_tree_vertex_interface@d1@23@PEAV4523@@Z\n\n; Task dispatcher (task_dispatcher.cpp)\n?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@@Z\n?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@G@Z\n?execute_and_wait@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@AEAVwait_context@523@1@Z\n?execution_slot@r1@detail@tbb@@YAGPEBUexecution_data@d1@23@@Z\n?wait@r1@detail@tbb@@YAXAEAVwait_context@d1@23@AEAVtask_group_context@523@@Z\n?submit@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@PEAVarena@123@_K@Z\n?current_context@r1@detail@tbb@@YAPEAVtask_group_context@d1@23@XZ\n\n; Task group context (task_group_context.cpp)\n?initialize@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z\n?destroy@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z\n?is_group_execution_cancelled@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z\n?reset@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z\n?cancel_group_execution@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z\n?capture_fp_settings@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z\n\n; Task arena (arena.cpp)\n?max_concurrency@r1@detail@tbb@@YAHPEBVtask_arena_base@d1@23@@Z\n?initialize@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z\n?terminate@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z\n?execute@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@AEAVdelegate_base@523@@Z\n?wait@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z\n?attach@r1@detail@tbb@@YA_NAEAVtask_arena_base@d1@23@@Z\n?isolate_within_arena@r1@detail@tbb@@YAXAEAVdelegate_base@d1@23@_J@Z\n?enqueue@r1@detail@tbb@@YAXAEAVtask@d1@23@PEAVtask_arena_base@523@@Z\n?enqueue@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@PEAVtask_arena_base@523@@Z\n?execution_slot@r1@detail@tbb@@YAGAEBVtask_arena_base@d1@23@@Z\n\n; System topology parsing and threads pinning (governor.cpp)\n?numa_node_count@r1@detail@tbb@@YAIXZ\n?fill_numa_indices@r1@detail@tbb@@YAXPEAH@Z\n?core_type_count@r1@detail@tbb@@YAI_J@Z\n?fill_core_type_indices@r1@detail@tbb@@YAXPEAH_J@Z\n?numa_default_concurrency@r1@detail@tbb@@YAHH@Z\n?constraints_default_concurrency@r1@detail@tbb@@YAHAEBUconstraints@d1@23@_J@Z\n?constraints_threads_per_core@r1@detail@tbb@@YAHAEBUconstraints@d1@23@_J@Z\n\n; Observer (observer_proxy.cpp)\n?observe@r1@detail@tbb@@YAXAEAVtask_scheduler_observer@d1@23@_N@Z\n\n; Queuing RW Mutex (queuing_rw_mutex.cpp)\n?construct@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@@Z\n?try_acquire@r1@detail@tbb@@YA_NAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z\n?acquire@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z\n?release@r1@detail@tbb@@YAXAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?upgrade_to_writer@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?downgrade_to_reader@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z\n?is_writer@r1@detail@tbb@@YA_NAEBVscoped_lock@queuing_rw_mutex@d1@23@@Z\n\n; Global control (global_control.cpp)\n?global_control_active_value@r1@detail@tbb@@YA_KH@Z\n?create@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z\n?destroy@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z\n?get@r1@detail@tbb@@YAXAEAVtask_scheduler_handle@d1@23@@Z\n?finalize@r1@detail@tbb@@YA_NAEAVtask_scheduler_handle@d1@23@_J@Z\n\n; Parallel pipeline (parallel_pipeline.cpp)\n?set_end_of_input@r1@detail@tbb@@YAXAEAVbase_filter@d1@23@@Z\n?parallel_pipeline@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@_KAEBVfilter_node@523@@Z\n\n; Concurrent bounded queue (concurrent_bounded_queue.cpp)\n?allocate_bounded_queue_rep@r1@detail@tbb@@YAPEAE_K@Z\n?deallocate_bounded_queue_rep@r1@detail@tbb@@YAXPEAE_K@Z\n?wait_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K_JAEAVdelegate_base@d1@23@@Z\n?abort_bounded_queue_monitors@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@@Z\n?notify_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K1@Z\n\n; Concurrent monitor (address_waiter.cpp)\n?wait_on_address@r1@detail@tbb@@YAXPEAXAEAVdelegate_base@d1@23@_K@Z\n?notify_by_address@r1@detail@tbb@@YAXPEAX_K@Z\n?notify_by_address_one@r1@detail@tbb@@YAXPEAX@Z\n?notify_by_address_all@r1@detail@tbb@@YAXPEAX@Z\n\n;; Versioning (version.cpp)\nTBB_runtime_interface_version\nTBB_runtime_version\n" }, { "path": "src/tbb/src/tbb/dynamic_link.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"dynamic_link.h\"\n#include \"environment.h\"\n\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n\n/*\n This file is used by both TBB and OpenMP RTL. Do not use __TBB_ASSERT() macro\n and runtime_warning() function because they are not available in OpenMP. Use\n __TBB_ASSERT_EX and DYNAMIC_LINK_WARNING instead.\n*/\n\n#include // va_list etc.\n#include // strrchr\n#if _WIN32\n #include \n\n // Unify system calls\n #define dlopen( name, flags ) LoadLibrary( name )\n #define dlsym( handle, name ) GetProcAddress( handle, name )\n // FreeLibrary return bool value that is not used.\n #define dlclose( handle ) (void)( ! FreeLibrary( handle ) )\n #define dlerror() GetLastError()\n#ifndef PATH_MAX\n #define PATH_MAX MAX_PATH\n#endif\n#else /* _WIN32 */\n #include \n #include \n\n #include \n #include \n#endif /* _WIN32 */\n\n#if __TBB_WEAK_SYMBOLS_PRESENT && !__TBB_DYNAMIC_LOAD_ENABLED\n //TODO: use function attribute for weak symbols instead of the pragma.\n #pragma weak dlopen\n #pragma weak dlsym\n #pragma weak dlclose\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT && !__TBB_DYNAMIC_LOAD_ENABLED */\n\n\n#define __USE_STATIC_DL_INIT ( !__ANDROID__ )\n\n\n/*\ndynamic_link is a common interface for searching for required symbols in an\nexecutable and dynamic libraries.\n\ndynamic_link provides certain guarantees:\n 1. Either all or none of the requested symbols are resolved. Moreover, if\n symbols are not resolved, the dynamic_link_descriptor table is not modified;\n 2. All returned symbols have secured lifetime: this means that none of them\n can be invalidated until dynamic_unlink is called;\n 3. Any loaded library is loaded only via the full path. The full path is that\n from which the runtime itself was loaded. (This is done to avoid security\n issues caused by loading libraries from insecure paths).\n\ndynamic_link searches for the requested symbols in three stages, stopping as\nsoon as all of the symbols have been resolved.\n\n 1. Search the global scope:\n a. On Windows: dynamic_link tries to obtain the handle of the requested\n library and if it succeeds it resolves the symbols via that handle.\n b. On Linux: dynamic_link tries to search for the symbols in the global\n scope via the main program handle. If the symbols are present in the global\n scope their lifetime is not guaranteed (since dynamic_link does not know\n anything about the library from which they are exported). Therefore it\n tries to \"pin\" the symbols by obtaining the library name and reopening it.\n dlopen may fail to reopen the library in two cases:\n i. The symbols are exported from the executable. Currently dynamic _link\n cannot handle this situation, so it will not find these symbols in this\n step.\n ii. The necessary library has been unloaded and cannot be reloaded. It\n seems there is nothing that can be done in this case. No symbols are\n returned.\n\n 2. Dynamic load: an attempt is made to load the requested library via the\n full path.\n The full path used is that from which the runtime itself was loaded. If the\n library can be loaded, then an attempt is made to resolve the requested\n symbols in the newly loaded library.\n If the symbols are not found the library is unloaded.\n\n 3. Weak symbols: if weak symbols are available they are returned.\n*/\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED\n\n#if !defined(DYNAMIC_LINK_WARNING) && !__TBB_WIN8UI_SUPPORT && __TBB_DYNAMIC_LOAD_ENABLED\n // Report runtime errors and continue.\n #define DYNAMIC_LINK_WARNING dynamic_link_warning\n static void dynamic_link_warning( dynamic_link_error_t code, ... ) {\n suppress_unused_warning(code);\n } // library_warning\n#endif /* !defined(DYNAMIC_LINK_WARNING) && !__TBB_WIN8UI_SUPPORT && __TBB_DYNAMIC_LOAD_ENABLED */\n\n static bool resolve_symbols( dynamic_link_handle module, const dynamic_link_descriptor descriptors[], std::size_t required )\n {\n if ( !module )\n return false;\n\n #if !__TBB_DYNAMIC_LOAD_ENABLED /* only __TBB_WEAK_SYMBOLS_PRESENT is defined */\n if ( !dlsym ) return false;\n #endif /* !__TBB_DYNAMIC_LOAD_ENABLED */\n\n const std::size_t n_desc=20; // Usually we don't have more than 20 descriptors per library\n __TBB_ASSERT_EX( required <= n_desc, \"Too many descriptors is required\" );\n if ( required > n_desc ) return false;\n pointer_to_handler h[n_desc];\n\n for ( std::size_t k = 0; k < required; ++k ) {\n dynamic_link_descriptor const & desc = descriptors[k];\n pointer_to_handler addr = (pointer_to_handler)dlsym( module, desc.name );\n if ( !addr ) {\n return false;\n }\n h[k] = addr;\n }\n\n // Commit the entry points.\n // Cannot use memset here, because the writes must be atomic.\n for( std::size_t k = 0; k < required; ++k )\n *descriptors[k].handler = h[k];\n return true;\n }\n\n#if __TBB_WIN8UI_SUPPORT\n bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, dynamic_link_handle*, int flags ) {\n dynamic_link_handle tmp_handle = nullptr;\n TCHAR wlibrary[256];\n if ( MultiByteToWideChar(CP_UTF8, 0, library, -1, wlibrary, 255) == 0 ) return false;\n if ( flags & DYNAMIC_LINK_LOAD )\n tmp_handle = LoadPackagedLibrary( wlibrary, 0 );\n if (tmp_handle != nullptr){\n return resolve_symbols(tmp_handle, descriptors, required);\n }else{\n return false;\n }\n }\n void dynamic_unlink( dynamic_link_handle ) {}\n void dynamic_unlink_all() {}\n#else\n#if __TBB_DYNAMIC_LOAD_ENABLED\n/*\n There is a security issue on Windows: LoadLibrary() may load and execute malicious code.\n See http://www.microsoft.com/technet/security/advisory/2269637.mspx for details.\n To avoid the issue, we have to pass full path (not just library name) to LoadLibrary. This\n function constructs full path to the specified library (it is assumed the library located\n side-by-side with the tbb.dll.\n\n The function constructs absolute path for given relative path. Important: Base directory is not\n current one, it is the directory tbb.dll loaded from.\n\n Example:\n Let us assume \"tbb.dll\" is located in \"c:\\program files\\common\\intel\\\" directory, e.g.\n absolute path of the library is \"c:\\program files\\common\\intel\\tbb.dll\". Absolute path for\n \"tbbmalloc.dll\" would be \"c:\\program files\\common\\intel\\tbbmalloc.dll\". Absolute path for\n \"malloc\\tbbmalloc.dll\" would be \"c:\\program files\\common\\intel\\malloc\\tbbmalloc.dll\".\n*/\n\n // Struct handle_storage is used by dynamic_link routine to store handles of\n // all loaded or pinned dynamic libraries. When TBB is shut down, it calls\n // dynamic_unlink_all() that unloads modules referenced by handle_storage.\n // This struct should not have any constructors since it may be used before\n // the constructor is called.\n #define MAX_LOADED_MODULES 8 // The number of maximum possible modules which can be loaded\n\n using atomic_incrementer = std::atomic;\n\n static struct handles_t {\n atomic_incrementer my_size;\n dynamic_link_handle my_handles[MAX_LOADED_MODULES];\n\n void add(const dynamic_link_handle &handle) {\n const std::size_t ind = my_size++;\n __TBB_ASSERT_EX( ind < MAX_LOADED_MODULES, \"Too many modules are loaded\" );\n my_handles[ind] = handle;\n }\n\n void free() {\n const std::size_t size = my_size;\n for (std::size_t i=0; i( PATH_MAX ) );\n if ( drc == 0 ) { // Error occurred.\n int err = GetLastError();\n DYNAMIC_LINK_WARNING( dl_sys_fail, \"GetModuleFileName\", err );\n return;\n }\n if ( drc >= PATH_MAX ) { // Buffer too short.\n DYNAMIC_LINK_WARNING( dl_buff_too_small );\n return;\n }\n // Find the position of the last backslash.\n char *backslash = std::strrchr( ap_data._path, '\\\\' );\n\n if ( !backslash ) { // Backslash not found.\n __TBB_ASSERT_EX( backslash != nullptr, \"Unbelievable.\");\n return;\n }\n __TBB_ASSERT_EX( backslash >= ap_data._path, \"Unbelievable.\");\n ap_data._len = (std::size_t)(backslash - ap_data._path) + 1;\n *(backslash+1) = 0;\n #else\n // Get the library path\n Dl_info dlinfo;\n int res = dladdr( (void*)&dynamic_link, &dlinfo ); // any function inside the library can be used for the address\n if ( !res ) {\n char const * err = dlerror();\n DYNAMIC_LINK_WARNING( dl_sys_fail, \"dladdr\", err );\n return;\n } else {\n __TBB_ASSERT_EX( dlinfo.dli_fname!=nullptr, \"Unbelievable.\" );\n }\n\n char const *slash = std::strrchr( dlinfo.dli_fname, '/' );\n std::size_t fname_len=0;\n if ( slash ) {\n __TBB_ASSERT_EX( slash >= dlinfo.dli_fname, \"Unbelievable.\");\n fname_len = (std::size_t)(slash - dlinfo.dli_fname) + 1;\n }\n\n std::size_t rc;\n if ( dlinfo.dli_fname[0]=='/' ) {\n // The library path is absolute\n rc = 0;\n ap_data._len = 0;\n } else {\n // The library path is relative so get the current working directory\n if ( !getcwd( ap_data._path, sizeof(ap_data._path)/sizeof(ap_data._path[0]) ) ) {\n DYNAMIC_LINK_WARNING( dl_buff_too_small );\n return;\n }\n ap_data._len = std::strlen( ap_data._path );\n ap_data._path[ap_data._len++]='/';\n rc = ap_data._len;\n }\n\n if ( fname_len>0 ) {\n ap_data._len += fname_len;\n if ( ap_data._len>PATH_MAX ) {\n DYNAMIC_LINK_WARNING( dl_buff_too_small );\n ap_data._len=0;\n return;\n }\n std::strncpy( ap_data._path+rc, dlinfo.dli_fname, fname_len );\n ap_data._path[ap_data._len]=0;\n }\n #endif /* _WIN32 */\n }\n\n static void init_dl_data() {\n init_ap_data();\n }\n\n /*\n The function constructs absolute path for given relative path. Important: Base directory is not\n current one, it is the directory libtbb.so loaded from.\n\n Arguments:\n in name -- Name of a file (may be with relative path; it must not be an absolute one).\n out path -- Buffer to save result (absolute path) to.\n in len -- Size of buffer.\n ret -- 0 -- Error occurred.\n > len -- Buffer too short, required size returned.\n otherwise -- Ok, number of characters (incl. terminating null) written to buffer.\n */\n static std::size_t abs_path( char const * name, char * path, std::size_t len ) {\n if ( ap_data._len == 0 )\n return 0;\n\n std::size_t name_len = std::strlen( name );\n std::size_t full_len = name_len+ap_data._len;\n if ( full_len < len ) {\n __TBB_ASSERT( ap_data._path[ap_data._len] == 0, nullptr);\n __TBB_ASSERT( std::strlen(ap_data._path) == ap_data._len, nullptr);\n std::strncpy( path, ap_data._path, ap_data._len + 1 );\n __TBB_ASSERT( path[ap_data._len] == 0, nullptr);\n std::strncat( path, name, len - ap_data._len );\n __TBB_ASSERT( std::strlen(path) == full_len, nullptr);\n }\n return full_len+1; // +1 for null character\n }\n#endif // __TBB_DYNAMIC_LOAD_ENABLED\n void init_dynamic_link_data() {\n #if __TBB_DYNAMIC_LOAD_ENABLED\n std::call_once( init_dl_data_state, init_dl_data );\n #endif\n }\n\n #if __USE_STATIC_DL_INIT\n // ap_data structure is initialized with current directory on Linux.\n // So it should be initialized as soon as possible since the current directory may be changed.\n // static_init_ap_data object provides this initialization during library loading.\n static struct static_init_dl_data_t {\n static_init_dl_data_t() {\n init_dynamic_link_data();\n }\n } static_init_dl_data;\n #endif\n\n #if __TBB_WEAK_SYMBOLS_PRESENT\n static bool weak_symbol_link( const dynamic_link_descriptor descriptors[], std::size_t required )\n {\n // Check if the required entries are present in what was loaded into our process.\n for ( std::size_t k = 0; k < required; ++k )\n if ( !descriptors[k].ptr )\n return false;\n // Commit the entry points.\n for ( std::size_t k = 0; k < required; ++k )\n *descriptors[k].handler = (pointer_to_handler) descriptors[k].ptr;\n return true;\n }\n #else\n static bool weak_symbol_link( const dynamic_link_descriptor[], std::size_t ) {\n return false;\n }\n #endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\n void dynamic_unlink( dynamic_link_handle handle ) {\n #if !__TBB_DYNAMIC_LOAD_ENABLED /* only __TBB_WEAK_SYMBOLS_PRESENT is defined */\n if ( !dlclose ) return;\n #endif\n if ( handle ) {\n dlclose( handle );\n }\n }\n\n void dynamic_unlink_all() {\n #if __TBB_DYNAMIC_LOAD_ENABLED\n handles.free();\n #endif\n }\n\n static dynamic_link_handle global_symbols_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required ) {\n dynamic_link_handle library_handle{};\n#if _WIN32\n auto res = GetModuleHandleEx(0, library, &library_handle);\n __TBB_ASSERT_EX((res && library_handle) || (!res && !library_handle), nullptr);\n#else /* _WIN32 */\n #if !__TBB_DYNAMIC_LOAD_ENABLED /* only __TBB_WEAK_SYMBOLS_PRESENT is defined */\n if ( !dlopen ) return 0;\n #endif /* !__TBB_DYNAMIC_LOAD_ENABLED */\n // RTLD_GLOBAL - to guarantee that old TBB will find the loaded library\n // RTLD_NOLOAD - not to load the library without the full path\n library_handle = dlopen(library, RTLD_LAZY | RTLD_GLOBAL | RTLD_NOLOAD);\n#endif /* _WIN32 */\n if (library_handle) {\n if (!resolve_symbols(library_handle, descriptors, required)) {\n dynamic_unlink(library_handle);\n library_handle = nullptr;\n }\n }\n return library_handle;\n }\n\n static void save_library_handle( dynamic_link_handle src, dynamic_link_handle *dst ) {\n __TBB_ASSERT_EX( src, \"The library handle to store must be non-zero\" );\n if ( dst )\n *dst = src;\n #if __TBB_DYNAMIC_LOAD_ENABLED\n else\n handles.add( src );\n #endif /* __TBB_DYNAMIC_LOAD_ENABLED */\n }\n\n#if !_WIN32\n int loading_flags(bool local_binding) {\n int flags = RTLD_NOW;\n if (local_binding) {\n flags = flags | RTLD_LOCAL;\n#if (__linux__ && __GLIBC__) && !__TBB_USE_SANITIZERS\n if( !GetBoolEnvironmentVariable(\"TBB_ENABLE_SANITIZERS\") ) {\n flags = flags | RTLD_DEEPBIND;\n }\n#endif\n } else {\n flags = flags | RTLD_GLOBAL;\n }\n return flags;\n }\n#endif\n\n dynamic_link_handle dynamic_load( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, bool local_binding ) {\n ::tbb::detail::suppress_unused_warning( library, descriptors, required, local_binding );\n#if __TBB_DYNAMIC_LOAD_ENABLED\n std::size_t const len = PATH_MAX + 1;\n char path[ len ];\n std::size_t rc = abs_path( library, path, len );\n if ( 0 < rc && rc <= len ) {\n#if _WIN32\n // Prevent Windows from displaying silly message boxes if it fails to load library\n // (e.g. because of MS runtime problems - one of those crazy manifest related ones)\n UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS);\n#endif /* _WIN32 */\n // The second argument (loading_flags) is ignored on Windows\n dynamic_link_handle library_handle = dlopen( path, loading_flags(local_binding) );\n#if _WIN32\n SetErrorMode (prev_mode);\n#endif /* _WIN32 */\n if( library_handle ) {\n if( !resolve_symbols( library_handle, descriptors, required ) ) {\n // The loaded library does not contain all the expected entry points\n dynamic_unlink( library_handle );\n library_handle = nullptr;\n }\n } else\n DYNAMIC_LINK_WARNING( dl_lib_not_found, path, dlerror() );\n return library_handle;\n } else if ( rc>len )\n DYNAMIC_LINK_WARNING( dl_buff_too_small );\n // rc == 0 means failing of init_ap_data so the warning has already been issued.\n\n#endif /* __TBB_DYNAMIC_LOAD_ENABLED */\n return nullptr;\n }\n\n bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], std::size_t required, dynamic_link_handle *handle, int flags ) {\n init_dynamic_link_data();\n\n // TODO: May global_symbols_link find weak symbols?\n dynamic_link_handle library_handle = ( flags & DYNAMIC_LINK_GLOBAL ) ? global_symbols_link( library, descriptors, required ) : nullptr;\n\n#if defined(_MSC_VER) && _MSC_VER <= 1900\n// #pragma warning (push)\n// MSVC 2015 warning: 'int': forcing value to bool 'true' or 'false'\n// #pragma warning (disable: 4800)\n#endif\n if ( !library_handle && ( flags & DYNAMIC_LINK_LOAD ) )\n library_handle = dynamic_load( library, descriptors, required, flags & DYNAMIC_LINK_LOCAL );\n\n#if defined(_MSC_VER) && _MSC_VER <= 1900\n// #pragma warning (pop)\n#endif\n if ( !library_handle && ( flags & DYNAMIC_LINK_WEAK ) )\n return weak_symbol_link( descriptors, required );\n\n if ( library_handle ) {\n save_library_handle( library_handle, handle );\n return true;\n }\n return false;\n }\n\n#endif /*__TBB_WIN8UI_SUPPORT*/\n#else /* __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED */\n bool dynamic_link( const char*, const dynamic_link_descriptor*, std::size_t, dynamic_link_handle *handle, int ) {\n if ( handle )\n *handle=0;\n return false;\n }\n void dynamic_unlink( dynamic_link_handle ) {}\n void dynamic_unlink_all() {}\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT || __TBB_DYNAMIC_LOAD_ENABLED */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/dynamic_link.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_dynamic_link\n#define __TBB_dynamic_link\n\n// Support for dynamic loading entry points from other shared libraries.\n\n#include \"oneapi/tbb/detail/_config.h\"\n\n#include \n#include \n\n/** By default, symbols declared and defined here go into namespace tbb::internal.\n To put them in other namespace, define macros OPEN_INTERNAL_NAMESPACE\n and CLOSE_INTERNAL_NAMESPACE to override the following default definitions. **/\n\n#include \n#ifdef _WIN32\n#include \n#endif /* _WIN32 */\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Type definition for a pointer to a void somefunc(void)\ntypedef void (*pointer_to_handler)();\n\n//! The helper to construct dynamic_link_descriptor structure\n// Double cast through the void* in DLD macro is necessary to\n// prevent warnings from some compilers (g++ 4.1)\n#if __TBB_WEAK_SYMBOLS_PRESENT\n#define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h), (pointer_to_handler)&s}\n#define DLD_NOWEAK(s,h) {#s, (pointer_to_handler*)(void*)(&h), nullptr}\n#else\n#define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h)}\n#define DLD_NOWEAK(s,h) DLD(s,h)\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n//! Association between a handler name and location of pointer to it.\nstruct dynamic_link_descriptor {\n //! Name of the handler\n const char* name;\n //! Pointer to the handler\n pointer_to_handler* handler;\n#if __TBB_WEAK_SYMBOLS_PRESENT\n //! Weak symbol\n pointer_to_handler ptr;\n#endif\n};\n\n#if _WIN32\nusing dynamic_link_handle = HMODULE;\n#else\nusing dynamic_link_handle = void*;\n#endif /* _WIN32 */\n\nconst int DYNAMIC_LINK_GLOBAL = 0x01;\nconst int DYNAMIC_LINK_LOAD = 0x02;\nconst int DYNAMIC_LINK_WEAK = 0x04;\nconst int DYNAMIC_LINK_LOCAL = 0x08;\n\nconst int DYNAMIC_LINK_LOCAL_BINDING = DYNAMIC_LINK_LOCAL | DYNAMIC_LINK_LOAD;\nconst int DYNAMIC_LINK_DEFAULT = DYNAMIC_LINK_GLOBAL | DYNAMIC_LINK_LOAD | DYNAMIC_LINK_WEAK;\n\n//! Fill in dynamically linked handlers.\n/** 'library' is the name of the requested library. It should not contain a full\n path since dynamic_link adds the full path (from which the runtime itself\n was loaded) to the library name.\n 'required' is the number of the initial entries in the array descriptors[]\n that have to be found in order for the call to succeed. If the library and\n all the required handlers are found, then the corresponding handler\n pointers are set, and the return value is true. Otherwise the original\n array of descriptors is left untouched and the return value is false.\n 'required' is limited by 20 (exceeding of this value will result in failure\n to load the symbols and the return value will be false).\n 'handle' is the handle of the library if it is loaded. Otherwise it is left\n untouched.\n 'flags' is the set of DYNAMIC_LINK_* flags. Each of the DYNAMIC_LINK_* flags\n allows its corresponding linking stage.\n**/\nbool dynamic_link( const char* library,\n const dynamic_link_descriptor descriptors[],\n std::size_t required,\n dynamic_link_handle* handle = nullptr,\n int flags = DYNAMIC_LINK_DEFAULT );\n\nvoid dynamic_unlink( dynamic_link_handle handle );\n\nvoid dynamic_unlink_all();\n\nenum dynamic_link_error_t {\n dl_success = 0,\n dl_lib_not_found, // char const * lib, dlerr_t err\n dl_sym_not_found, // char const * sym, dlerr_t err\n // Note: dlerr_t depends on OS: it is char const * on Linux* and macOS*, int on Windows*.\n dl_sys_fail, // char const * func, int err\n dl_buff_too_small // none\n}; // dynamic_link_error_t\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_dynamic_link */\n" }, { "path": "src/tbb/src/tbb/environment.h", "content": "/*\n Copyright (c) 2018-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbb_environment_H\n#define __TBB_tbb_environment_H\n\n#include \n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_WIN8UI_SUPPORT\nstatic inline bool GetBoolEnvironmentVariable( const char * ) {\n return false;\n}\n\nstatic inline long GetIntegralEnvironmentVariable( const char * ) {\n return -1;\n}\n#else /* __TBB_WIN8UI_SUPPORT */\nstatic inline bool GetBoolEnvironmentVariable( const char * name ) {\n if ( const char* s = std::getenv(name) ) {\n // The result is defined as true only if the environment variable contains\n // no characters except one '1' character and an arbitrary number of spaces\n // (including the absence of spaces).\n size_t index = std::strspn(s, \" \");\n if (s[index] != '1') return false;\n index++;\n // Memory access after incrementing is safe, since the getenv() returns a\n // null-terminated string, and even if the character getting by index is '1',\n // and this character is the end of string, after incrementing we will get\n // an index of character, that contains '\\0'\n index += std::strspn(&s[index], \" \");\n return !s[index];\n }\n return false;\n}\n\nstatic inline long GetIntegralEnvironmentVariable( const char * name ) {\n if ( const char* s = std::getenv(name) ) {\n char* end = nullptr;\n errno = 0;\n long value = std::strtol(s, &end, 10);\n\n // We have exceeded the range, value is negative or string is incovertable\n if ( errno == ERANGE || value < 0 || end==s ) {\n return -1;\n }\n for ( ; *end != '\\0'; end++ ) {\n if ( !std::isspace(*end) ) {\n return -1;\n }\n }\n return value;\n }\n return -1;\n}\n#endif /* __TBB_WIN8UI_SUPPORT */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_tbb_environment_H\n" }, { "path": "src/tbb/src/tbb/exception.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_exception.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n\n#include \n#include \n#include // std::runtime_error\n#include \n#include \n\n#define __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN \\\n (__GLIBCXX__ && __TBB_GLIBCXX_VERSION>=40700 && __TBB_GLIBCXX_VERSION<60000 && TBB_USE_EXCEPTIONS)\n\n#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN\n// GCC ABI declarations necessary for a workaround\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nconst char* bad_last_alloc::what() const noexcept(true) { return \"bad allocation in previous or concurrent attempt\"; }\nconst char* user_abort::what() const noexcept(true) { return \"User-initiated abort has terminated this operation\"; }\nconst char* missing_wait::what() const noexcept(true) { return \"wait() was not called on the structured_task_group\"; }\n\n#if TBB_USE_EXCEPTIONS\n template \n /*[[noreturn]]*/ void do_throw_noexcept(F throw_func) noexcept {\n throw_func();\n }\n\n /*[[noreturn]]*/ void do_throw_noexcept(void (*throw_func)()) noexcept {\n throw_func();\n#if __GNUC__ == 7\n // In release, GCC 7 loses noexcept attribute during tail call optimization.\n // The following statement prevents tail call optimization.\n volatile bool reach_this_point = true;\n suppress_unused_warning(reach_this_point);\n#endif\n }\n\n bool terminate_on_exception(); // defined in global_control.cpp and ipc_server.cpp\n\n template \n /*[[noreturn]]*/ void do_throw(F throw_func) {\n if (terminate_on_exception()) {\n do_throw_noexcept(throw_func);\n }\n throw_func();\n }\n\n #define DO_THROW(exc, init_args) do_throw( []{ throw exc init_args; } );\n#else /* !TBB_USE_EXCEPTIONS */\n #define PRINT_ERROR_AND_ABORT(exc_name, msg) \\\n std::fprintf (stderr, \"Exception %s with message %s would have been thrown, \" \\\n \"if exception handling had not been disabled. Aborting.\\n\", exc_name, msg); \\\n std::fflush(stderr); \\\n std::abort();\n #define DO_THROW(exc, init_args) PRINT_ERROR_AND_ABORT(#exc, #init_args)\n#endif /* !TBB_USE_EXCEPTIONS */\n\nvoid throw_exception ( exception_id eid ) {\n switch ( eid ) {\n case exception_id::bad_alloc: DO_THROW(std::bad_alloc, ()); break;\n case exception_id::bad_last_alloc: DO_THROW(bad_last_alloc, ()); break;\n case exception_id::user_abort: DO_THROW( user_abort, () ); break;\n case exception_id::nonpositive_step: DO_THROW(std::invalid_argument, (\"Step must be positive\") ); break;\n case exception_id::out_of_range: DO_THROW(std::out_of_range, (\"Index out of requested size range\")); break;\n case exception_id::reservation_length_error: DO_THROW(std::length_error, (\"Attempt to exceed implementation defined length limits\")); break;\n case exception_id::missing_wait: DO_THROW(missing_wait, ()); break;\n case exception_id::invalid_load_factor: DO_THROW(std::out_of_range, (\"Invalid hash load factor\")); break;\n case exception_id::invalid_key: DO_THROW(std::out_of_range, (\"invalid key\")); break;\n case exception_id::bad_tagged_msg_cast: DO_THROW(std::runtime_error, (\"Illegal tagged_msg cast\")); break;\n case exception_id::unsafe_wait: DO_THROW(unsafe_wait, (\"Unsafe to wait further\")); break;\n default: __TBB_ASSERT ( false, \"Unknown exception ID\" );\n }\n __TBB_ASSERT(false, \"Unreachable code\");\n}\n\n/* The \"what\" should be fairly short, not more than about 128 characters.\n Because we control all the call sites to handle_perror, it is pointless\n to bullet-proof it for very long strings.\n\n Design note: ADR put this routine off to the side in tbb_misc.cpp instead of\n Task.cpp because the throw generates a pathetic lot of code, and ADR wanted\n this large chunk of code to be placed on a cold page. */\nvoid handle_perror( int error_code, const char* what ) {\n const int BUF_SIZE = 255;\n char buf[BUF_SIZE + 1] = { 0 };\n std::strncat(buf, what, BUF_SIZE);\n std::size_t buf_len = std::strlen(buf);\n if (error_code) {\n std::strncat(buf, \": \", BUF_SIZE - buf_len);\n buf_len = std::strlen(buf);\n std::strncat(buf, std::strerror(error_code), BUF_SIZE - buf_len);\n buf_len = std::strlen(buf);\n }\n __TBB_ASSERT(buf_len <= BUF_SIZE && buf[buf_len] == 0, nullptr);\n#if TBB_USE_EXCEPTIONS\n do_throw([&buf] { throw std::runtime_error(buf); });\n#else\n PRINT_ERROR_AND_ABORT( \"runtime_error\", buf);\n#endif /* !TBB_USE_EXCEPTIONS */\n}\n\n#if __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN\n// Runtime detection and workaround for the GCC bug 62258.\n// The problem is that std::rethrow_exception() does not increment a counter\n// of active exceptions, causing std::uncaught_exception() to return a wrong value.\n// The code is created after, and roughly reflects, the workaround\n// at https://gcc.gnu.org/bugzilla/attachment.cgi?id=34683\n\nvoid fix_broken_rethrow() {\n struct gcc_eh_data {\n void * caughtExceptions;\n unsigned int uncaughtExceptions;\n };\n gcc_eh_data* eh_data = punned_cast( abi::__cxa_get_globals() );\n ++eh_data->uncaughtExceptions;\n}\n\nbool gcc_rethrow_exception_broken() {\n bool is_broken;\n __TBB_ASSERT( !std::uncaught_exception(),\n \"gcc_rethrow_exception_broken() must not be called when an exception is active\" );\n try {\n // Throw, catch, and rethrow an exception\n try {\n throw __TBB_GLIBCXX_VERSION;\n } catch(...) {\n std::rethrow_exception( std::current_exception() );\n }\n } catch(...) {\n // Check the bug presence\n is_broken = std::uncaught_exception();\n }\n if( is_broken ) fix_broken_rethrow();\n __TBB_ASSERT( !std::uncaught_exception(), nullptr);\n return is_broken;\n}\n#else\nvoid fix_broken_rethrow() {}\nbool gcc_rethrow_exception_broken() { return false; }\n#endif /* __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n" }, { "path": "src/tbb/src/tbb/global_control.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/global_control.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/spin_mutex.h\"\n\n#include \"governor.h\"\n#include \"threading_control.h\"\n#include \"market.h\"\n#include \"misc.h\"\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Comparator for a set of global_control objects\nstruct control_storage_comparator {\n bool operator()(const d1::global_control* lhs, const d1::global_control* rhs) const;\n};\n\nclass control_storage {\n friend struct global_control_impl;\n friend std::size_t global_control_active_value(int);\n friend void global_control_lock();\n friend void global_control_unlock();\n friend std::size_t global_control_active_value_unsafe(d1::global_control::parameter);\nprotected:\n std::size_t my_active_value{0};\n std::set> my_list{};\n spin_mutex my_list_mutex{};\npublic:\n virtual ~control_storage() = default;\n virtual std::size_t default_value() const = 0;\n virtual void apply_active(std::size_t new_active) {\n my_active_value = new_active;\n }\n virtual bool is_first_arg_preferred(std::size_t a, std::size_t b) const {\n return a>b; // prefer max by default\n }\n virtual std::size_t active_value() {\n spin_mutex::scoped_lock lock(my_list_mutex); // protect my_list.empty() call\n return !my_list.empty() ? my_active_value : default_value();\n }\n\n std::size_t active_value_unsafe() {\n return !my_list.empty() ? my_active_value : default_value();\n }\n};\n\nclass alignas(max_nfs_size) allowed_parallelism_control : public control_storage {\n std::size_t default_value() const override {\n return max(1U, governor::default_num_threads());\n }\n bool is_first_arg_preferred(std::size_t a, std::size_t b) const override {\n return a= 1, nullptr);\n // -1 to take external thread into account\n threading_control::set_active_num_workers(my_active_value - 1);\n }\n std::size_t active_value() override {\n spin_mutex::scoped_lock lock(my_list_mutex); // protect my_list.empty() call\n if (my_list.empty()) {\n return default_value();\n }\n\n // non-zero, if market is active\n const std::size_t workers = threading_control::max_num_workers();\n // We can't exceed market's maximal number of workers.\n // +1 to take external thread into account\n return workers ? min(workers + 1, my_active_value) : my_active_value;\n }\n};\n\nclass alignas(max_nfs_size) stack_size_control : public control_storage {\n std::size_t default_value() const override {\n#if _WIN32_WINNT >= 0x0602 /* _WIN32_WINNT_WIN8 */\n static auto ThreadStackSizeDefault = [] {\n ULONG_PTR hi, lo;\n GetCurrentThreadStackLimits(&lo, &hi);\n return hi - lo;\n }();\n return ThreadStackSizeDefault;\n#elif defined(EMSCRIPTEN)\n return __TBB_EMSCRIPTEN_STACK_SIZE;\n#else\n return ThreadStackSize;\n#endif\n }\n void apply_active(std::size_t new_active) override {\n control_storage::apply_active(new_active);\n#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\n __TBB_ASSERT( false, \"For Windows 8 Store* apps we must not set stack size\" );\n#endif\n }\n};\n\nclass alignas(max_nfs_size) terminate_on_exception_control : public control_storage {\n std::size_t default_value() const override {\n return 0;\n }\n};\n\nclass alignas(max_nfs_size) lifetime_control : public control_storage {\n bool is_first_arg_preferred(std::size_t, std::size_t) const override {\n return false; // not interested\n }\n std::size_t default_value() const override {\n return 0;\n }\n void apply_active(std::size_t new_active) override {\n if (new_active == 1) {\n // reserve the market reference\n threading_control::register_lifetime_control();\n } else if (new_active == 0) { // new_active == 0\n threading_control::unregister_lifetime_control(/*blocking_terminate*/ false);\n }\n control_storage::apply_active(new_active);\n }\n};\n\nstatic control_storage* controls[] = {nullptr, nullptr, nullptr, nullptr};\n\nvoid global_control_acquire() {\n controls[0] = new (cache_aligned_allocate(sizeof(allowed_parallelism_control))) allowed_parallelism_control{};\n controls[1] = new (cache_aligned_allocate(sizeof(stack_size_control))) stack_size_control{};\n controls[2] = new (cache_aligned_allocate(sizeof(terminate_on_exception_control))) terminate_on_exception_control{};\n controls[3] = new (cache_aligned_allocate(sizeof(lifetime_control))) lifetime_control{};\n}\n\nvoid global_control_release() {\n for (auto& ptr : controls) {\n ptr->~control_storage();\n cache_aligned_deallocate(ptr);\n ptr = nullptr;\n }\n}\n\nvoid global_control_lock() {\n for (auto& ctl : controls) {\n ctl->my_list_mutex.lock();\n }\n}\n\nvoid global_control_unlock() {\n int N = std::distance(std::begin(controls), std::end(controls));\n for (int i = N - 1; i >= 0; --i) {\n controls[i]->my_list_mutex.unlock();\n }\n}\n\nstd::size_t global_control_active_value_unsafe(d1::global_control::parameter param) {\n __TBB_ASSERT_RELEASE(param < d1::global_control::parameter_max, nullptr);\n return controls[param]->active_value_unsafe();\n}\n\n//! Comparator for a set of global_control objects\ninline bool control_storage_comparator::operator()(const d1::global_control* lhs, const d1::global_control* rhs) const {\n __TBB_ASSERT_RELEASE(lhs->my_param < d1::global_control::parameter_max , nullptr);\n return lhs->my_value < rhs->my_value || (lhs->my_value == rhs->my_value && lhs < rhs);\n}\n\nbool terminate_on_exception() {\n return d1::global_control::active_value(d1::global_control::terminate_on_exception) == 1;\n}\n\nstruct global_control_impl {\nprivate:\n static bool erase_if_present(control_storage* const c, d1::global_control& gc) {\n auto it = c->my_list.find(&gc);\n if (it != c->my_list.end()) {\n c->my_list.erase(it);\n return true;\n }\n return false;\n }\n\npublic:\n\n static void create(d1::global_control& gc) {\n __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr);\n control_storage* const c = controls[gc.my_param];\n\n spin_mutex::scoped_lock lock(c->my_list_mutex);\n if (c->my_list.empty() || c->is_first_arg_preferred(gc.my_value, c->my_active_value)) {\n // to guarantee that apply_active() is called with current active value,\n // calls it here and in internal_destroy() under my_list_mutex\n c->apply_active(gc.my_value);\n }\n c->my_list.insert(&gc);\n }\n\n static void destroy(d1::global_control& gc) {\n __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr);\n control_storage* const c = controls[gc.my_param];\n // Concurrent reading and changing global parameter is possible.\n spin_mutex::scoped_lock lock(c->my_list_mutex);\n __TBB_ASSERT(gc.my_param == d1::global_control::scheduler_handle || !c->my_list.empty(), nullptr);\n std::size_t new_active = (std::size_t)(-1), old_active = c->my_active_value;\n\n if (!erase_if_present(c, gc)) {\n __TBB_ASSERT(gc.my_param == d1::global_control::scheduler_handle , nullptr);\n return;\n }\n if (c->my_list.empty()) {\n __TBB_ASSERT(new_active == (std::size_t) - 1, nullptr);\n new_active = c->default_value();\n } else {\n new_active = (*c->my_list.begin())->my_value;\n }\n if (new_active != old_active) {\n c->apply_active(new_active);\n }\n }\n\n static bool remove_and_check_if_empty(d1::global_control& gc) {\n __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr);\n control_storage* const c = controls[gc.my_param];\n\n spin_mutex::scoped_lock lock(c->my_list_mutex);\n __TBB_ASSERT(!c->my_list.empty(), nullptr);\n erase_if_present(c, gc);\n return c->my_list.empty();\n }\n#if TBB_USE_ASSERT\n static bool is_present(d1::global_control& gc) {\n __TBB_ASSERT_RELEASE(gc.my_param < d1::global_control::parameter_max, nullptr);\n control_storage* const c = controls[gc.my_param];\n\n spin_mutex::scoped_lock lock(c->my_list_mutex);\n auto it = c->my_list.find(&gc);\n if (it != c->my_list.end()) {\n return true;\n }\n return false;\n }\n#endif // TBB_USE_ASSERT\n};\n\nvoid __TBB_EXPORTED_FUNC create(d1::global_control& gc) {\n global_control_impl::create(gc);\n}\nvoid __TBB_EXPORTED_FUNC destroy(d1::global_control& gc) {\n global_control_impl::destroy(gc);\n}\n\nbool remove_and_check_if_empty(d1::global_control& gc) {\n return global_control_impl::remove_and_check_if_empty(gc);\n}\n#if TBB_USE_ASSERT\nbool is_present(d1::global_control& gc) {\n return global_control_impl::is_present(gc);\n}\n#endif // TBB_USE_ASSERT\nstd::size_t __TBB_EXPORTED_FUNC global_control_active_value(int param) {\n __TBB_ASSERT_RELEASE(param < d1::global_control::parameter_max, nullptr);\n return controls[param]->active_value();\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/governor.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"governor.h\"\n#include \"threading_control.h\"\n#include \"main.h\"\n#include \"thread_data.h\"\n#include \"market.h\"\n#include \"arena.h\"\n#include \"dynamic_link.h\"\n#include \"concurrent_monitor.h\"\n#include \"thread_dispatcher.h\"\n\n#include \"oneapi/tbb/task_group.h\"\n#include \"oneapi/tbb/global_control.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/info.h\"\n\n#include \"task_dispatcher.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#ifdef EMSCRIPTEN\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if TBB_USE_ASSERT\nstd::atomic the_observer_proxy_count;\n#endif /* TBB_USE_ASSERT */\n\nvoid clear_address_waiter_table();\nvoid global_control_acquire();\nvoid global_control_release();\n\n//! global_control.cpp contains definition\nbool remove_and_check_if_empty(d1::global_control& gc);\nbool is_present(d1::global_control& gc);\n\nnamespace rml {\ntbb_server* make_private_server( tbb_client& client );\n} // namespace rml\n\nnamespace system_topology {\n void destroy();\n}\n\n//------------------------------------------------------------------------\n// governor\n//------------------------------------------------------------------------\n\nvoid governor::acquire_resources () {\n global_control_acquire();\n#if __TBB_USE_POSIX\n int status = theTLS.create(auto_terminate);\n#else\n int status = theTLS.create();\n#endif\n if( status )\n handle_perror(status, \"TBB failed to initialize task scheduler TLS\\n\");\n detect_cpu_features(cpu_features);\n\n is_rethrow_broken = gcc_rethrow_exception_broken();\n}\n\nvoid governor::release_resources () {\n theRMLServerFactory.close();\n destroy_process_mask();\n\n __TBB_ASSERT(!(__TBB_InitOnce::initialization_done() && theTLS.get()), \"TBB is unloaded while thread data still alive?\");\n\n int status = theTLS.destroy();\n if( status )\n runtime_warning(\"failed to destroy task scheduler TLS: %s\", std::strerror(status));\n clear_address_waiter_table();\n\n#if TBB_USE_ASSERT\n if (the_observer_proxy_count != 0) {\n runtime_warning(\"Leaked %ld observer_proxy objects\\n\", long(the_observer_proxy_count));\n }\n#endif /* TBB_USE_ASSERT */\n\n system_topology::destroy();\n dynamic_unlink_all();\n global_control_release();\n}\n\nrml::tbb_server* governor::create_rml_server ( rml::tbb_client& client ) {\n rml::tbb_server* server = nullptr;\n if( !UsePrivateRML ) {\n ::rml::factory::status_type status = theRMLServerFactory.make_server( server, client );\n if( status != ::rml::factory::st_success ) {\n UsePrivateRML = true;\n runtime_warning( \"rml::tbb_factory::make_server failed with status %x, falling back on private rml\", status );\n }\n }\n if ( !server ) {\n __TBB_ASSERT( UsePrivateRML, nullptr);\n server = rml::make_private_server( client );\n }\n __TBB_ASSERT( server, \"Failed to create RML server\" );\n return server;\n}\n\nvoid governor::one_time_init() {\n if ( !__TBB_InitOnce::initialization_done() ) {\n DoOneTimeInitialization();\n }\n}\n\nbool governor::does_client_join_workers(const rml::tbb_client &client) {\n return ((const thread_dispatcher&)client).must_join_workers();\n}\n\n/*\n There is no portable way to get stack base address in Posix, however the modern\n Linux versions provide pthread_attr_np API that can be used to obtain thread's\n stack size and base address. Unfortunately even this function does not provide\n enough information for the main thread on IA-64 architecture (RSE spill area\n and memory stack are allocated as two separate discontinuous chunks of memory),\n and there is no portable way to discern the main and the secondary threads.\n Thus for macOS* and IA-64 architecture for Linux* OS we use the TBB worker stack size for\n all threads and use the current stack top as the stack base. This simplified\n approach is based on the following assumptions:\n 1) If the default stack size is insufficient for the user app needs, the\n required amount will be explicitly specified by the user at the point of the\n TBB scheduler initialization (as an argument to tbb::task_scheduler_init\n constructor).\n 2) When an external thread initializes the scheduler, it has enough space on its\n stack. Here \"enough\" means \"at least as much as worker threads have\".\n 3) If the user app strives to conserve the memory by cutting stack size, it\n should do this for TBB workers too (as in the #1).\n*/\nstatic std::uintptr_t get_stack_base(std::size_t stack_size) {\n // Stacks are growing top-down. Highest address is called \"stack base\",\n // and the lowest is \"stack limit\".\n#if __TBB_USE_WINAPI\n suppress_unused_warning(stack_size);\n NT_TIB* pteb = (NT_TIB*)NtCurrentTeb();\n __TBB_ASSERT(&pteb < pteb->StackBase && &pteb > pteb->StackLimit, \"invalid stack info in TEB\");\n return reinterpret_cast(pteb->StackBase);\n#elif defined(EMSCRIPTEN)\n suppress_unused_warning(stack_size);\n return reinterpret_cast(emscripten_stack_get_base());\n#else\n // There is no portable way to get stack base address in Posix, so we use\n // non-portable method (on all modern Linux) or the simplified approach\n // based on the common sense assumptions. The most important assumption\n // is that the main thread's stack size is not less than that of other threads.\n\n // Points to the lowest addressable byte of a stack.\n void* stack_limit = nullptr;\n#if __linux__ && !__bg__\n size_t np_stack_size = 0;\n pthread_attr_t np_attr_stack;\n if (0 == pthread_getattr_np(pthread_self(), &np_attr_stack)) {\n if (0 == pthread_attr_getstack(&np_attr_stack, &stack_limit, &np_stack_size)) {\n __TBB_ASSERT( &stack_limit > stack_limit, \"stack size must be positive\" );\n }\n pthread_attr_destroy(&np_attr_stack);\n }\n#endif /* __linux__ */\n std::uintptr_t stack_base{};\n if (stack_limit) {\n stack_base = reinterpret_cast(stack_limit) + stack_size;\n } else {\n // Use an anchor as a base stack address.\n int anchor{};\n stack_base = reinterpret_cast(&anchor);\n }\n return stack_base;\n#endif /* __TBB_USE_WINAPI */\n}\n\n#if (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED\nstatic void register_external_thread_destructor() {\n struct thread_destructor {\n ~thread_destructor() {\n governor::terminate_external_thread();\n }\n };\n // ~thread_destructor() will be call during the calling thread termination\n static thread_local thread_destructor thr_destructor;\n}\n#endif // (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED\n\nvoid governor::init_external_thread() {\n one_time_init();\n // Create new scheduler instance with arena\n int num_slots = default_num_threads();\n // TODO_REVAMP: support an external thread without an implicit arena\n int num_reserved_slots = 1;\n unsigned arena_priority_level = 1; // corresponds to tbb::task_arena::priority::normal\n std::size_t stack_size = 0;\n threading_control* thr_control = threading_control::register_public_reference();\n arena& a = arena::create(thr_control, num_slots, num_reserved_slots, arena_priority_level);\n // External thread always occupies the first slot\n thread_data& td = *new(cache_aligned_allocate(sizeof(thread_data))) thread_data(0, false);\n td.attach_arena(a, /*slot index*/ 0);\n __TBB_ASSERT(td.my_inbox.is_idle_state(false), nullptr);\n\n stack_size = a.my_threading_control->worker_stack_size();\n std::uintptr_t stack_base = get_stack_base(stack_size);\n task_dispatcher& task_disp = td.my_arena_slot->default_task_dispatcher();\n td.enter_task_dispatcher(task_disp, calculate_stealing_threshold(stack_base, stack_size));\n\n td.my_arena_slot->occupy();\n thr_control->register_thread(td);\n set_thread_data(td);\n#if (_WIN32||_WIN64) && !__TBB_DYNAMIC_LOAD_ENABLED\n // The external thread destructor is called from dllMain but it is not available with a static build.\n // Therefore, we need to register the current thread to call the destructor during thread termination.\n register_external_thread_destructor();\n#endif\n}\n\nvoid governor::auto_terminate(void* tls) {\n __TBB_ASSERT(get_thread_data_if_initialized() == nullptr ||\n get_thread_data_if_initialized() == tls, nullptr);\n if (tls) {\n thread_data* td = static_cast(tls);\n\n auto clear_tls = [td] {\n td->~thread_data();\n cache_aligned_deallocate(td);\n clear_thread_data();\n };\n\n // Only external thread can be inside an arena during termination.\n if (td->my_arena_slot) {\n arena* a = td->my_arena;\n threading_control* thr_control = a->my_threading_control;\n\n // If the TLS slot is already cleared by OS or underlying concurrency\n // runtime, restore its value to properly clean up arena\n if (!is_thread_data_set(td)) {\n set_thread_data(*td);\n }\n\n a->my_observers.notify_exit_observers(td->my_last_observer, td->my_is_worker);\n\n td->leave_task_dispatcher();\n td->my_arena_slot->release();\n // Release an arena\n a->on_thread_leaving(arena::ref_external);\n\n thr_control->unregister_thread(*td);\n\n // The tls should be cleared before market::release because\n // market can destroy the tls key if we keep the last reference\n clear_tls();\n\n // If there was an associated arena, it added a public market reference\n thr_control->unregister_public_reference(/* blocking terminate =*/ false);\n } else {\n clear_tls();\n }\n }\n __TBB_ASSERT(get_thread_data_if_initialized() == nullptr, nullptr);\n}\n\nvoid governor::initialize_rml_factory () {\n ::rml::factory::status_type res = theRMLServerFactory.open();\n UsePrivateRML = res != ::rml::factory::st_success;\n}\n\nvoid __TBB_EXPORTED_FUNC get(d1::task_scheduler_handle& handle) {\n handle.m_ctl = new(allocate_memory(sizeof(global_control))) global_control(global_control::scheduler_handle, 1);\n}\n\nvoid release_impl(d1::task_scheduler_handle& handle) {\n if (handle.m_ctl != nullptr) {\n handle.m_ctl->~global_control();\n deallocate_memory(handle.m_ctl);\n handle.m_ctl = nullptr;\n }\n}\n\nbool finalize_impl(d1::task_scheduler_handle& handle) {\n __TBB_ASSERT_RELEASE(handle, \"trying to finalize with null handle\");\n __TBB_ASSERT(is_present(*handle.m_ctl), \"finalize or release was already called on this object\");\n\n bool ok = true; // ok if threading_control does not exist yet\n if (threading_control::is_present()) {\n thread_data* td = governor::get_thread_data_if_initialized();\n if (td) {\n task_dispatcher* task_disp = td->my_task_dispatcher;\n __TBB_ASSERT(task_disp, nullptr);\n if (task_disp->m_properties.outermost && !td->my_is_worker) { // is not inside a parallel region\n governor::auto_terminate(td);\n }\n }\n\n if (remove_and_check_if_empty(*handle.m_ctl)) {\n ok = threading_control::unregister_lifetime_control(/*blocking_terminate*/ true);\n } else {\n ok = false;\n }\n }\n\n return ok;\n}\n\nbool __TBB_EXPORTED_FUNC finalize(d1::task_scheduler_handle& handle, std::intptr_t mode) {\n if (mode == d1::release_nothrowing) {\n release_impl(handle);\n return true;\n } else {\n bool ok = finalize_impl(handle);\n // TODO: it is unsafe when finalize is called concurrently and further library unload\n release_impl(handle);\n if (mode == d1::finalize_throwing && !ok) {\n throw_exception(exception_id::unsafe_wait);\n }\n return ok;\n }\n}\n\n#if __TBB_ARENA_BINDING\n\n#if __TBB_WEAK_SYMBOLS_PRESENT\n#pragma weak __TBB_internal_initialize_system_topology\n#pragma weak __TBB_internal_destroy_system_topology\n#pragma weak __TBB_internal_allocate_binding_handler\n#pragma weak __TBB_internal_deallocate_binding_handler\n#pragma weak __TBB_internal_apply_affinity\n#pragma weak __TBB_internal_restore_affinity\n#pragma weak __TBB_internal_get_default_concurrency\n\nextern \"C\" {\nvoid __TBB_internal_initialize_system_topology(\n size_t groups_num,\n int& numa_nodes_count, int*& numa_indexes_list,\n int& core_types_count, int*& core_types_indexes_list\n);\nvoid __TBB_internal_destroy_system_topology( );\n\n//TODO: consider renaming to `create_binding_handler` and `destroy_binding_handler`\nbinding_handler* __TBB_internal_allocate_binding_handler( int slot_num, int numa_id, int core_type_id, int max_threads_per_core );\nvoid __TBB_internal_deallocate_binding_handler( binding_handler* handler_ptr );\n\nvoid __TBB_internal_apply_affinity( binding_handler* handler_ptr, int slot_num );\nvoid __TBB_internal_restore_affinity( binding_handler* handler_ptr, int slot_num );\n\nint __TBB_internal_get_default_concurrency( int numa_id, int core_type_id, int max_threads_per_core );\n}\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\n// Stubs that will be used if TBBbind library is unavailable.\nstatic void dummy_destroy_system_topology ( ) { }\nstatic binding_handler* dummy_allocate_binding_handler ( int, int, int, int ) { return nullptr; }\nstatic void dummy_deallocate_binding_handler ( binding_handler* ) { }\nstatic void dummy_apply_affinity ( binding_handler*, int ) { }\nstatic void dummy_restore_affinity ( binding_handler*, int ) { }\nstatic int dummy_get_default_concurrency( int, int, int ) { return governor::default_num_threads(); }\n\n// Handlers for communication with TBBbind\nstatic void (*initialize_system_topology_ptr)(\n size_t groups_num,\n int& numa_nodes_count, int*& numa_indexes_list,\n int& core_types_count, int*& core_types_indexes_list\n) = nullptr;\nstatic void (*destroy_system_topology_ptr)( ) = dummy_destroy_system_topology;\n\nstatic binding_handler* (*allocate_binding_handler_ptr)( int slot_num, int numa_id, int core_type_id, int max_threads_per_core )\n = dummy_allocate_binding_handler;\nstatic void (*deallocate_binding_handler_ptr)( binding_handler* handler_ptr )\n = dummy_deallocate_binding_handler;\nstatic void (*apply_affinity_ptr)( binding_handler* handler_ptr, int slot_num )\n = dummy_apply_affinity;\nstatic void (*restore_affinity_ptr)( binding_handler* handler_ptr, int slot_num )\n = dummy_restore_affinity;\nint (*get_default_concurrency_ptr)( int numa_id, int core_type_id, int max_threads_per_core )\n = dummy_get_default_concurrency;\n\n#if _WIN32 || _WIN64 || __unix__ || __APPLE__\n\n// Table describing how to link the handlers.\nstatic const dynamic_link_descriptor TbbBindLinkTable[] = {\n DLD(__TBB_internal_initialize_system_topology, initialize_system_topology_ptr),\n DLD(__TBB_internal_destroy_system_topology, destroy_system_topology_ptr),\n#if __TBB_CPUBIND_PRESENT\n DLD(__TBB_internal_allocate_binding_handler, allocate_binding_handler_ptr),\n DLD(__TBB_internal_deallocate_binding_handler, deallocate_binding_handler_ptr),\n DLD(__TBB_internal_apply_affinity, apply_affinity_ptr),\n DLD(__TBB_internal_restore_affinity, restore_affinity_ptr),\n#endif\n DLD(__TBB_internal_get_default_concurrency, get_default_concurrency_ptr)\n};\n\nstatic const unsigned LinkTableSize = sizeof(TbbBindLinkTable) / sizeof(dynamic_link_descriptor);\n\n#if TBB_USE_DEBUG\n#define DEBUG_SUFFIX \"_debug\"\n#else\n#define DEBUG_SUFFIX\n#endif /* TBB_USE_DEBUG */\n\n#if _WIN32 || _WIN64\n#define LIBRARY_EXTENSION \".dll\"\n#define LIBRARY_PREFIX\n#elif __APPLE__\n#define LIBRARY_EXTENSION __TBB_STRING(.3.dylib)\n#define LIBRARY_PREFIX \"lib\"\n#elif __unix__\n#define LIBRARY_EXTENSION __TBB_STRING(.so.3)\n#define LIBRARY_PREFIX \"lib\"\n#endif /* __unix__ */\n\n#define TBBBIND_NAME LIBRARY_PREFIX \"tbbbind\" DEBUG_SUFFIX LIBRARY_EXTENSION\n#define TBBBIND_2_0_NAME LIBRARY_PREFIX \"tbbbind_2_0\" DEBUG_SUFFIX LIBRARY_EXTENSION\n\n#define TBBBIND_2_5_NAME LIBRARY_PREFIX \"tbbbind_2_5\" DEBUG_SUFFIX LIBRARY_EXTENSION\n#endif /* _WIN32 || _WIN64 || __unix__ */\n\n// Representation of system hardware topology information on the TBB side.\n// System topology may be initialized by third-party component (e.g. hwloc)\n// or just filled in with default stubs.\nnamespace system_topology {\n\nconstexpr int automatic = -1;\n\nstatic std::atomic initialization_state;\n\nnamespace {\nint numa_nodes_count = 0;\nint* numa_nodes_indexes = nullptr;\n\nint core_types_count = 0;\nint* core_types_indexes = nullptr;\n\nconst char* load_tbbbind_shared_object() {\n#if _WIN32 || _WIN64 || __unix__ || __APPLE__\n#if _WIN32 && !_WIN64\n // For 32-bit Windows applications, process affinity masks can only support up to 32 logical CPUs.\n SYSTEM_INFO si;\n GetNativeSystemInfo(&si);\n if (si.dwNumberOfProcessors > 32) return nullptr;\n#endif /* _WIN32 && !_WIN64 */\n for (const auto& tbbbind_version : {TBBBIND_2_5_NAME, TBBBIND_2_0_NAME, TBBBIND_NAME}) {\n if (dynamic_link(tbbbind_version, TbbBindLinkTable, LinkTableSize, nullptr, DYNAMIC_LINK_LOCAL_BINDING)) {\n return tbbbind_version;\n }\n }\n#endif /* _WIN32 || _WIN64 || __unix__ || __APPLE__ */\n return nullptr;\n}\n\nint processor_groups_num() {\n#if _WIN32\n return NumberOfProcessorGroups();\n#else\n // Stub to improve code readability by reducing number of the compile-time conditions\n return 1;\n#endif\n}\n} // internal namespace\n\n// Tries to load TBBbind library API, if success, gets NUMA topology information from it,\n// in another case, fills NUMA topology by stubs.\nvoid initialization_impl() {\n governor::one_time_init();\n\n if (const char* tbbbind_name = load_tbbbind_shared_object()) {\n initialize_system_topology_ptr(\n processor_groups_num(),\n numa_nodes_count, numa_nodes_indexes,\n core_types_count, core_types_indexes\n );\n\n PrintExtraVersionInfo(\"TBBBIND\", tbbbind_name);\n return;\n }\n\n static int dummy_index = automatic;\n\n numa_nodes_count = 1;\n numa_nodes_indexes = &dummy_index;\n\n core_types_count = 1;\n core_types_indexes = &dummy_index;\n\n PrintExtraVersionInfo(\"TBBBIND\", \"UNAVAILABLE\");\n}\n\nvoid initialize() {\n atomic_do_once(initialization_impl, initialization_state);\n}\n\nvoid destroy() {\n destroy_system_topology_ptr();\n}\n} // namespace system_topology\n\nbinding_handler* construct_binding_handler(int slot_num, int numa_id, int core_type_id, int max_threads_per_core) {\n system_topology::initialize();\n return allocate_binding_handler_ptr(slot_num, numa_id, core_type_id, max_threads_per_core);\n}\n\nvoid destroy_binding_handler(binding_handler* handler_ptr) {\n __TBB_ASSERT(deallocate_binding_handler_ptr, \"tbbbind loading was not performed\");\n deallocate_binding_handler_ptr(handler_ptr);\n}\n\nvoid apply_affinity_mask(binding_handler* handler_ptr, int slot_index) {\n __TBB_ASSERT(slot_index >= 0, \"Negative thread index\");\n __TBB_ASSERT(apply_affinity_ptr, \"tbbbind loading was not performed\");\n apply_affinity_ptr(handler_ptr, slot_index);\n}\n\nvoid restore_affinity_mask(binding_handler* handler_ptr, int slot_index) {\n __TBB_ASSERT(slot_index >= 0, \"Negative thread index\");\n __TBB_ASSERT(restore_affinity_ptr, \"tbbbind loading was not performed\");\n restore_affinity_ptr(handler_ptr, slot_index);\n}\n\nunsigned __TBB_EXPORTED_FUNC numa_node_count() {\n system_topology::initialize();\n return system_topology::numa_nodes_count;\n}\n\nvoid __TBB_EXPORTED_FUNC fill_numa_indices(int* index_array) {\n system_topology::initialize();\n std::memcpy(index_array, system_topology::numa_nodes_indexes, system_topology::numa_nodes_count * sizeof(int));\n}\n\nint __TBB_EXPORTED_FUNC numa_default_concurrency(int node_id) {\n if (node_id >= 0) {\n system_topology::initialize();\n int result = get_default_concurrency_ptr(\n node_id,\n /*core_type*/system_topology::automatic,\n /*threads_per_core*/system_topology::automatic\n );\n if (result > 0) return result;\n }\n return governor::default_num_threads();\n}\n\nunsigned __TBB_EXPORTED_FUNC core_type_count(intptr_t /*reserved*/) {\n system_topology::initialize();\n return system_topology::core_types_count;\n}\n\nvoid __TBB_EXPORTED_FUNC fill_core_type_indices(int* index_array, intptr_t /*reserved*/) {\n system_topology::initialize();\n std::memcpy(index_array, system_topology::core_types_indexes, system_topology::core_types_count * sizeof(int));\n}\n\nvoid constraints_assertion(d1::constraints c) {\n bool is_topology_initialized = system_topology::initialization_state == do_once_state::initialized;\n __TBB_ASSERT_RELEASE(c.max_threads_per_core == system_topology::automatic || c.max_threads_per_core > 0,\n \"Wrong max_threads_per_core constraints field value.\");\n\n auto numa_nodes_begin = system_topology::numa_nodes_indexes;\n auto numa_nodes_end = system_topology::numa_nodes_indexes + system_topology::numa_nodes_count;\n __TBB_ASSERT_RELEASE(\n c.numa_id == system_topology::automatic ||\n (is_topology_initialized && std::find(numa_nodes_begin, numa_nodes_end, c.numa_id) != numa_nodes_end),\n \"The constraints::numa_id value is not known to the library. Use tbb::info::numa_nodes() to get the list of possible values.\");\n\n int* core_types_begin = system_topology::core_types_indexes;\n int* core_types_end = system_topology::core_types_indexes + system_topology::core_types_count;\n __TBB_ASSERT_RELEASE(c.core_type == system_topology::automatic ||\n (is_topology_initialized && std::find(core_types_begin, core_types_end, c.core_type) != core_types_end),\n \"The constraints::core_type value is not known to the library. Use tbb::info::core_types() to get the list of possible values.\");\n}\n\nint __TBB_EXPORTED_FUNC constraints_default_concurrency(const d1::constraints& c, intptr_t /*reserved*/) {\n constraints_assertion(c);\n\n if (c.numa_id >= 0 || c.core_type >= 0 || c.max_threads_per_core > 0) {\n system_topology::initialize();\n return get_default_concurrency_ptr(c.numa_id, c.core_type, c.max_threads_per_core);\n }\n return governor::default_num_threads();\n}\n\nint __TBB_EXPORTED_FUNC constraints_threads_per_core(const d1::constraints&, intptr_t /*reserved*/) {\n return system_topology::automatic;\n}\n#endif /* __TBB_ARENA_BINDING */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/governor.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_governor_H\n#define _TBB_governor_H\n\n#include \"rml_tbb.h\"\n\n#include \"misc.h\" // for AvailableHwConcurrency\n#include \"tls.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass market;\nclass thread_data;\nclass __TBB_InitOnce;\n\n#if __TBB_USE_ITT_NOTIFY\n//! Defined in profiling.cpp\nextern bool ITT_Present;\n#endif\n\ntypedef std::size_t stack_size_type;\n\n//------------------------------------------------------------------------\n// Class governor\n//------------------------------------------------------------------------\n\n//! The class handles access to the single instance of market, and to TLS to keep scheduler instances.\n/** It also supports automatic on-demand initialization of the TBB scheduler.\n The class contains only static data members and methods.*/\nclass governor {\nprivate:\n friend class __TBB_InitOnce;\n friend class thread_dispatcher;\n friend class threading_control_impl;\n\n // TODO: consider using thread_local (measure performance and side effects)\n //! TLS for scheduler instances associated with individual threads\n static basic_tls theTLS;\n\n // TODO (TBB_REVAMP_TODO): reconsider constant names\n static rml::tbb_factory theRMLServerFactory;\n\n static bool UsePrivateRML;\n\n // Flags for runtime-specific conditions\n static cpu_features_type cpu_features;\n static bool is_rethrow_broken;\n\n //! Create key for thread-local storage and initialize RML.\n static void acquire_resources ();\n\n //! Destroy the thread-local storage key and deinitialize RML.\n static void release_resources ();\n\n static rml::tbb_server* create_rml_server ( rml::tbb_client& );\n\npublic:\n static unsigned default_num_threads () {\n // Caches the maximal level of parallelism supported by the hardware\n static unsigned num_threads = AvailableHwConcurrency();\n return num_threads;\n }\n static std::size_t default_page_size () {\n // Caches the size of OS regular memory page\n static std::size_t page_size = DefaultSystemPageSize();\n return page_size;\n }\n static void one_time_init();\n //! Processes scheduler initialization request (possibly nested) in an external thread\n /** If necessary creates new instance of arena and/or local scheduler.\n The auto_init argument specifies if the call is due to automatic initialization. **/\n static void init_external_thread();\n\n //! The routine to undo automatic initialization.\n /** The signature is written with void* so that the routine\n can be the destructor argument to pthread_key_create. */\n static void auto_terminate(void* tls);\n\n //! Obtain the thread-local instance of the thread data.\n /** If the scheduler has not been initialized yet, initialization is done automatically.\n Note that auto-initialized scheduler instance is destroyed only when its thread terminates. **/\n static thread_data* get_thread_data() {\n thread_data* td = theTLS.get();\n if (td) {\n return td;\n }\n init_external_thread();\n td = theTLS.get();\n __TBB_ASSERT(td, nullptr);\n return td;\n }\n\n static void set_thread_data(thread_data& td) {\n theTLS.set(&td);\n }\n\n static void clear_thread_data() {\n theTLS.set(nullptr);\n }\n\n static thread_data* get_thread_data_if_initialized () {\n return theTLS.get();\n }\n\n static bool is_thread_data_set(thread_data* td) {\n return theTLS.get() == td;\n }\n\n //! Undo automatic initialization if necessary; call when a thread exits.\n static void terminate_external_thread() {\n auto_terminate(get_thread_data_if_initialized());\n }\n\n static void initialize_rml_factory ();\n\n static bool does_client_join_workers (const rml::tbb_client &client);\n\n static bool speculation_enabled() { return cpu_features.rtm_enabled; }\n\n#if __TBB_WAITPKG_INTRINSICS_PRESENT\n static bool wait_package_enabled() { return cpu_features.waitpkg_enabled; }\n#endif\n\n static bool hybrid_cpu() { return cpu_features.hybrid; }\n\n static bool rethrow_exception_broken() { return is_rethrow_broken; }\n\n static bool is_itt_present() {\n#if __TBB_USE_ITT_NOTIFY\n return ITT_Present;\n#else\n return false;\n#endif\n }\n}; // class governor\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_governor_H */\n" }, { "path": "src/tbb/src/tbb/intrusive_list.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_intrusive_list_H\n#define _TBB_intrusive_list_H\n\n#include \"oneapi/tbb/detail/_intrusive_list_node.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nusing d1::intrusive_list_node;\n\n//! List of element of type T, where T is derived from intrusive_list_node\n/** The class is not thread safe. **/\ntemplate \nclass intrusive_list_base {\n //! Pointer to the head node\n intrusive_list_node my_head;\n\n //! Number of list elements\n std::size_t my_size;\n\n static intrusive_list_node& node ( T& item ) { return List::node(item); }\n\n static T& item ( intrusive_list_node* node ) { return List::item(node); }\n\n static const T& item( const intrusive_list_node* node ) { return List::item(node); }\n\n template \n class iterator_impl {\n static_assert(std::is_same::value ||\n std::is_same::value,\n \"Incorrect DereferenceType in iterator_impl\");\n\n using pointer_type = typename std::conditional::value,\n intrusive_list_node*,\n const intrusive_list_node*>::type;\n\n public:\n iterator_impl() : my_pos(nullptr) {}\n\n iterator_impl( pointer_type pos ) : my_pos(pos) {}\n\n iterator_impl& operator++() {\n my_pos = my_pos->my_next_node;\n return *this;\n }\n\n iterator_impl operator++( int ) {\n iterator_impl it(*this);\n ++*this;\n return it;\n }\n\n iterator_impl& operator--() {\n my_pos = my_pos->my_prev_node;\n return *this;\n }\n\n iterator_impl operator--( int ) {\n iterator_impl it(*this);\n --*this;\n return it;\n }\n\n bool operator==( const iterator_impl& rhs ) const {\n return my_pos == rhs.my_pos;\n }\n\n bool operator!=( const iterator_impl& rhs ) const {\n return my_pos != rhs.my_pos;\n }\n\n DereferenceType& operator*() const {\n return intrusive_list_base::item(my_pos);\n }\n\n DereferenceType* operator->() const {\n return &intrusive_list_base::item(my_pos);\n }\n private:\n // Node the iterator points to at the moment\n pointer_type my_pos;\n }; // class iterator_impl\n\n void assert_ok () const {\n __TBB_ASSERT( (my_head.my_prev_node == &my_head && !my_size) ||\n (my_head.my_next_node != &my_head && my_size >0), \"intrusive_list_base corrupted\" );\n#if TBB_USE_ASSERT >= 2\n std::size_t i = 0;\n for ( intrusive_list_node *n = my_head.my_next_node; n != &my_head; n = n->my_next_node )\n ++i;\n __TBB_ASSERT( my_size == i, \"Wrong size\" );\n#endif /* TBB_USE_ASSERT >= 2 */\n }\n\npublic:\n using iterator = iterator_impl;\n using const_iterator = iterator_impl;\n\n intrusive_list_base () : my_size(0) {\n my_head.my_prev_node = &my_head;\n my_head.my_next_node = &my_head;\n }\n\n bool empty () const { return my_head.my_next_node == &my_head; }\n\n std::size_t size () const { return my_size; }\n\n iterator begin () { return iterator(my_head.my_next_node); }\n\n iterator end () { return iterator(&my_head); }\n\n const_iterator begin () const { return const_iterator(my_head.my_next_node); }\n\n const_iterator end () const { return const_iterator(&my_head); }\n\n void push_front ( T& val ) {\n __TBB_ASSERT( node(val).my_prev_node == &node(val) && node(val).my_next_node == &node(val),\n \"Object with intrusive list node can be part of only one intrusive list simultaneously\" );\n // An object can be part of only one intrusive list at the given moment via the given node member\n node(val).my_prev_node = &my_head;\n node(val).my_next_node = my_head.my_next_node;\n my_head.my_next_node->my_prev_node = &node(val);\n my_head.my_next_node = &node(val);\n ++my_size;\n assert_ok();\n }\n\n void remove( T& val ) {\n __TBB_ASSERT( node(val).my_prev_node != &node(val) && node(val).my_next_node != &node(val), \"Element to remove is not in the list\" );\n __TBB_ASSERT( node(val).my_prev_node->my_next_node == &node(val) && node(val).my_next_node->my_prev_node == &node(val), \"Element to remove is not in the list\" );\n --my_size;\n node(val).my_next_node->my_prev_node = node(val).my_prev_node;\n node(val).my_prev_node->my_next_node = node(val).my_next_node;\n#if TBB_USE_ASSERT\n node(val).my_prev_node = node(val).my_next_node = &node(val);\n#endif\n assert_ok();\n }\n\n iterator erase ( iterator it ) {\n T& val = *it;\n ++it;\n remove( val );\n return it;\n }\n\n}; // intrusive_list_base\n\n#if __TBB_TODO\n// With standard compliant compilers memptr_intrusive_list could be named simply intrusive_list,\n// and inheritance based intrusive_list version would become its partial specialization.\n// Here are the corresponding declarations:\n\nstruct dummy_intrusive_list_item { intrusive_list_node my_node; };\n\ntemplate \nclass intrusive_list : public intrusive_list_base, T>;\n\ntemplate \nclass intrusive_list\n : public intrusive_list_base, T>;\n\n#endif /* __TBB_TODO */\n\n//! Double linked list of items of type T containing a member of type intrusive_list_node.\n/** NodePtr is a member pointer to the node data field. Class U is either T or\n a base class of T containing the node member. Default values exist for the sake\n of a partial specialization working with inheritance case.\n\n The list does not have ownership of its items. Its purpose is to avoid dynamic\n memory allocation when forming lists of existing objects.\n\n The class is not thread safe. **/\ntemplate \nclass memptr_intrusive_list : public intrusive_list_base, T>\n{\n friend class intrusive_list_base, T>;\n\n static intrusive_list_node& node ( T& val ) { return val.*NodePtr; }\n\n static T& item ( intrusive_list_node* node ) {\n // Cannot use __TBB_offsetof (and consequently __TBB_get_object_ref) macro\n // with *NodePtr argument because gcc refuses to interpret pasted \"->\" and \"*\"\n // as member pointer dereferencing operator, and explicit usage of ## in\n // __TBB_offsetof implementation breaks operations with normal member names.\n return *reinterpret_cast((char*)node - ((ptrdiff_t)&(reinterpret_cast(0x1000)->*NodePtr) - 0x1000));\n }\n\n static const T& item( const intrusive_list_node* node ) {\n return item(const_cast(node));\n }\n\n}; // intrusive_list\n\n//! Double linked list of items of type T that is derived from intrusive_list_node class.\n/** The list does not have ownership of its items. Its purpose is to avoid dynamic\n memory allocation when forming lists of existing objects.\n\n The class is not thread safe. **/\ntemplate \nclass intrusive_list : public intrusive_list_base, T>\n{\n friend class intrusive_list_base, T>;\n\n static intrusive_list_node& node ( T& val ) { return val; }\n\n static T& item ( intrusive_list_node* node ) { return *static_cast(node); }\n\n static const T& item( const intrusive_list_node* node ) { return *static_cast(node); }\n}; // intrusive_list\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_intrusive_list_H */\n" }, { "path": "src/tbb/src/tbb/itt_notify.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#if __TBB_USE_ITT_NOTIFY\n\n#if _WIN32||_WIN64\n #ifndef UNICODE\n #define UNICODE\n #endif\n#else\n #pragma weak dlopen\n #pragma weak dlsym\n #pragma weak dlerror\n#endif /* WIN */\n\n#if __TBB_BUILD\n\nextern \"C\" void ITT_DoOneTimeInitialization();\n#define __itt_init_ittlib_name(x,y) (ITT_DoOneTimeInitialization(), true)\n\n#elif __TBBMALLOC_BUILD\n\nextern \"C\" void MallocInitializeITT();\n#define __itt_init_ittlib_name(x,y) (MallocInitializeITT(), true)\n\n#else\n#error This file is expected to be used for either TBB or TBB allocator build.\n#endif // __TBB_BUILD\n\n#include \"tools_api/ittnotify_static.c\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n/** This extra proxy method is necessary since __itt_init_lib is declared as static **/\nint __TBB_load_ittnotify() {\n#if !(_WIN32||_WIN64)\n // tool_api crashes without dlopen, check that it's present. Common case\n // for lack of dlopen is static binaries, i.e. ones build with -static.\n if (dlopen == nullptr)\n return 0;\n#endif\n return __itt_init_ittlib(nullptr, // groups for:\n (__itt_group_id)(__itt_group_sync // prepare/cancel/acquired/releasing\n | __itt_group_thread // name threads\n | __itt_group_stitch // stack stitching\n | __itt_group_structure\n ));\n}\n\n} //namespace r1\n} //namespace detail\n} // namespace tbb\n\n#endif /* __TBB_USE_ITT_NOTIFY */\n" }, { "path": "src/tbb/src/tbb/itt_notify.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_ITT_NOTIFY\n#define _TBB_ITT_NOTIFY\n\n#include \"oneapi/tbb/detail/_config.h\"\n\n#if __TBB_USE_ITT_NOTIFY\n\n#if _WIN32||_WIN64\n #ifndef UNICODE\n #define UNICODE\n #endif\n#endif /* WIN */\n\n#ifndef INTEL_ITTNOTIFY_API_PRIVATE\n#define INTEL_ITTNOTIFY_API_PRIVATE\n#endif\n\n#include \"tools_api/ittnotify.h\"\n#include \"tools_api/legacy/ittnotify.h\"\nextern \"C\" void __itt_fini_ittlib(void);\nextern \"C\" void __itt_release_resources(void);\n\n#if _WIN32||_WIN64\n #undef _T\n#endif /* WIN */\n\n#endif /* __TBB_USE_ITT_NOTIFY */\n\n#if !ITT_CALLER_NULL\n#define ITT_CALLER_NULL ((__itt_caller)0)\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Unicode support\n#if (_WIN32||_WIN64)\n //! Unicode character type. Always wchar_t on Windows.\n /** We do not use typedefs from Windows TCHAR family to keep consistence of TBB coding style. **/\n using tchar = wchar_t;\n //! Standard Windows macro to markup the string literals.\n #define _T(string_literal) L ## string_literal\n#else /* !WIN */\n using tchar = char;\n //! Standard Windows style macro to markup the string literals.\n #define _T(string_literal) string_literal\n#endif /* !WIN */\n\n//! Display names of internal synchronization types\nextern const tchar\n *SyncType_Scheduler;\n//! Display names of internal synchronization components/scenarios\nextern const tchar\n *SyncObj_ContextsList\n ;\n\n#if __TBB_USE_ITT_NOTIFY\n// const_cast() is necessary to cast off volatility\n#define ITT_NOTIFY(name,obj) __itt_##name(const_cast(static_cast(obj)))\n#define ITT_THREAD_SET_NAME(name) __itt_thread_set_name(name)\n#define ITT_FINI_ITTLIB() __itt_fini_ittlib()\n#define ITT_RELEASE_RESOURCES() __itt_release_resources()\n#define ITT_SYNC_CREATE(obj, type, name) __itt_sync_create((void*)(obj), type, name, 2)\n#define ITT_STACK_CREATE(obj) obj = __itt_stack_caller_create()\n#define ITT_STACK_DESTROY(obj) (obj!=nullptr) ? __itt_stack_caller_destroy(static_cast<__itt_caller>(obj)) : ((void)0)\n#define ITT_CALLEE_ENTER(cond, t, obj) if(cond) {\\\n __itt_stack_callee_enter(static_cast<__itt_caller>(obj));\\\n __itt_sync_acquired(t);\\\n }\n#define ITT_CALLEE_LEAVE(cond, obj) (cond) ? __itt_stack_callee_leave(static_cast<__itt_caller>(obj)) : ((void)0)\n\n#define ITT_TASK_GROUP(obj,name,parent) r1::itt_make_task_group(d1::ITT_DOMAIN_MAIN,(void*)(obj),ALGORITHM,(void*)(parent),(parent!=nullptr) ? ALGORITHM : FLOW_NULL,name)\n#define ITT_TASK_BEGIN(obj,name,id) r1::itt_task_begin(d1::ITT_DOMAIN_MAIN,(void*)(id),ALGORITHM,(void*)(obj),ALGORITHM,name)\n#define ITT_TASK_END r1::itt_task_end(d1::ITT_DOMAIN_MAIN)\n\n\n#else /* !__TBB_USE_ITT_NOTIFY */\n\n#define ITT_NOTIFY(name,obj) ((void)0)\n#define ITT_THREAD_SET_NAME(name) ((void)0)\n#define ITT_FINI_ITTLIB() ((void)0)\n#define ITT_RELEASE_RESOURCES() ((void)0)\n#define ITT_SYNC_CREATE(obj, type, name) ((void)0)\n#define ITT_STACK_CREATE(obj) ((void)0)\n#define ITT_STACK_DESTROY(obj) ((void)0)\n#define ITT_CALLEE_ENTER(cond, t, obj) ((void)0)\n#define ITT_CALLEE_LEAVE(cond, obj) ((void)0)\n#define ITT_TASK_GROUP(type,name,parent) ((void)0)\n#define ITT_TASK_BEGIN(type,name,id) ((void)0)\n#define ITT_TASK_END ((void)0)\n\n#endif /* !__TBB_USE_ITT_NOTIFY */\n\nint __TBB_load_ittnotify();\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_ITT_NOTIFY */\n" }, { "path": "src/tbb/src/tbb/mailbox.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_mailbox_H\n#define _TBB_mailbox_H\n\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/detail/_small_object_pool.h\"\n\n#include \"scheduler_common.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct task_proxy : public d1::task {\n static const intptr_t pool_bit = 1<<0;\n static const intptr_t mailbox_bit = 1<<1;\n static const intptr_t location_mask = pool_bit | mailbox_bit;\n /* All but two low-order bits represent a (task*).\n Two low-order bits mean:\n 1 = proxy is/was/will be in task pool\n 2 = proxy is/was/will be in mailbox */\n std::atomic task_and_tag;\n\n //! Pointer to next task_proxy in a mailbox\n std::atomic next_in_mailbox;\n\n //! Mailbox to which this was mailed.\n mail_outbox* outbox;\n\n //! Task affinity id which is referenced\n d1::slot_id slot;\n\n d1::small_object_allocator allocator;\n\n //! True if the proxy is stored both in its sender's pool and in the destination mailbox.\n static bool is_shared ( intptr_t tat ) {\n return (tat & location_mask) == location_mask;\n }\n\n //! Returns a pointer to the encapsulated task or nullptr.\n static task* task_ptr ( intptr_t tat ) {\n return (task*)(tat & ~location_mask);\n }\n\n //! Returns a pointer to the encapsulated task or nullptr, and frees proxy if necessary.\n template\n inline task* extract_task () {\n // __TBB_ASSERT( prefix().extra_state == es_task_proxy, \"Normal task misinterpreted as a proxy?\" );\n intptr_t tat = task_and_tag.load(std::memory_order_acquire);\n __TBB_ASSERT( tat == from_bit || (is_shared(tat) && task_ptr(tat)),\n \"Proxy's tag cannot specify both locations if the proxy \"\n \"was retrieved from one of its original locations\" );\n if ( tat != from_bit ) {\n const intptr_t cleaner_bit = location_mask & ~from_bit;\n // Attempt to transition the proxy to the \"empty\" state with\n // cleaner_bit specifying entity responsible for its eventual freeing.\n // Explicit cast to void* is to work around a seeming ICC 11.1 bug.\n if ( task_and_tag.compare_exchange_strong(tat, cleaner_bit) ) {\n // Successfully grabbed the task, and left new owner with the job of freeing the proxy\n return task_ptr(tat);\n }\n }\n // Proxied task has already been claimed from another proxy location.\n __TBB_ASSERT( task_and_tag.load(std::memory_order_relaxed) == from_bit, \"Empty proxy cannot contain non-zero task pointer\" );\n return nullptr;\n }\n\n task* execute(d1::execution_data&) override {\n __TBB_ASSERT_RELEASE(false, nullptr);\n return nullptr;\n }\n task* cancel(d1::execution_data&) override {\n __TBB_ASSERT_RELEASE(false, nullptr);\n return nullptr;\n }\n}; // struct task_proxy\n\n//! Internal representation of mail_outbox, without padding.\nclass unpadded_mail_outbox {\nprotected:\n typedef std::atomic atomic_proxy_ptr;\n\n //! Pointer to first task_proxy in mailbox, or nullptr if box is empty.\n atomic_proxy_ptr my_first;\n\n //! Pointer to pointer that will point to next item in the queue. Never nullptr.\n std::atomic my_last;\n\n //! Owner of mailbox is not executing a task, and has drained its own task pool.\n std::atomic my_is_idle;\n};\n\n// TODO: - consider moving to arena slot\n//! Class representing where mail is put.\n/** Padded to occupy a cache line. */\nclass mail_outbox : padded {\n\n task_proxy* internal_pop( isolation_type isolation ) {\n task_proxy* curr = my_first.load(std::memory_order_acquire);\n if ( !curr )\n return nullptr;\n atomic_proxy_ptr* prev_ptr = &my_first;\n if ( isolation != no_isolation ) {\n while ( task_accessor::isolation(*curr) != isolation ) {\n prev_ptr = &curr->next_in_mailbox;\n // The next_in_mailbox should be read with acquire to guarantee (*curr) consistency.\n curr = curr->next_in_mailbox.load(std::memory_order_acquire);\n if ( !curr )\n return nullptr;\n }\n }\n // There is a first item in the mailbox. See if there is a second.\n // The next_in_mailbox should be read with acquire to guarantee (*second) consistency.\n if ( task_proxy* second = curr->next_in_mailbox.load(std::memory_order_acquire) ) {\n // There are at least two items, so first item can be popped easily.\n prev_ptr->store(second, std::memory_order_relaxed);\n } else {\n // There is only one item. Some care is required to pop it.\n\n prev_ptr->store(nullptr, std::memory_order_relaxed);\n atomic_proxy_ptr* expected = &curr->next_in_mailbox;\n if ( my_last.compare_exchange_strong( expected, prev_ptr ) ) {\n // Successfully transitioned mailbox from having one item to having none.\n __TBB_ASSERT( !curr->next_in_mailbox.load(std::memory_order_relaxed), nullptr);\n } else {\n // Some other thread updated my_last but has not filled in first->next_in_mailbox\n // Wait until first item points to second item.\n atomic_backoff backoff;\n // The next_in_mailbox should be read with acquire to guarantee (*second) consistency.\n while ( !(second = curr->next_in_mailbox.load(std::memory_order_acquire)) ) backoff.pause();\n prev_ptr->store( second, std::memory_order_relaxed);\n }\n }\n assert_pointer_valid(curr);\n return curr;\n }\npublic:\n friend class mail_inbox;\n\n //! Push task_proxy onto the mailbox queue of another thread.\n /** Implementation is wait-free. */\n void push( task_proxy* t ) {\n assert_pointer_valid(t);\n t->next_in_mailbox.store(nullptr, std::memory_order_relaxed);\n atomic_proxy_ptr* const link = my_last.exchange(&t->next_in_mailbox);\n // Logically, the release fence is not required because the exchange above provides the\n // release-acquire semantic that guarantees that (*t) will be consistent when another thread\n // loads the link atomic. However, C++11 memory model guarantees consistency of(*t) only\n // when the same atomic is used for synchronization.\n link->store(t, std::memory_order_release);\n }\n\n //! Return true if mailbox is empty\n bool empty() {\n return my_first.load(std::memory_order_relaxed) == nullptr;\n }\n\n //! Construct *this as a mailbox from zeroed memory.\n /** Raise assertion if *this is not previously zeroed, or sizeof(this) is wrong.\n This method is provided instead of a full constructor since we know the object\n will be constructed in zeroed memory. */\n void construct() {\n __TBB_ASSERT( sizeof(*this)==max_nfs_size, nullptr );\n __TBB_ASSERT( !my_first.load(std::memory_order_relaxed), nullptr );\n __TBB_ASSERT( !my_last.load(std::memory_order_relaxed), nullptr );\n __TBB_ASSERT( !my_is_idle.load(std::memory_order_relaxed), nullptr );\n my_last = &my_first;\n suppress_unused_warning(pad);\n }\n\n //! Drain the mailbox\n void drain() {\n // No fences here because other threads have already quit.\n for( ; task_proxy* t = my_first; ) {\n my_first.store(t->next_in_mailbox, std::memory_order_relaxed);\n t->allocator.delete_object(t);\n }\n }\n\n //! True if thread that owns this mailbox is looking for work.\n bool recipient_is_idle() {\n return my_is_idle.load(std::memory_order_relaxed);\n }\n}; // class mail_outbox\n\n//! Class representing source of mail.\nclass mail_inbox {\n //! Corresponding sink where mail that we receive will be put.\n mail_outbox* my_putter;\npublic:\n //! Construct unattached inbox\n mail_inbox() : my_putter(nullptr) {}\n\n //! Attach inbox to a corresponding outbox.\n void attach( mail_outbox& putter ) {\n my_putter = &putter;\n }\n //! Detach inbox from its outbox\n void detach() {\n __TBB_ASSERT(my_putter,\"not attached\");\n my_putter = nullptr;\n }\n //! Get next piece of mail, or nullptr if mailbox is empty.\n task_proxy* pop( isolation_type isolation ) {\n return my_putter->internal_pop( isolation );\n }\n //! Return true if mailbox is empty\n bool empty() {\n return my_putter->empty();\n }\n //! Indicate whether thread that reads this mailbox is idle.\n /** Raises assertion failure if mailbox is redundantly marked as not idle. */\n void set_is_idle( bool value ) {\n if( my_putter ) {\n __TBB_ASSERT( my_putter->my_is_idle.load(std::memory_order_relaxed) || value, \"attempt to redundantly mark mailbox as not idle\" );\n my_putter->my_is_idle.store(value, std::memory_order_relaxed);\n }\n }\n //! Indicate whether thread that reads this mailbox is idle.\n bool is_idle_state ( bool value ) const {\n return !my_putter || my_putter->my_is_idle.load(std::memory_order_relaxed) == value;\n }\n}; // class mail_inbox\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_mailbox_H */\n" }, { "path": "src/tbb/src/tbb/main.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n\n#include \"main.h\"\n#include \"governor.h\"\n#include \"threading_control.h\"\n#include \"environment.h\"\n#include \"market.h\"\n#include \"tcm_adaptor.h\"\n#include \"misc.h\"\n#include \"itt_notify.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// Begin shared data layout.\n// The following global data items are mostly read-only after initialization.\n//------------------------------------------------------------------------\n\n//------------------------------------------------------------------------\n// governor data\nbasic_tls governor::theTLS;\nrml::tbb_factory governor::theRMLServerFactory;\nbool governor::UsePrivateRML;\nbool governor::is_rethrow_broken;\n\n//------------------------------------------------------------------------\n// threading_control data\nthreading_control* threading_control::g_threading_control;\nthreading_control::global_mutex_type threading_control::g_threading_control_mutex;\n\n//------------------------------------------------------------------------\n// context propagation data\ncontext_state_propagation_mutex_type the_context_state_propagation_mutex;\nstd::atomic the_context_state_propagation_epoch{};\n\n//------------------------------------------------------------------------\n// One time initialization data\n\n//! Counter of references to global shared resources such as TLS.\nstd::atomic __TBB_InitOnce::count{};\n\nstd::atomic_flag __TBB_InitOnce::InitializationLock = ATOMIC_FLAG_INIT;\n\n//! Flag that is set to true after one-time initializations are done.\nstd::atomic __TBB_InitOnce::InitializationDone{};\n\n#if __TBB_USE_ITT_NOTIFY\n//! Defined in profiling.cpp\nextern bool ITT_Present;\nvoid ITT_DoUnsafeOneTimeInitialization();\n#endif\n\n#if !(_WIN32||_WIN64) || __TBB_SOURCE_DIRECTLY_INCLUDED\nstatic __TBB_InitOnce __TBB_InitOnceHiddenInstance;\n#endif\n\n//------------------------------------------------------------------------\n// __TBB_InitOnce\n//------------------------------------------------------------------------\n\nvoid __TBB_InitOnce::add_ref() {\n if (++count == 1) {\n governor::acquire_resources();\n tcm_adaptor::initialize();\n }\n}\n\nvoid __TBB_InitOnce::remove_ref() {\n int k = --count;\n __TBB_ASSERT(k>=0,\"removed __TBB_InitOnce ref that was not added?\");\n if( k==0 ) {\n governor::release_resources();\n ITT_FINI_ITTLIB();\n ITT_RELEASE_RESOURCES();\n }\n}\n\n//------------------------------------------------------------------------\n// One-time Initializations\n//------------------------------------------------------------------------\n\n//! Defined in cache_aligned_allocator.cpp\nvoid initialize_cache_aligned_allocator();\n\n//! Performs thread-safe lazy one-time general TBB initialization.\nvoid DoOneTimeInitialization() {\n __TBB_InitOnce::lock();\n // No fence required for load of InitializationDone, because we are inside a critical section.\n if( !__TBB_InitOnce::InitializationDone ) {\n __TBB_InitOnce::add_ref();\n if( GetBoolEnvironmentVariable(\"TBB_VERSION\") ) {\n PrintVersion();\n tcm_adaptor::print_version();\n }\n bool itt_present = false;\n#if __TBB_USE_ITT_NOTIFY\n ITT_DoUnsafeOneTimeInitialization();\n itt_present = ITT_Present;\n#endif /* __TBB_USE_ITT_NOTIFY */\n initialize_cache_aligned_allocator();\n governor::initialize_rml_factory();\n // Force processor groups support detection\n governor::default_num_threads();\n // Force OS regular page size detection\n governor::default_page_size();\n PrintExtraVersionInfo( \"TOOLS SUPPORT\", itt_present ? \"enabled\" : \"disabled\" );\n __TBB_InitOnce::InitializationDone = true;\n }\n __TBB_InitOnce::unlock();\n}\n\n#if (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED\n//! Windows \"DllMain\" that handles startup and shutdown of dynamic library.\nextern \"C\" bool WINAPI DllMain( HANDLE /*hinstDLL*/, DWORD reason, LPVOID lpvReserved ) {\n switch( reason ) {\n case DLL_PROCESS_ATTACH:\n __TBB_InitOnce::add_ref();\n break;\n case DLL_PROCESS_DETACH:\n // Since THREAD_DETACH is not called for the main thread, call auto-termination\n // here as well - but not during process shutdown (due to risk of a deadlock).\n if ( lpvReserved == nullptr ) { // library unload\n governor::terminate_external_thread();\n }\n __TBB_InitOnce::remove_ref();\n // It is assumed that InitializationDone is not set after DLL_PROCESS_DETACH,\n // and thus no race on InitializationDone is possible.\n if ( __TBB_InitOnce::initialization_done() ) {\n // Remove reference that we added in DoOneTimeInitialization.\n __TBB_InitOnce::remove_ref();\n }\n break;\n case DLL_THREAD_DETACH:\n governor::terminate_external_thread();\n break;\n }\n return true;\n}\n#endif /* (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/main.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_main_H\n#define _TBB_main_H\n\n#include \"governor.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nvoid DoOneTimeInitialization();\n\n//------------------------------------------------------------------------\n// __TBB_InitOnce\n//------------------------------------------------------------------------\n\n// TODO (TBB_REVAMP_TODO): consider better names\n//! Class that supports TBB initialization.\n/** It handles acquisition and release of global resources (e.g. TLS) during startup and shutdown,\n as well as synchronization for DoOneTimeInitialization. */\nclass __TBB_InitOnce {\n friend void DoOneTimeInitialization();\n friend void ITT_DoUnsafeOneTimeInitialization();\n\n static std::atomic count;\n\n //! Platform specific code to acquire resources.\n static void acquire_resources();\n\n //! Platform specific code to release resources.\n static void release_resources();\n\n //! Specifies if the one-time initializations has been done.\n static std::atomic InitializationDone;\n\n //! Global initialization lock\n /** Scenarios are possible when tools interop has to be initialized before the\n TBB itself. This imposes a requirement that the global initialization lock\n has to support valid static initialization, and does not issue any tool\n notifications in any build mode. **/\n static std::atomic_flag InitializationLock;\n\npublic:\n static void lock() {\n tbb::detail::atomic_backoff backoff;\n while( InitializationLock.test_and_set() ) backoff.pause();\n }\n\n static void unlock() { InitializationLock.clear(std::memory_order_release); }\n\n static bool initialization_done() { return InitializationDone.load(std::memory_order_acquire); }\n\n //! Add initial reference to resources.\n /** We assume that dynamic loading of the library prevents any other threads\n from entering the library until this constructor has finished running. **/\n __TBB_InitOnce() { add_ref(); }\n\n //! Remove the initial reference to resources.\n /** This is not necessarily the last reference if other threads are still running. **/\n ~__TBB_InitOnce() {\n governor::terminate_external_thread(); // TLS dtor not called for the main thread\n remove_ref();\n // We assume that InitializationDone is not set after file-scope destructors\n // start running, and thus no race on InitializationDone is possible.\n if ( initialization_done() ) {\n // Remove an extra reference that was added in DoOneTimeInitialization.\n remove_ref();\n }\n }\n //! Add reference to resources. If first reference added, acquire the resources.\n static void add_ref();\n\n //! Remove reference to resources. If last reference removed, release the resources.\n static void remove_ref();\n\n}; // class __TBB_InitOnce\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_main_H */\n" }, { "path": "src/tbb/src/tbb/market.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"arena.h\"\n#include \"market.h\"\n\n#include // std::find\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n\nclass tbb_permit_manager_client : public pm_client {\npublic:\n tbb_permit_manager_client(arena& a) : pm_client(a) {}\n\n void register_thread() override {}\n\n void unregister_thread() override {}\n\n void set_allotment(unsigned allotment) {\n my_arena.set_allotment(allotment);\n }\n};\n\n//------------------------------------------------------------------------\n// market\n//------------------------------------------------------------------------\n\nmarket::market(unsigned workers_soft_limit)\n : my_num_workers_soft_limit(workers_soft_limit)\n{}\n\npm_client* market::create_client(arena& a) {\n return new (cache_aligned_allocate(sizeof(tbb_permit_manager_client))) tbb_permit_manager_client(a);\n}\n\nvoid market::register_client(pm_client* c, d1::constraints&) {\n mutex_type::scoped_lock lock(my_mutex);\n my_clients[c->priority_level()].push_back(c);\n}\n\nvoid market::unregister_and_destroy_client(pm_client& c) {\n {\n mutex_type::scoped_lock lock(my_mutex);\n auto& clients = my_clients[c.priority_level()];\n auto it = std::find(clients.begin(), clients.end(), &c);\n __TBB_ASSERT(it != clients.end(), \"Destroying of an unregistered client\");\n clients.erase(it);\n }\n\n auto client = static_cast(&c);\n client->~tbb_permit_manager_client();\n cache_aligned_deallocate(client);\n}\n\nvoid market::update_allotment() {\n int effective_soft_limit = my_mandatory_num_requested > 0 && my_num_workers_soft_limit == 0 ? 1 : my_num_workers_soft_limit;\n int max_workers = min(my_total_demand, effective_soft_limit);\n __TBB_ASSERT(max_workers >= 0, nullptr);\n\n int unassigned_workers = max_workers;\n int assigned = 0;\n int carry = 0;\n unsigned max_priority_level = num_priority_levels;\n for (unsigned list_idx = 0; list_idx < num_priority_levels; ++list_idx ) {\n int assigned_per_priority = min(my_priority_level_demand[list_idx], unassigned_workers);\n unassigned_workers -= assigned_per_priority;\n // We use reverse iterator there to serve last added clients first\n for (auto it = my_clients[list_idx].rbegin(); it != my_clients[list_idx].rend(); ++it) {\n tbb_permit_manager_client& client = static_cast(**it);\n if (client.max_workers() == 0) {\n client.set_allotment(0);\n continue;\n }\n\n if (max_priority_level == num_priority_levels) {\n max_priority_level = list_idx;\n }\n\n int allotted = 0;\n if (my_num_workers_soft_limit == 0) {\n __TBB_ASSERT(max_workers == 0 || max_workers == 1, nullptr);\n allotted = client.min_workers() > 0 && assigned < max_workers ? 1 : 0;\n } else {\n int tmp = client.max_workers() * assigned_per_priority + carry;\n allotted = tmp / my_priority_level_demand[list_idx];\n carry = tmp % my_priority_level_demand[list_idx];\n __TBB_ASSERT(allotted <= client.max_workers(), nullptr);\n }\n client.set_allotment(allotted);\n client.set_top_priority(list_idx == max_priority_level);\n assigned += allotted;\n }\n }\n __TBB_ASSERT(assigned == max_workers, nullptr);\n}\n\nvoid market::set_active_num_workers(int soft_limit) {\n mutex_type::scoped_lock lock(my_mutex);\n if (my_num_workers_soft_limit != soft_limit) {\n my_num_workers_soft_limit = soft_limit;\n update_allotment();\n }\n}\n\nvoid market::adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) {\n __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr);\n\n int delta{};\n {\n mutex_type::scoped_lock lock(my_mutex);\n // Update client's state\n delta = c.update_request(mandatory_delta, workers_delta);\n\n // Update market's state\n my_total_demand += delta;\n my_priority_level_demand[c.priority_level()] += delta;\n my_mandatory_num_requested += mandatory_delta;\n\n update_allotment();\n }\n\n notify_thread_request(delta);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/market.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_market_H\n#define _TBB_market_H\n\n#include \"oneapi/tbb/rw_mutex.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/task_arena.h\"\n\n#include \"permit_manager.h\"\n#include \"pm_client.h\"\n\n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass market : public permit_manager {\npublic:\n market(unsigned soft_limit);\n\n pm_client* create_client(arena& a) override;\n void register_client(pm_client* client, d1::constraints&) override;\n void unregister_and_destroy_client(pm_client& c) override;\n\n //! Request that arena's need in workers should be adjusted.\n void adjust_demand(pm_client&, int mandatory_delta, int workers_delta) override;\n\n //! Set number of active workers\n void set_active_num_workers(int soft_limit) override;\nprivate:\n //! Recalculates the number of workers assigned to each arena in the list.\n void update_allotment();\n\n //! Keys for the arena map array. The lower the value the higher priority of the arena list.\n static constexpr unsigned num_priority_levels = d1::num_priority_levels;\n\n using mutex_type = d1::rw_mutex;\n mutex_type my_mutex;\n\n //! Current application-imposed limit on the number of workers\n int my_num_workers_soft_limit;\n\n //! Number of workers that were requested by all arenas on all priority levels\n int my_total_demand{0};\n\n //! Number of workers that were requested by arenas per single priority list item\n int my_priority_level_demand[num_priority_levels] = {0};\n\n //! How many times mandatory concurrency was requested from the market\n int my_mandatory_num_requested{0};\n\n //! Per priority list of registered arenas\n using clients_container_type = std::vector>;\n clients_container_type my_clients[num_priority_levels];\n}; // class market\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_market_H */\n" }, { "path": "src/tbb/src/tbb/market_concurrent_monitor.h", "content": "/*\n Copyright (c) 2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_market_concurrent_monitor_H\n#define __TBB_market_concurrent_monitor_H\n\n#include \"concurrent_monitor.h\"\n#include \"scheduler_common.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct market_context {\n market_context() = default;\n\n market_context(std::uintptr_t first_addr, arena* a) :\n my_uniq_addr(first_addr), my_arena_addr(a)\n {}\n\n std::uintptr_t my_uniq_addr{0};\n arena* my_arena_addr{nullptr};\n};\n\n#if __TBB_RESUMABLE_TASKS\nclass resume_node : public wait_node {\n using base_type = wait_node;\npublic:\n resume_node(market_context ctx, execution_data_ext& ed_ext, task_dispatcher& target)\n : base_type(ctx), my_curr_dispatcher(ed_ext.task_disp), my_target_dispatcher(&target)\n , my_suspend_point(my_curr_dispatcher->get_suspend_point())\n {}\n\n ~resume_node() override {\n if (this->my_skipped_wakeup) {\n spin_wait_until_eq(this->my_notify_calls, 1);\n }\n\n poison_pointer(my_curr_dispatcher);\n poison_pointer(my_target_dispatcher);\n poison_pointer(my_suspend_point);\n }\n\n void init() override {\n base_type::init();\n }\n\n void wait() override {\n my_curr_dispatcher->resume(*my_target_dispatcher);\n __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), \"Still in the queue?\");\n }\n\n void reset() override {\n base_type::reset();\n spin_wait_until_eq(this->my_notify_calls, 1);\n my_notify_calls.store(0, std::memory_order_relaxed);\n }\n\n // notify is called (perhaps, concurrently) twice from:\n // - concurrent_monitor::notify\n // - post_resume_action::register_waiter\n // The second notify is called after thread switches the stack\n // (Because we can not call resume while the stack is occupied)\n // We need calling resume only when both notifications are performed.\n void notify() override {\n if (++my_notify_calls == 2) {\n r1::resume(my_suspend_point);\n }\n }\n\nprivate:\n friend class thread_data;\n friend struct suspend_point_type::resume_task;\n task_dispatcher* my_curr_dispatcher;\n task_dispatcher* my_target_dispatcher;\n suspend_point_type* my_suspend_point;\n std::atomic my_notify_calls{0};\n};\n#endif // __TBB_RESUMABLE_TASKS\n\nclass market_concurrent_monitor : public concurrent_monitor_base {\n using base_type = concurrent_monitor_base;\npublic:\n using base_type::base_type;\n\n ~market_concurrent_monitor() {\n destroy();\n }\n\n /** per-thread descriptor for concurrent_monitor */\n using thread_context = sleep_node;\n#if __TBB_RESUMABLE_TASKS\n using resume_context = resume_node;\n#endif\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_market_concurrent_monitor_H\n" }, { "path": "src/tbb/src/tbb/misc.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// Source file for miscellaneous entities that are infrequently referenced by\n// an executing program.\n\n#include \"oneapi/tbb/detail/_exception.h\"\n#include \"oneapi/tbb/detail/_machine.h\"\n\n#include \"oneapi/tbb/version.h\"\n\n#include \"misc.h\"\n#include \"governor.h\"\n#include \"assert_impl.h\" // Out-of-line TBB assertion handling routines are instantiated here.\n#include \"concurrent_monitor_mutex.h\"\n\n#include \n#include \n#include \n#include \n#include \n\n#if _WIN32||_WIN64\n#include \n#endif\n\n#if !_WIN32\n#include // sysconf(_SC_PAGESIZE)\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// governor data\n//------------------------------------------------------------------------\ncpu_features_type governor::cpu_features;\n\n//------------------------------------------------------------------------\n// concurrent_monitor_mutex data\n//------------------------------------------------------------------------\n#if !__TBB_USE_FUTEX\nstd::mutex concurrent_monitor_mutex::my_init_mutex;\n#endif\n\n\nsize_t DefaultSystemPageSize() {\n#if _WIN32\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n return si.dwPageSize;\n#else\n return sysconf(_SC_PAGESIZE);\n#endif\n}\n\n/** The leading \"\\0\" is here so that applying \"strings\" to the binary delivers a clean result. */\nstatic const char VersionString[] = \"\\0\" TBB_VERSION_STRINGS;\n\nstatic bool PrintVersionFlag = false;\n\nvoid PrintVersion() {\n PrintVersionFlag = true;\n std::fputs(VersionString+1,stderr);\n}\n\nvoid PrintExtraVersionInfo( const char* category, const char* format, ... ) {\n if( PrintVersionFlag ) {\n char str[1024]; std::memset(str, 0, 1024);\n va_list args; va_start(args, format);\n // Note: correct vsnprintf definition obtained from tbb_assert_impl.h\n std::vsnprintf( str, 1024-1, format, args);\n va_end(args);\n std::fprintf(stderr, \"oneTBB: %s\\t%s\\n\", category, str );\n }\n}\n\n//! check for transaction support.\n#if _MSC_VER\n#include // for __cpuid\n#elif __APPLE__\n#include \n#endif\n\n#if __TBB_x86_32 || __TBB_x86_64\nvoid check_cpuid(int leaf, int sub_leaf, int registers[4]) {\n#if _MSC_VER\n __cpuidex(registers, leaf, sub_leaf);\n#else\n int reg_eax = 0;\n int reg_ebx = 0;\n int reg_ecx = 0;\n int reg_edx = 0;\n#if __TBB_x86_32 && __PIC__\n // On 32-bit systems with position-independent code GCC fails to work around the stuff in EBX\n // register. We help it using backup and restore.\n __asm__(\"mov %%ebx, %%esi\\n\\t\"\n \"cpuid\\n\\t\"\n \"xchg %%ebx, %%esi\"\n : \"=a\"(reg_eax), \"=S\"(reg_ebx), \"=c\"(reg_ecx), \"=d\"(reg_edx)\n : \"0\"(leaf), \"2\"(sub_leaf) // read value from eax and ecx\n );\n#else\n __asm__(\"cpuid\"\n : \"=a\"(reg_eax), \"=b\"(reg_ebx), \"=c\"(reg_ecx), \"=d\"(reg_edx)\n : \"0\"(leaf), \"2\"(sub_leaf) // read value from eax and ecx\n );\n#endif\n registers[0] = reg_eax;\n registers[1] = reg_ebx;\n registers[2] = reg_ecx;\n registers[3] = reg_edx;\n#endif\n}\n#endif\n\nvoid detect_cpu_features(cpu_features_type& cpu_features) {\n suppress_unused_warning(cpu_features);\n#if __TBB_x86_32 || __TBB_x86_64\n const int rtm_ebx_mask = 1 << 11;\n const int waitpkg_ecx_mask = 1 << 5;\n const int hybrid_edx_mask = 1 << 15;\n int registers[4] = {0};\n\n // Check RTM, WAITPKG, HYBRID\n check_cpuid(7, 0, registers);\n cpu_features.rtm_enabled = (registers[1] & rtm_ebx_mask) != 0;\n cpu_features.waitpkg_enabled = (registers[2] & waitpkg_ecx_mask) != 0;\n cpu_features.hybrid = (registers[3] & hybrid_edx_mask) != 0;\n#elif __APPLE__\n // Check HYBRID (hw.nperflevels > 1)\n uint64_t nperflevels = 0;\n size_t nperflevels_size = sizeof(nperflevels);\n if (!sysctlbyname(\"hw.nperflevels\", &nperflevels, &nperflevels_size, nullptr, 0)) {\n cpu_features.hybrid = (nperflevels > 1);\n }\n#endif\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/misc.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_tbb_misc_H\n#define _TBB_tbb_misc_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n\n#if __TBB_ARENA_BINDING\n#include \"oneapi/tbb/info.h\"\n#endif /*__TBB_ARENA_BINDING*/\n\n#if __unix__\n#include // __FreeBSD_version\n#if __FreeBSD_version >= 701000\n#include \n#endif\n#endif\n\n#include \n\n// Does the operating system have a system call to pin a thread to a set of OS processors?\n#define __TBB_OS_AFFINITY_SYSCALL_PRESENT ((__linux__ && !__ANDROID__) || (__FreeBSD_version >= 701000))\n// On IBM* Blue Gene* CNK nodes, the affinity API has restrictions that prevent its usability for TBB,\n// and also sysconf(_SC_NPROCESSORS_ONLN) already takes process affinity into account.\n#define __TBB_USE_OS_AFFINITY_SYSCALL (__TBB_OS_AFFINITY_SYSCALL_PRESENT && !__bg__)\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nvoid runtime_warning(const char* format, ... );\n\n#if __TBB_ARENA_BINDING\nclass task_arena;\nclass task_scheduler_observer;\n#endif /*__TBB_ARENA_BINDING*/\n\nconst std::size_t MByte = 1024*1024;\n\n#if __TBB_USE_WINAPI\n// The Microsoft Documentation about Thread Stack Size states that\n// \"The default stack reservation size used by the linker is 1 MB\"\nconst std::size_t ThreadStackSize = 1*MByte;\n#else\nconst std::size_t ThreadStackSize = (sizeof(uintptr_t) <= 4 ? 2 : 4 )*MByte;\n#endif\n\n#ifndef __TBB_HardwareConcurrency\n\n//! Returns maximal parallelism level supported by the current OS configuration.\nint AvailableHwConcurrency();\n\n#else\n\ninline int AvailableHwConcurrency() {\n int n = __TBB_HardwareConcurrency();\n return n > 0 ? n : 1; // Fail safety strap\n}\n#endif /* __TBB_HardwareConcurrency */\n\n//! Returns OS regular memory page size\nsize_t DefaultSystemPageSize();\n\n//! Returns number of processor groups in the current OS configuration.\n/** AvailableHwConcurrency must be called at least once before calling this method. **/\nint NumberOfProcessorGroups();\n\n#if _WIN32||_WIN64\n\n//! Retrieves index of processor group containing processor with the given index\nint FindProcessorGroupIndex ( int processorIndex );\n\n//! Affinitizes the thread to the specified processor group\nvoid MoveThreadIntoProcessorGroup( void* hThread, int groupIndex );\n\n#endif /* _WIN32||_WIN64 */\n\n//! Prints TBB version information on stderr\nvoid PrintVersion();\n\n//! Prints arbitrary extra TBB version information on stderr\nvoid PrintExtraVersionInfo( const char* category, const char* format, ... );\n\n//! A callback routine to print RML version information on stderr\nvoid PrintRMLVersionInfo( void* arg, const char* server_info );\n\n// For TBB compilation only; not to be used in public headers\n#if defined(min) || defined(max)\n#undef min\n#undef max\n#endif\n\n//! Utility template function returning lesser of the two values.\n/** Provided here to avoid including not strict safe .\\n\n In case operands cause signed/unsigned or size mismatch warnings it is caller's\n responsibility to do the appropriate cast before calling the function. **/\ntemplate\nT min ( const T& val1, const T& val2 ) {\n return val1 < val2 ? val1 : val2;\n}\n\n//! Utility template function returning greater of the two values.\n/** Provided here to avoid including not strict safe .\\n\n In case operands cause signed/unsigned or size mismatch warnings it is caller's\n responsibility to do the appropriate cast before calling the function. **/\ntemplate\nT max ( const T& val1, const T& val2 ) {\n return val1 < val2 ? val2 : val1;\n}\n\n//! Utility helper structure to ease overload resolution\ntemplate struct int_to_type {};\n\n//------------------------------------------------------------------------\n// FastRandom\n//------------------------------------------------------------------------\n\n//! A fast random number generator.\n/** Uses linear congruential method. */\nclass FastRandom {\nprivate:\n unsigned x, c;\n static const unsigned a = 0x9e3779b1; // a big prime number\npublic:\n //! Get a random number.\n unsigned short get() {\n return get(x);\n }\n //! Get a random number for the given seed; update the seed for next use.\n unsigned short get( unsigned& seed ) {\n unsigned short r = (unsigned short)(seed>>16);\n __TBB_ASSERT(c&1, \"c must be odd for big rng period\");\n seed = seed*a+c;\n return r;\n }\n //! Construct a random number generator.\n FastRandom( void* unique_ptr ) { init(uintptr_t(unique_ptr)); }\n\n template \n void init( T seed ) {\n init(seed,int_to_type());\n }\n void init( uint64_t seed , int_to_type<8> ) {\n init(uint32_t((seed>>32)+seed), int_to_type<4>());\n }\n void init( uint32_t seed, int_to_type<4> ) {\n // threads use different seeds for unique sequences\n c = (seed|1)*0xba5703f5; // c must be odd, shuffle by a prime number\n x = c^(seed>>1); // also shuffle x for the first get() invocation\n }\n};\n\n//------------------------------------------------------------------------\n// Atomic extensions\n//------------------------------------------------------------------------\n\n//! Atomically replaces value of dst with newValue if they satisfy condition of compare predicate\n/** Return value semantics is the same as for CAS. **/\ntemplate\nT1 atomic_update(std::atomic& dst, T1 newValue, Pred compare) {\n T1 oldValue = dst.load(std::memory_order_acquire);\n while ( compare(oldValue, newValue) ) {\n if ( dst.compare_exchange_strong(oldValue, newValue) )\n break;\n }\n return oldValue;\n}\n\n#if __TBB_USE_OS_AFFINITY_SYSCALL\n #if __linux__\n typedef cpu_set_t basic_mask_t;\n #elif __FreeBSD_version >= 701000\n typedef cpuset_t basic_mask_t;\n #else\n #error affinity_helper is not implemented in this OS\n #endif\n class affinity_helper : no_copy {\n basic_mask_t* threadMask;\n int is_changed;\n public:\n affinity_helper() : threadMask(nullptr), is_changed(0) {}\n ~affinity_helper();\n void protect_affinity_mask( bool restore_process_mask );\n void dismiss();\n };\n void destroy_process_mask();\n#else\n class affinity_helper : no_copy {\n public:\n void protect_affinity_mask( bool ) {}\n };\n inline void destroy_process_mask(){}\n#endif /* __TBB_USE_OS_AFFINITY_SYSCALL */\n\nstruct cpu_features_type {\n bool rtm_enabled{false};\n bool waitpkg_enabled{false};\n bool hybrid{false};\n};\n\nvoid detect_cpu_features(cpu_features_type& cpu_features);\n\n#if __TBB_ARENA_BINDING\nclass binding_handler;\n\nbinding_handler* construct_binding_handler(int slot_num, int numa_id, int core_type_id, int max_threads_per_core);\nvoid destroy_binding_handler(binding_handler* handler_ptr);\nvoid apply_affinity_mask(binding_handler* handler_ptr, int slot_num);\nvoid restore_affinity_mask(binding_handler* handler_ptr, int slot_num);\n\n#endif /*__TBB_ARENA_BINDING*/\n\n// RTM specific section\n// abort code for mutexes that detect a conflict with another thread.\nenum {\n speculation_not_supported = 0x00,\n speculation_transaction_aborted = 0x01,\n speculation_can_retry = 0x02,\n speculation_memadd_conflict = 0x04,\n speculation_buffer_overflow = 0x08,\n speculation_breakpoint_hit = 0x10,\n speculation_nested_abort = 0x20,\n speculation_xabort_mask = 0xFF000000,\n speculation_xabort_shift = 24,\n speculation_xabort_not_free = 0xFF, // The value (0xFF) below comes from the Intel(R) 64 and IA-32 Architectures Optimization Reference Manual 12.4.5 lock not free\n speculation_successful_begin = 0xFFFFFFFF,\n speculation_retry = speculation_transaction_aborted\n | speculation_can_retry\n | speculation_memadd_conflict\n};\n\n// We suppose that successful transactions are sequentially ordered and\n// do not require additional memory fences around them.\n// Technically it can be achieved only if xbegin has implicit\n// acquire memory semantics an xend/xabort has release memory semantics on compiler and hardware level.\n// See the article: https://arxiv.org/pdf/1710.04839.pdf\nstatic inline unsigned int begin_transaction() {\n#if __TBB_TSX_INTRINSICS_PRESENT\n return _xbegin();\n#else\n return speculation_not_supported; // return unsuccessful code\n#endif\n}\n\nstatic inline void end_transaction() {\n#if __TBB_TSX_INTRINSICS_PRESENT\n _xend();\n#endif\n}\n\nstatic inline void abort_transaction() {\n#if __TBB_TSX_INTRINSICS_PRESENT\n _xabort(speculation_xabort_not_free);\n#endif\n}\n\n#if TBB_USE_ASSERT\nstatic inline unsigned char is_in_transaction() {\n#if __TBB_TSX_INTRINSICS_PRESENT\n return _xtest();\n#else\n return 0;\n#endif\n}\n#endif // TBB_USE_ASSERT\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_tbb_misc_H */\n" }, { "path": "src/tbb/src/tbb/misc_ex.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// Source file for miscellaneous entities that are infrequently referenced by\n// an executing program, and implementation of which requires dynamic linking.\n\n#include \"misc.h\"\n\n#if !defined(__TBB_HardwareConcurrency)\n\n#include \"dynamic_link.h\"\n#include \n#include \n\n#if _WIN32||_WIN64\n#include \n#if __TBB_WIN8UI_SUPPORT\n#include \n#endif\n#else\n#include \n#if __unix__\n#if __linux__\n#include \n#endif\n#include \n#include \n#include \n#elif __sun\n#include \n#elif __FreeBSD__\n#include \n#include \n#include // Required by \n#include \n#endif\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_USE_OS_AFFINITY_SYSCALL\n\n#if __unix__\n// Handlers for interoperation with libiomp\nstatic int (*libiomp_try_restoring_original_mask)();\n// Table for mapping to libiomp entry points\nstatic const dynamic_link_descriptor iompLinkTable[] = {\n DLD_NOWEAK( kmp_set_thread_affinity_mask_initial, libiomp_try_restoring_original_mask )\n};\n#endif\n\nstatic void set_thread_affinity_mask( std::size_t maskSize, const basic_mask_t* threadMask ) {\n#if __FreeBSD__ || __NetBSD__ || __OpenBSD__\n if( cpuset_setaffinity( CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, maskSize, threadMask ) )\n#else /* __unix__ */\n if( sched_setaffinity( 0, maskSize, threadMask ) )\n#endif\n // Here and below the error severity is lowered from critical level\n // because it may happen during TBB library unload because of not\n // waiting for workers to complete (current RML policy, to be fixed).\n // handle_perror( errno, \"setaffinity syscall\" );\n runtime_warning( \"setaffinity syscall failed\" );\n}\n\nstatic void get_thread_affinity_mask( std::size_t maskSize, basic_mask_t* threadMask ) {\n#if __FreeBSD__ || __NetBSD__ || __OpenBSD__\n if( cpuset_getaffinity( CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, maskSize, threadMask ) )\n#else /* __unix__ */\n if( sched_getaffinity( 0, maskSize, threadMask ) )\n#endif\n runtime_warning( \"getaffinity syscall failed\" );\n}\n\nstatic basic_mask_t* process_mask;\nstatic int num_masks;\n\nvoid destroy_process_mask() {\n delete [] process_mask;\n process_mask = nullptr;\n}\n\n#define curMaskSize sizeof(basic_mask_t) * num_masks\naffinity_helper::~affinity_helper() {\n if( threadMask ) {\n if( is_changed ) {\n set_thread_affinity_mask( curMaskSize, threadMask );\n }\n delete [] threadMask;\n }\n}\nvoid affinity_helper::protect_affinity_mask( bool restore_process_mask ) {\n if( threadMask == nullptr && num_masks ) { // TODO: assert num_masks validity?\n threadMask = new basic_mask_t [num_masks];\n std::memset( threadMask, 0, curMaskSize );\n get_thread_affinity_mask( curMaskSize, threadMask );\n if( restore_process_mask ) {\n __TBB_ASSERT( process_mask, \"A process mask is requested but not yet stored\" );\n is_changed = memcmp( process_mask, threadMask, curMaskSize );\n if( is_changed )\n set_thread_affinity_mask( curMaskSize, process_mask );\n } else {\n // Assume that the mask will be changed by the caller.\n is_changed = 1;\n }\n }\n}\nvoid affinity_helper::dismiss() {\n delete [] threadMask;\n threadMask = nullptr;\n is_changed = 0;\n}\n#undef curMaskSize\n\nstatic std::atomic hardware_concurrency_info;\n\nstatic int theNumProcs;\n\nstatic void initialize_hardware_concurrency_info () {\n int err;\n int availableProcs = 0;\n int numMasks = 1;\n int maxProcs = sysconf(_SC_NPROCESSORS_ONLN);\n basic_mask_t* processMask;\n const std::size_t BasicMaskSize = sizeof(basic_mask_t);\n for (;;) {\n const int curMaskSize = BasicMaskSize * numMasks;\n processMask = new basic_mask_t[numMasks];\n std::memset( processMask, 0, curMaskSize );\n#if __FreeBSD__ || __NetBSD__ || __OpenBSD__\n // CPU_LEVEL_WHICH - anonymous (current) mask, CPU_LEVEL_CPUSET - assigned mask\n err = cpuset_getaffinity( CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, curMaskSize, processMask );\n if ( !err || errno != ERANGE || curMaskSize * CHAR_BIT >= 16 * 1024 )\n break;\n#else /* __unix__ */\n int pid = getpid();\n err = sched_getaffinity( pid, curMaskSize, processMask );\n if ( !err || errno != EINVAL || curMaskSize * CHAR_BIT >= 256 * 1024 )\n break;\n#endif\n delete[] processMask;\n numMasks <<= 1;\n }\n if ( !err ) {\n // We have found the mask size and captured the process affinity mask into processMask.\n num_masks = numMasks; // do here because it's needed for affinity_helper to work\n#if __unix__\n // For better coexistence with libiomp which might have changed the mask already,\n // check for its presence and ask it to restore the mask.\n dynamic_link_handle libhandle;\n if ( dynamic_link( \"libiomp5.so\", iompLinkTable, 1, &libhandle, DYNAMIC_LINK_GLOBAL ) ) {\n // We have found the symbol provided by libiomp5 for restoring original thread affinity.\n affinity_helper affhelp;\n affhelp.protect_affinity_mask( /*restore_process_mask=*/false );\n if ( libiomp_try_restoring_original_mask()==0 ) {\n // Now we have the right mask to capture, restored by libiomp.\n const int curMaskSize = BasicMaskSize * numMasks;\n std::memset( processMask, 0, curMaskSize );\n get_thread_affinity_mask( curMaskSize, processMask );\n } else\n affhelp.dismiss(); // thread mask has not changed\n dynamic_unlink( libhandle );\n // Destructor of affinity_helper restores the thread mask (unless dismissed).\n }\n#endif\n for ( int m = 0; availableProcs < maxProcs && m < numMasks; ++m ) {\n for ( std::size_t i = 0; (availableProcs < maxProcs) && (i < BasicMaskSize * CHAR_BIT); ++i ) {\n if ( CPU_ISSET( i, processMask + m ) )\n ++availableProcs;\n }\n }\n process_mask = processMask;\n }\n else {\n // Failed to get the process affinity mask; assume the whole machine can be used.\n availableProcs = (maxProcs == INT_MAX) ? sysconf(_SC_NPROCESSORS_ONLN) : maxProcs;\n delete[] processMask;\n }\n theNumProcs = availableProcs > 0 ? availableProcs : 1; // Fail safety strap\n __TBB_ASSERT( theNumProcs <= sysconf(_SC_NPROCESSORS_ONLN), nullptr);\n}\n\nint AvailableHwConcurrency() {\n atomic_do_once( &initialize_hardware_concurrency_info, hardware_concurrency_info );\n return theNumProcs;\n}\n\n/* End of __TBB_USE_OS_AFFINITY_SYSCALL implementation */\n#elif __ANDROID__\n\n// Work-around for Android that reads the correct number of available CPUs since system calls are unreliable.\n// Format of \"present\" file is: ([-|],)+\nint AvailableHwConcurrency() {\n FILE *fp = fopen(\"/sys/devices/system/cpu/present\", \"r\");\n if (fp == nullptr) return 1;\n int num_args, lower, upper, num_cpus=0;\n while ((num_args = fscanf(fp, \"%u-%u\", &lower, &upper)) != EOF) {\n switch(num_args) {\n case 2: num_cpus += upper - lower + 1; break;\n case 1: num_cpus += 1; break;\n }\n fscanf(fp, \",\");\n }\n fclose(fp);\n return (num_cpus > 0) ? num_cpus : 1;\n}\n\n#elif defined(_SC_NPROCESSORS_ONLN)\n\nint AvailableHwConcurrency() {\n int n = sysconf(_SC_NPROCESSORS_ONLN);\n return (n > 0) ? n : 1;\n}\n\n#elif _WIN32||_WIN64\n\nstatic std::atomic hardware_concurrency_info;\n\nstatic const WORD TBB_ALL_PROCESSOR_GROUPS = 0xffff;\n\n// Statically allocate an array for processor group information.\n// Windows 7 supports maximum 4 groups, but let's look ahead a little.\nstatic const WORD MaxProcessorGroups = 64;\n\nstruct ProcessorGroupInfo {\n DWORD_PTR mask; ///< Affinity mask covering the whole group\n int numProcs; ///< Number of processors in the group\n int numProcsRunningTotal; ///< Subtotal of processors in this and preceding groups\n\n //! Total number of processor groups in the system\n static int NumGroups;\n\n //! Index of the group with a slot reserved for the first external thread\n /** In the context of multiple processor groups support current implementation\n defines \"the first external thread\" as the first thread to invoke\n AvailableHwConcurrency().\n\n TODO: Implement a dynamic scheme remapping workers depending on the pending\n external threads affinity. **/\n static int HoleIndex;\n};\n\nint ProcessorGroupInfo::NumGroups = 1;\nint ProcessorGroupInfo::HoleIndex = 0;\n\nProcessorGroupInfo theProcessorGroups[MaxProcessorGroups];\nint calculate_numa[MaxProcessorGroups]; //Array needed for FindProcessorGroupIndex to calculate Processor Group when number of threads > number of cores to distribute threads evenly between processor groups\nint numaSum;\nstruct TBB_GROUP_AFFINITY {\n DWORD_PTR Mask;\n WORD Group;\n WORD Reserved[3];\n};\n\nstatic DWORD (WINAPI *TBB_GetActiveProcessorCount)( WORD groupIndex ) = nullptr;\nstatic WORD (WINAPI *TBB_GetActiveProcessorGroupCount)() = nullptr;\nstatic BOOL (WINAPI *TBB_SetThreadGroupAffinity)( HANDLE hThread,\n const TBB_GROUP_AFFINITY* newAff, TBB_GROUP_AFFINITY *prevAff );\nstatic BOOL (WINAPI *TBB_GetThreadGroupAffinity)( HANDLE hThread, TBB_GROUP_AFFINITY* );\n\nstatic const dynamic_link_descriptor ProcessorGroupsApiLinkTable[] = {\n DLD(GetActiveProcessorCount, TBB_GetActiveProcessorCount)\n , DLD(GetActiveProcessorGroupCount, TBB_GetActiveProcessorGroupCount)\n , DLD(SetThreadGroupAffinity, TBB_SetThreadGroupAffinity)\n , DLD(GetThreadGroupAffinity, TBB_GetThreadGroupAffinity)\n};\n\nstatic void initialize_hardware_concurrency_info () {\n suppress_unused_warning(TBB_ALL_PROCESSOR_GROUPS);\n#if __TBB_WIN8UI_SUPPORT\n // For these applications processor groups info is unavailable\n // Setting up a number of processors for one processor group\n theProcessorGroups[0].numProcs = theProcessorGroups[0].numProcsRunningTotal = std::thread::hardware_concurrency();\n#else /* __TBB_WIN8UI_SUPPORT */\n dynamic_link( \"Kernel32.dll\", ProcessorGroupsApiLinkTable,\n sizeof(ProcessorGroupsApiLinkTable)/sizeof(dynamic_link_descriptor) );\n SYSTEM_INFO si;\n GetNativeSystemInfo(&si);\n DWORD_PTR pam, sam, m = 1;\n GetProcessAffinityMask( GetCurrentProcess(), &pam, &sam );\n int nproc = 0;\n for ( std::size_t i = 0; i < sizeof(DWORD_PTR) * CHAR_BIT; ++i, m <<= 1 ) {\n if ( pam & m )\n ++nproc;\n }\n __TBB_ASSERT( nproc <= (int)si.dwNumberOfProcessors, nullptr);\n // By default setting up a number of processors for one processor group\n theProcessorGroups[0].numProcs = theProcessorGroups[0].numProcsRunningTotal = nproc;\n // Setting up processor groups in case the process does not restrict affinity mask and more than one processor group is present\n if ( nproc == (int)si.dwNumberOfProcessors && TBB_GetActiveProcessorCount ) {\n // The process does not have restricting affinity mask and multiple processor groups are possible\n ProcessorGroupInfo::NumGroups = (int)TBB_GetActiveProcessorGroupCount();\n __TBB_ASSERT( ProcessorGroupInfo::NumGroups <= MaxProcessorGroups, nullptr);\n // Fail safety bootstrap. Release versions will limit available concurrency\n // level, while debug ones would assert.\n if ( ProcessorGroupInfo::NumGroups > MaxProcessorGroups )\n ProcessorGroupInfo::NumGroups = MaxProcessorGroups;\n if ( ProcessorGroupInfo::NumGroups > 1 ) {\n TBB_GROUP_AFFINITY ga;\n if ( TBB_GetThreadGroupAffinity( GetCurrentThread(), &ga ) )\n ProcessorGroupInfo::HoleIndex = ga.Group;\n int nprocs = 0;\n int min_procs = INT_MAX;\n for ( WORD i = 0; i < ProcessorGroupInfo::NumGroups; ++i ) {\n ProcessorGroupInfo &pgi = theProcessorGroups[i];\n pgi.numProcs = (int)TBB_GetActiveProcessorCount(i);\n if (pgi.numProcs < min_procs) min_procs = pgi.numProcs; //Finding the minimum number of processors in the Processor Groups\n calculate_numa[i] = pgi.numProcs;\n __TBB_ASSERT( pgi.numProcs <= (int)sizeof(DWORD_PTR) * CHAR_BIT, nullptr);\n pgi.mask = pgi.numProcs == sizeof(DWORD_PTR) * CHAR_BIT ? ~(DWORD_PTR)0 : (DWORD_PTR(1) << pgi.numProcs) - 1;\n pgi.numProcsRunningTotal = nprocs += pgi.numProcs;\n }\n __TBB_ASSERT( nprocs == (int)TBB_GetActiveProcessorCount( TBB_ALL_PROCESSOR_GROUPS ), nullptr);\n\n calculate_numa[0] = (calculate_numa[0] / min_procs)-1;\n for (WORD i = 1; i < ProcessorGroupInfo::NumGroups; ++i) {\n calculate_numa[i] = calculate_numa[i-1] + (calculate_numa[i] / min_procs);\n }\n\n numaSum = calculate_numa[ProcessorGroupInfo::NumGroups - 1];\n\n }\n\n }\n#endif /* __TBB_WIN8UI_SUPPORT */\n\n PrintExtraVersionInfo(\"Processor groups\", \"%d\", ProcessorGroupInfo::NumGroups);\n if (ProcessorGroupInfo::NumGroups>1)\n for (int i=0; i= theProcessorGroups[current_grp_idx].numProcs && procIdx < theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal) {\n procIdx = procIdx - theProcessorGroups[current_grp_idx].numProcs;\n do {\n current_grp_idx = (current_grp_idx + 1) % (ProcessorGroupInfo::NumGroups);\n procIdx = procIdx - theProcessorGroups[current_grp_idx].numProcs;\n\n } while (procIdx >= 0);\n }\n else if (procIdx >= theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal) {\n int temp_grp_index = 0;\n procIdx = procIdx - theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal; \n procIdx = procIdx % (numaSum+1); //ProcIdx to stay between 0 and numaSum\n\n while (procIdx - calculate_numa[temp_grp_index] > 0) {\n temp_grp_index = (temp_grp_index + 1) % ProcessorGroupInfo::NumGroups;\n }\n current_grp_idx = temp_grp_index;\n }\n __TBB_ASSERT(current_grp_idx < ProcessorGroupInfo::NumGroups, nullptr);\n\n return current_grp_idx;\n}\n\nvoid MoveThreadIntoProcessorGroup( void* hThread, int groupIndex ) {\n __TBB_ASSERT( hardware_concurrency_info == do_once_state::initialized, \"MoveThreadIntoProcessorGroup is used before AvailableHwConcurrency\" );\n if ( !TBB_SetThreadGroupAffinity )\n return;\n TBB_GROUP_AFFINITY ga = { theProcessorGroups[groupIndex].mask, (WORD)groupIndex, {0,0,0} };\n TBB_SetThreadGroupAffinity( hThread, &ga, nullptr);\n}\n\nint AvailableHwConcurrency() {\n atomic_do_once( &initialize_hardware_concurrency_info, hardware_concurrency_info );\n return theProcessorGroups[ProcessorGroupInfo::NumGroups - 1].numProcsRunningTotal;\n}\n\n/* End of _WIN32||_WIN64 implementation */\n#else\n #error AvailableHwConcurrency is not implemented for this OS\n#endif\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* !__TBB_HardwareConcurrency */\n" }, { "path": "src/tbb/src/tbb/observer_proxy.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n\n#include \"observer_proxy.h\"\n#include \"arena.h\"\n#include \"main.h\"\n#include \"thread_data.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if TBB_USE_ASSERT\nextern std::atomic the_observer_proxy_count;\n#endif /* TBB_USE_ASSERT */\n\nobserver_proxy::observer_proxy( d1::task_scheduler_observer& tso )\n : my_ref_count(1), my_list(nullptr), my_next(nullptr), my_prev(nullptr), my_observer(&tso)\n{\n#if TBB_USE_ASSERT\n ++the_observer_proxy_count;\n#endif /* TBB_USE_ASSERT */\n}\n\nobserver_proxy::~observer_proxy() {\n __TBB_ASSERT( !my_ref_count, \"Attempt to destroy proxy still in use\" );\n poison_value(my_ref_count);\n poison_pointer(my_prev);\n poison_pointer(my_next);\n#if TBB_USE_ASSERT\n --the_observer_proxy_count;\n#endif /* TBB_USE_ASSERT */\n}\n\nvoid observer_list::clear() {\n {\n scoped_lock lock(mutex(), /*is_writer=*/true);\n observer_proxy *next = my_head.load(std::memory_order_relaxed);\n while ( observer_proxy *p = next ) {\n next = p->my_next;\n // Both proxy p and observer p->my_observer (if non-null) are guaranteed\n // to be alive while the list is locked.\n d1::task_scheduler_observer *obs = p->my_observer;\n // Make sure that possible concurrent observer destruction does not\n // conflict with the proxy list cleanup.\n if (!obs || !(p = obs->my_proxy.exchange(nullptr))) {\n continue;\n }\n // accessing 'obs' after detaching of obs->my_proxy leads to the race with observer destruction\n __TBB_ASSERT(!next || p == next->my_prev, nullptr);\n __TBB_ASSERT(is_alive(p->my_ref_count), \"Observer's proxy died prematurely\");\n __TBB_ASSERT(p->my_ref_count.load(std::memory_order_relaxed) == 1, \"Reference for observer is missing\");\n poison_pointer(p->my_observer);\n remove(p);\n --p->my_ref_count;\n delete p;\n }\n }\n\n // If observe(false) is called concurrently with the destruction of the arena,\n // need to wait until all proxies are removed.\n for (atomic_backoff backoff; ; backoff.pause()) {\n scoped_lock lock(mutex(), /*is_writer=*/false);\n if (my_head.load(std::memory_order_relaxed) == nullptr) {\n break;\n }\n }\n\n __TBB_ASSERT(my_head.load(std::memory_order_relaxed) == nullptr && my_tail.load(std::memory_order_relaxed) == nullptr, nullptr);\n}\n\nvoid observer_list::insert( observer_proxy* p ) {\n scoped_lock lock(mutex(), /*is_writer=*/true);\n if (my_head.load(std::memory_order_relaxed)) {\n p->my_prev = my_tail.load(std::memory_order_relaxed);\n my_tail.load(std::memory_order_relaxed)->my_next = p;\n } else {\n my_head.store(p, std::memory_order_relaxed);\n }\n my_tail.store(p, std::memory_order_relaxed);\n}\n\nvoid observer_list::remove(observer_proxy* p) {\n __TBB_ASSERT(my_head.load(std::memory_order_relaxed), \"Attempt to remove an item from an empty list\");\n __TBB_ASSERT(!my_tail.load(std::memory_order_relaxed)->my_next, \"Last item's my_next must be nullptr\");\n if (p == my_tail.load(std::memory_order_relaxed)) {\n __TBB_ASSERT(!p->my_next, nullptr);\n my_tail.store(p->my_prev, std::memory_order_relaxed);\n } else {\n __TBB_ASSERT(p->my_next, nullptr);\n p->my_next->my_prev = p->my_prev;\n }\n if (p == my_head.load(std::memory_order_relaxed)) {\n __TBB_ASSERT(!p->my_prev, nullptr);\n my_head.store(p->my_next, std::memory_order_relaxed);\n } else {\n __TBB_ASSERT(p->my_prev, nullptr);\n p->my_prev->my_next = p->my_next;\n }\n __TBB_ASSERT((my_head.load(std::memory_order_relaxed) && my_tail.load(std::memory_order_relaxed)) ||\n (!my_head.load(std::memory_order_relaxed) && !my_tail.load(std::memory_order_relaxed)), nullptr);\n}\n\nvoid observer_list::remove_ref(observer_proxy* p) {\n std::uintptr_t r = p->my_ref_count.load(std::memory_order_acquire);\n __TBB_ASSERT(is_alive(r), nullptr);\n while (r > 1) {\n if (p->my_ref_count.compare_exchange_strong(r, r - 1)) {\n return;\n }\n }\n __TBB_ASSERT(r == 1, nullptr);\n // Reference count might go to zero\n {\n // Use lock to avoid resurrection by a thread concurrently walking the list\n observer_list::scoped_lock lock(mutex(), /*is_writer=*/true);\n r = --p->my_ref_count;\n if (!r) {\n remove(p);\n }\n }\n __TBB_ASSERT(r || !p->my_ref_count, nullptr);\n if (!r) {\n delete p;\n }\n}\n\nvoid observer_list::do_notify_entry_observers(observer_proxy*& last, bool worker) {\n // Pointer p marches though the list from last (exclusively) to the end.\n observer_proxy* p = last, * prev = p;\n for (;;) {\n d1::task_scheduler_observer* tso = nullptr;\n // Hold lock on list only long enough to advance to the next proxy in the list.\n {\n scoped_lock lock(mutex(), /*is_writer=*/false);\n do {\n if (p) {\n // We were already processing the list.\n if (observer_proxy* q = p->my_next) {\n if (p == prev) {\n remove_ref_fast(prev); // sets prev to nullptr if successful\n }\n p = q;\n } else {\n // Reached the end of the list.\n if (p == prev) {\n // Keep the reference as we store the 'last' pointer in scheduler\n __TBB_ASSERT(int(p->my_ref_count.load(std::memory_order_relaxed)) >= 1 + (p->my_observer ? 1 : 0), nullptr);\n } else {\n // The last few proxies were empty\n __TBB_ASSERT(int(p->my_ref_count.load(std::memory_order_relaxed)), nullptr);\n ++p->my_ref_count;\n if (prev) {\n lock.release();\n remove_ref(prev);\n }\n }\n last = p;\n return;\n }\n } else {\n // Starting pass through the list\n p = my_head.load(std::memory_order_relaxed);\n if (!p) {\n return;\n }\n }\n tso = p->my_observer;\n } while (!tso);\n ++p->my_ref_count;\n ++tso->my_busy_count;\n }\n __TBB_ASSERT(!prev || p != prev, nullptr);\n // Release the proxy pinned before p\n if (prev) {\n remove_ref(prev);\n }\n // Do not hold any locks on the list while calling user's code.\n // Do not intercept any exceptions that may escape the callback so that\n // they are either handled by the TBB scheduler or passed to the debugger.\n tso->on_scheduler_entry(worker);\n __TBB_ASSERT(p->my_ref_count.load(std::memory_order_relaxed), nullptr);\n intptr_t bc = --tso->my_busy_count;\n __TBB_ASSERT_EX(bc >= 0, \"my_busy_count underflowed\");\n prev = p;\n }\n}\n\nvoid observer_list::do_notify_exit_observers(observer_proxy* last, bool worker) {\n // Pointer p marches though the list from the beginning to last (inclusively).\n observer_proxy* p = nullptr, * prev = nullptr;\n for (;;) {\n d1::task_scheduler_observer* tso = nullptr;\n // Hold lock on list only long enough to advance to the next proxy in the list.\n {\n scoped_lock lock(mutex(), /*is_writer=*/false);\n do {\n if (p) {\n // We were already processing the list.\n if (p != last) {\n __TBB_ASSERT(p->my_next, \"List items before 'last' must have valid my_next pointer\");\n if (p == prev)\n remove_ref_fast(prev); // sets prev to nullptr if successful\n p = p->my_next;\n } else {\n // remove the reference from the last item\n remove_ref_fast(p);\n if (p) {\n lock.release();\n if (p != prev && prev) {\n remove_ref(prev);\n }\n remove_ref(p);\n }\n return;\n }\n } else {\n // Starting pass through the list\n p = my_head.load(std::memory_order_relaxed);\n __TBB_ASSERT(p, \"Nonzero 'last' must guarantee that the global list is non-empty\");\n }\n tso = p->my_observer;\n } while (!tso);\n // The item is already refcounted\n if (p != last) // the last is already referenced since entry notification\n ++p->my_ref_count;\n ++tso->my_busy_count;\n }\n __TBB_ASSERT(!prev || p != prev, nullptr);\n if (prev)\n remove_ref(prev);\n // Do not hold any locks on the list while calling user's code.\n // Do not intercept any exceptions that may escape the callback so that\n // they are either handled by the TBB scheduler or passed to the debugger.\n tso->on_scheduler_exit(worker);\n __TBB_ASSERT(p->my_ref_count || p == last, nullptr);\n intptr_t bc = --tso->my_busy_count;\n __TBB_ASSERT_EX(bc >= 0, \"my_busy_count underflowed\");\n prev = p;\n }\n}\n\nvoid __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer &tso, bool enable) {\n if( enable ) {\n if( !tso.my_proxy.load(std::memory_order_relaxed) ) {\n observer_proxy* p = new observer_proxy(tso);\n tso.my_proxy.store(p, std::memory_order_relaxed);\n tso.my_busy_count.store(0, std::memory_order_relaxed);\n\n thread_data* td = governor::get_thread_data_if_initialized();\n if (p->my_observer->my_task_arena == nullptr) {\n if (!(td && td->my_arena)) {\n td = governor::get_thread_data();\n }\n __TBB_ASSERT(__TBB_InitOnce::initialization_done(), nullptr);\n __TBB_ASSERT(td && td->my_arena, nullptr);\n p->my_list = &td->my_arena->my_observers;\n } else {\n d1::task_arena* ta = p->my_observer->my_task_arena;\n arena* a = ta->my_arena.load(std::memory_order_acquire);\n if (a == nullptr) { // Avoid recursion during arena initialization\n ta->initialize();\n a = ta->my_arena.load(std::memory_order_relaxed);\n }\n __TBB_ASSERT(a != nullptr, nullptr);\n p->my_list = &a->my_observers;\n }\n p->my_list->insert(p);\n // Notify newly activated observer and other pending ones if it belongs to current arena\n if (td && td->my_arena && &td->my_arena->my_observers == p->my_list) {\n p->my_list->notify_entry_observers(td->my_last_observer, td->my_is_worker);\n }\n }\n } else {\n // Make sure that possible concurrent proxy list cleanup does not conflict\n // with the observer destruction here.\n if ( observer_proxy* proxy = tso.my_proxy.exchange(nullptr) ) {\n // List destruction should not touch this proxy after we've won the above interlocked exchange.\n __TBB_ASSERT( proxy->my_observer == &tso, nullptr);\n __TBB_ASSERT( is_alive(proxy->my_ref_count.load(std::memory_order_relaxed)), \"Observer's proxy died prematurely\" );\n __TBB_ASSERT( proxy->my_ref_count.load(std::memory_order_relaxed) >= 1, \"reference for observer missing\" );\n observer_list &list = *proxy->my_list;\n {\n // Ensure that none of the list walkers relies on observer pointer validity\n observer_list::scoped_lock lock(list.mutex(), /*is_writer=*/true);\n proxy->my_observer = nullptr;\n // Proxy may still be held by other threads (to track the last notified observer)\n if( !--proxy->my_ref_count ) {// nobody can increase it under exclusive lock\n list.remove(proxy);\n __TBB_ASSERT( !proxy->my_ref_count, nullptr);\n delete proxy;\n }\n }\n spin_wait_until_eq(tso.my_busy_count, 0); // other threads are still accessing the callback\n }\n }\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\nnamespace tbb {\nnamespace internal {\n\nvoid __TBB_EXPORTED_FUNC task_scheduler_observer_v3::observe( bool enable ) {\n auto* tso = (tbb::detail::d1::task_scheduler_observer*) (this);\n tbb::detail::r1::observe(*tso, enable);\n}\n\n} // namespace internal\n} // namespace tbb" }, { "path": "src/tbb/src/tbb/observer_proxy.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_observer_proxy_H\n#define __TBB_observer_proxy_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_aligned_space.h\"\n\n#include \"oneapi/tbb/task_scheduler_observer.h\"\n#include \"oneapi/tbb/spin_rw_mutex.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass observer_list {\n friend class arena;\n\n // Mutex is wrapped with aligned_space to shut up warnings when its destructor\n // is called while threads are still using it.\n typedef aligned_space my_mutex_type;\n\n //! Pointer to the head of this list.\n std::atomic my_head{nullptr};\n\n //! Pointer to the tail of this list.\n std::atomic my_tail{nullptr};\n\n //! Mutex protecting this list.\n my_mutex_type my_mutex;\n\n //! Back-pointer to the arena this list belongs to.\n arena* my_arena;\n\n //! Decrement refcount of the proxy p if there are other outstanding references.\n /** In case of success sets p to nullptr. Must be invoked from under the list lock. **/\n inline static void remove_ref_fast( observer_proxy*& p );\n\n //! Implements notify_entry_observers functionality.\n void do_notify_entry_observers( observer_proxy*& last, bool worker );\n\n //! Implements notify_exit_observers functionality.\n void do_notify_exit_observers( observer_proxy* last, bool worker );\n\npublic:\n observer_list () = default;\n\n //! Removes and destroys all observer proxies from the list.\n /** Cannot be used concurrently with other methods. **/\n void clear ();\n\n //! Add observer proxy to the tail of the list.\n void insert ( observer_proxy* p );\n\n //! Remove observer proxy from the list.\n void remove ( observer_proxy* p );\n\n //! Decrement refcount of the proxy and destroy it if necessary.\n /** When refcount reaches zero removes the proxy from the list and destructs it. **/\n void remove_ref( observer_proxy* p );\n\n //! Type of the scoped lock for the reader-writer mutex associated with the list.\n typedef spin_rw_mutex::scoped_lock scoped_lock;\n\n //! Accessor to the reader-writer mutex associated with the list.\n spin_rw_mutex& mutex () { return my_mutex.begin()[0]; }\n\n //! Call entry notifications on observers added after last was notified.\n /** Updates last to become the last notified observer proxy (in the global list)\n or leaves it to be nullptr. The proxy has its refcount incremented. **/\n inline void notify_entry_observers( observer_proxy*& last, bool worker );\n\n //! Call exit notifications on last and observers added before it.\n inline void notify_exit_observers( observer_proxy*& last, bool worker );\n}; // class observer_list\n\n//! Wrapper for an observer object\n/** To maintain shared lists of observers the scheduler first wraps each observer\n object into a proxy so that a list item remained valid even after the corresponding\n proxy object is destroyed by the user code. **/\nclass observer_proxy {\n friend class d1::task_scheduler_observer;\n friend class observer_list;\n friend void observe(d1::task_scheduler_observer&, bool);\n //! Reference count used for garbage collection.\n /** 1 for reference from my task_scheduler_observer.\n 1 for each task dispatcher's last observer pointer.\n No accounting for neighbors in the shared list. */\n std::atomic my_ref_count;\n //! Reference to the list this observer belongs to.\n observer_list* my_list;\n //! Pointer to next observer in the list specified by my_head.\n /** nullptr for the last item in the list. **/\n observer_proxy* my_next;\n //! Pointer to the previous observer in the list specified by my_head.\n /** For the head of the list points to the last item. **/\n observer_proxy* my_prev;\n //! Associated observer\n d1::task_scheduler_observer* my_observer;\n\n //! Constructs proxy for the given observer and adds it to the specified list.\n observer_proxy( d1::task_scheduler_observer& );\n\n ~observer_proxy();\n}; // class observer_proxy\n\nvoid observer_list::remove_ref_fast( observer_proxy*& p ) {\n if( p->my_observer ) {\n // Can decrement refcount quickly, as it cannot drop to zero while under the lock.\n std::uintptr_t r = --p->my_ref_count;\n __TBB_ASSERT_EX( r, nullptr);\n p = nullptr;\n } else {\n // Use slow form of refcount decrementing, after the lock is released.\n }\n}\n\nvoid observer_list::notify_entry_observers(observer_proxy*& last, bool worker) {\n if (last == my_tail.load(std::memory_order_relaxed))\n return;\n do_notify_entry_observers(last, worker);\n}\n\nvoid observer_list::notify_exit_observers( observer_proxy*& last, bool worker ) {\n if (last == nullptr) {\n return;\n }\n __TBB_ASSERT(!is_poisoned(last), nullptr);\n do_notify_exit_observers( last, worker );\n __TBB_ASSERT(last != nullptr, nullptr);\n poison_pointer(last);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_observer_proxy_H */\n" }, { "path": "src/tbb/src/tbb/parallel_pipeline.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/parallel_pipeline.h\"\n#include \"oneapi/tbb/spin_mutex.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"itt_notify.h\"\n#include \"tls.h\"\n#include \"oneapi/tbb/detail/_exception.h\"\n#include \"oneapi/tbb/detail/_small_object_pool.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nvoid handle_perror(int error_code, const char* aux_info);\n\nusing Token = unsigned long;\n\n//! A processing pipeline that applies filters to items.\n/** @ingroup algorithms */\nclass pipeline {\n friend void parallel_pipeline(d1::task_group_context&, std::size_t, const d1::filter_node&);\npublic:\n\n //! Construct empty pipeline.\n pipeline(d1::task_group_context& cxt, std::size_t max_token) :\n my_context(cxt),\n first_filter(nullptr),\n last_filter(nullptr),\n input_tokens(Token(max_token)),\n end_of_input(false),\n wait_ctx(0) {\n __TBB_ASSERT( max_token>0, \"pipeline::run must have at least one token\" );\n }\n\n ~pipeline();\n\n //! Add filter to end of pipeline.\n void add_filter( d1::base_filter& );\n\n //! Traverse tree of fitler-node in-order and add filter for each leaf\n void fill_pipeline(const d1::filter_node& root) {\n if( root.left && root.right ) {\n fill_pipeline(*root.left);\n fill_pipeline(*root.right);\n }\n else {\n __TBB_ASSERT(!root.left && !root.right, \"tree should be full\");\n add_filter(*root.create_filter());\n }\n }\n\nprivate:\n friend class stage_task;\n friend class base_filter;\n friend void set_end_of_input(d1::base_filter& bf);\n\n task_group_context& my_context;\n\n //! Pointer to first filter in the pipeline.\n d1::base_filter* first_filter;\n\n //! Pointer to last filter in the pipeline.\n d1::base_filter* last_filter;\n\n //! Number of idle tokens waiting for input stage.\n std::atomic input_tokens;\n\n //! False until flow_control::stop() is called.\n std::atomic end_of_input;\n\n d1::wait_context wait_ctx;\n};\n\n//! This structure is used to store task information in an input buffer\nstruct task_info {\n void* my_object = nullptr;\n //! Invalid unless a task went through an ordered stage.\n Token my_token = 0;\n //! False until my_token is set.\n bool my_token_ready = false;\n //! True if my_object is valid.\n bool is_valid = false;\n //! Set to initial state (no object, no token)\n void reset() {\n my_object = nullptr;\n my_token = 0;\n my_token_ready = false;\n is_valid = false;\n }\n};\n\n//! A buffer of input items for a filter.\n/** Each item is a task_info, inserted into a position in the buffer corresponding to a Token. */\nclass input_buffer {\n friend class base_filter;\n friend class stage_task;\n friend class pipeline;\n friend void set_end_of_input(d1::base_filter& bf);\n\n using size_type = Token;\n\n //! Array of deferred tasks that cannot yet start executing.\n task_info* array;\n\n //! Size of array\n /** Always 0 or a power of 2 */\n size_type array_size;\n\n //! Lowest token that can start executing.\n /** All prior Token have already been seen. */\n Token low_token;\n\n //! Serializes updates.\n spin_mutex array_mutex;\n\n //! Resize \"array\".\n /** Caller is responsible to acquiring a lock on \"array_mutex\". */\n void grow( size_type minimum_size );\n\n //! Initial size for \"array\"\n /** Must be a power of 2 */\n static const size_type initial_buffer_size = 4;\n\n //! Used for out of order buffer, and for assigning my_token if is_ordered and my_token not already assigned\n Token high_token;\n\n //! True for ordered filter, false otherwise.\n const bool is_ordered;\n\n //! for parallel filters that accepts nullptrs, thread-local flag for reaching end_of_input\n using end_of_input_tls_t = basic_tls;\n end_of_input_tls_t end_of_input_tls;\n bool end_of_input_tls_allocated; // no way to test pthread creation of TLS\n\npublic:\n input_buffer(const input_buffer&) = delete;\n input_buffer& operator=(const input_buffer&) = delete;\n\n //! Construct empty buffer.\n input_buffer( bool ordered) :\n array(nullptr),\n array_size(0),\n low_token(0),\n high_token(0),\n is_ordered(ordered),\n end_of_input_tls(),\n end_of_input_tls_allocated(false) {\n grow(initial_buffer_size);\n __TBB_ASSERT( array, nullptr );\n }\n\n //! Destroy the buffer.\n ~input_buffer() {\n __TBB_ASSERT( array, nullptr );\n cache_aligned_allocator().deallocate(array,array_size);\n poison_pointer( array );\n if( end_of_input_tls_allocated ) {\n destroy_my_tls();\n }\n }\n\n //! Define order when the first filter is serial_in_order.\n Token get_ordered_token(){\n return high_token++;\n }\n\n //! Put a token into the buffer.\n /** If task information was placed into buffer, returns true;\n otherwise returns false, informing the caller to create and spawn a task.\n */\n bool try_put_token( task_info& info ) {\n info.is_valid = true;\n spin_mutex::scoped_lock lock( array_mutex );\n Token token;\n if( is_ordered ) {\n if( !info.my_token_ready ) {\n info.my_token = high_token++;\n info.my_token_ready = true;\n }\n token = info.my_token;\n } else\n token = high_token++;\n __TBB_ASSERT( (long)(token-low_token)>=0, nullptr );\n if( token!=low_token ) {\n // Trying to put token that is beyond low_token.\n // Need to wait until low_token catches up before dispatching.\n if( token-low_token>=array_size )\n grow( token-low_token+1 );\n ITT_NOTIFY( sync_releasing, this );\n array[token&(array_size-1)] = info;\n return true;\n }\n return false;\n }\n\n //! Note that processing of a token is finished.\n /** Fires up processing of the next token, if processing was deferred. */\n // Uses template to avoid explicit dependency on stage_task.\n template\n void try_to_spawn_task_for_next_token(StageTask& spawner, d1::execution_data& ed) {\n task_info wakee;\n {\n spin_mutex::scoped_lock lock( array_mutex );\n // Wake the next task\n task_info& item = array[++low_token & (array_size-1)];\n ITT_NOTIFY( sync_acquired, this );\n wakee = item;\n item.is_valid = false;\n }\n if( wakee.is_valid )\n spawner.spawn_stage_task(wakee, ed);\n }\n\n // end_of_input signal for parallel_pipeline, parallel input filters with 0 tokens allowed.\n void create_my_tls() {\n int status = end_of_input_tls.create();\n if(status)\n handle_perror(status, \"TLS not allocated for filter\");\n end_of_input_tls_allocated = true;\n }\n void destroy_my_tls() {\n int status = end_of_input_tls.destroy();\n if(status)\n handle_perror(status, \"Failed to destroy filter TLS\");\n }\n bool my_tls_end_of_input() {\n return end_of_input_tls.get() != nullptr;\n }\n void set_my_tls_end_of_input() {\n end_of_input_tls.set(this);\n }\n};\n\nvoid input_buffer::grow( size_type minimum_size ) {\n size_type old_size = array_size;\n size_type new_size = old_size ? 2*old_size : initial_buffer_size;\n while( new_size().allocate(new_size);\n task_info* old_array = array;\n for( size_type i=0; i().deallocate(old_array,old_size);\n}\n\nclass stage_task : public d1::task, public task_info {\nprivate:\n friend class pipeline;\n pipeline& my_pipeline;\n d1::base_filter* my_filter;\n d1::small_object_allocator m_allocator;\n //! True if this task has not yet read the input.\n bool my_at_start;\n\n //! True if this can be executed again.\n bool execute_filter(d1::execution_data& ed);\n\n //! Spawn task if token is available.\n void try_spawn_stage_task(d1::execution_data& ed) {\n ITT_NOTIFY( sync_releasing, &my_pipeline.input_tokens );\n if( (my_pipeline.input_tokens.fetch_sub(1, std::memory_order_release)) > 1 ) {\n d1::small_object_allocator alloc{};\n r1::spawn( *alloc.new_object(ed, my_pipeline, alloc ), my_pipeline.my_context );\n }\n }\n\npublic:\n\n //! Construct stage_task for first stage in a pipeline.\n /** Such a stage has not read any input yet. */\n stage_task(pipeline& pipeline, d1::small_object_allocator& alloc ) :\n my_pipeline(pipeline),\n my_filter(pipeline.first_filter),\n m_allocator(alloc),\n my_at_start(true)\n {\n task_info::reset();\n my_pipeline.wait_ctx.reserve();\n }\n //! Construct stage_task for a subsequent stage in a pipeline.\n stage_task(pipeline& pipeline, d1::base_filter* filter, const task_info& info, d1::small_object_allocator& alloc) :\n task_info(info),\n my_pipeline(pipeline),\n my_filter(filter),\n m_allocator(alloc),\n my_at_start(false)\n {\n my_pipeline.wait_ctx.reserve();\n }\n //! Roughly equivalent to the constructor of input stage task\n void reset() {\n task_info::reset();\n my_filter = my_pipeline.first_filter;\n my_at_start = true;\n }\n void finalize(d1::execution_data& ed) {\n m_allocator.delete_object(this, ed);\n }\n //! The virtual task execution method\n task* execute(d1::execution_data& ed) override {\n if(!execute_filter(ed)) {\n finalize(ed);\n return nullptr;\n }\n return this;\n }\n task* cancel(d1::execution_data& ed) override {\n finalize(ed);\n return nullptr;\n }\n\n ~stage_task() override {\n if ( my_filter && my_object ) {\n my_filter->finalize(my_object);\n my_object = nullptr;\n }\n my_pipeline.wait_ctx.release();\n }\n //! Creates and spawns stage_task from task_info\n void spawn_stage_task(const task_info& info, d1::execution_data& ed) {\n d1::small_object_allocator alloc{};\n stage_task* clone = alloc.new_object(ed, my_pipeline, my_filter, info, alloc);\n r1::spawn(*clone, my_pipeline.my_context);\n }\n};\n\nbool stage_task::execute_filter(d1::execution_data& ed) {\n __TBB_ASSERT( !my_at_start || !my_object, \"invalid state of task\" );\n if( my_at_start ) {\n if( my_filter->is_serial() ) {\n my_object = (*my_filter)(my_object);\n if( my_object || ( my_filter->object_may_be_null() && !my_pipeline.end_of_input.load(std::memory_order_relaxed)) ) {\n if( my_filter->is_ordered() ) {\n my_token = my_filter->my_input_buffer->get_ordered_token();\n my_token_ready = true;\n }\n if( !my_filter->next_filter_in_pipeline ) { // we're only filter in pipeline\n reset();\n return true;\n } else {\n try_spawn_stage_task(ed);\n }\n } else {\n my_pipeline.end_of_input.store(true, std::memory_order_relaxed);\n return false;\n }\n } else /*not is_serial*/ {\n if ( my_pipeline.end_of_input.load(std::memory_order_relaxed) ) {\n return false;\n }\n\n try_spawn_stage_task(ed);\n\n my_object = (*my_filter)(my_object);\n if( !my_object && (!my_filter->object_may_be_null() || my_filter->my_input_buffer->my_tls_end_of_input()) ){\n my_pipeline.end_of_input.store(true, std::memory_order_relaxed);\n return false;\n }\n }\n my_at_start = false;\n } else {\n my_object = (*my_filter)(my_object);\n if( my_filter->is_serial() )\n my_filter->my_input_buffer->try_to_spawn_task_for_next_token(*this, ed);\n }\n my_filter = my_filter->next_filter_in_pipeline;\n if( my_filter ) {\n // There is another filter to execute.\n if( my_filter->is_serial() ) {\n // The next filter must execute tokens when they are available (in order for serial_in_order)\n if( my_filter->my_input_buffer->try_put_token(*this) ){\n my_filter = nullptr; // To prevent deleting my_object twice if exception occurs\n return false;\n }\n }\n } else {\n // Reached end of the pipe.\n std::size_t ntokens_avail = my_pipeline.input_tokens.fetch_add(1, std::memory_order_acquire);\n\n if( ntokens_avail>0 // Only recycle if there is one available token\n || my_pipeline.end_of_input.load(std::memory_order_relaxed) ) {\n return false; // No need to recycle for new input\n }\n ITT_NOTIFY( sync_acquired, &my_pipeline.input_tokens );\n // Recycle as an input stage task.\n reset();\n }\n return true;\n}\n\npipeline::~pipeline() {\n while( first_filter ) {\n d1::base_filter* f = first_filter;\n if( input_buffer* b = f->my_input_buffer ) {\n b->~input_buffer();\n deallocate_memory(b);\n }\n first_filter = f->next_filter_in_pipeline;\n f->~base_filter();\n deallocate_memory(f);\n }\n}\n\nvoid pipeline::add_filter( d1::base_filter& new_fitler ) {\n __TBB_ASSERT( new_fitler.next_filter_in_pipeline==d1::base_filter::not_in_pipeline(), \"filter already part of pipeline?\" );\n new_fitler.my_pipeline = this;\n if ( first_filter == nullptr )\n first_filter = &new_fitler;\n else\n last_filter->next_filter_in_pipeline = &new_fitler;\n new_fitler.next_filter_in_pipeline = nullptr;\n last_filter = &new_fitler;\n if( new_fitler.is_serial() ) {\n new_fitler.my_input_buffer = new (allocate_memory(sizeof(input_buffer))) input_buffer( new_fitler.is_ordered() );\n } else {\n if( first_filter == &new_fitler && new_fitler.object_may_be_null() ) {\n //TODO: buffer only needed to hold TLS; could improve\n new_fitler.my_input_buffer = new (allocate_memory(sizeof(input_buffer))) input_buffer( /*is_ordered*/false );\n new_fitler.my_input_buffer->create_my_tls();\n }\n }\n}\n\nvoid __TBB_EXPORTED_FUNC parallel_pipeline(d1::task_group_context& cxt, std::size_t max_token, const d1::filter_node& fn) {\n pipeline pipe(cxt, max_token);\n\n pipe.fill_pipeline(fn);\n\n d1::small_object_allocator alloc{};\n stage_task& st = *alloc.new_object(pipe, alloc);\n\n // Start execution of tasks\n r1::execute_and_wait(st, cxt, pipe.wait_ctx, cxt);\n}\n\nvoid __TBB_EXPORTED_FUNC set_end_of_input(d1::base_filter& bf) {\n __TBB_ASSERT(bf.my_input_buffer, nullptr);\n __TBB_ASSERT(bf.object_may_be_null(), nullptr);\n if(bf.is_serial() ) {\n bf.my_pipeline->end_of_input.store(true, std::memory_order_relaxed);\n } else {\n __TBB_ASSERT(bf.my_input_buffer->end_of_input_tls_allocated, nullptr);\n bf.my_input_buffer->set_my_tls_end_of_input();\n }\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/permit_manager.h", "content": "/*\n Copyright (c) 2022-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_permit_manager_H\n#define _TBB_permit_manager_H\n\n#include \"oneapi/tbb/info.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"thread_request_serializer.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass arena;\nclass pm_client;\n\nclass permit_manager : no_copy {\npublic:\n virtual ~permit_manager() {}\n virtual pm_client* create_client(arena& a) = 0;\n virtual void register_client(pm_client* client, d1::constraints& constraints) = 0;\n virtual void unregister_and_destroy_client(pm_client& c) = 0;\n\n virtual void set_active_num_workers(int soft_limit) = 0;\n virtual void adjust_demand(pm_client&, int mandatory_delta, int workers_delta) = 0;\n\n void set_thread_request_observer(thread_request_observer& tr_observer) {\n __TBB_ASSERT(!my_thread_request_observer, \"set_thread_request_observer was called already?\");\n my_thread_request_observer = &tr_observer;\n }\nprotected:\n void notify_thread_request(int delta) {\n __TBB_ASSERT(my_thread_request_observer, \"set_thread_request_observer was not called?\");\n if (delta) {\n my_thread_request_observer->update(delta);\n }\n }\nprivate:\n thread_request_observer* my_thread_request_observer{nullptr};\n};\n\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_permit_manager_H\n" }, { "path": "src/tbb/src/tbb/pm_client.h", "content": "/*\n Copyright (c) 2022-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_pm_client_H\n#define _TBB_pm_client_H\n\n#include \"arena.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass pm_client {\npublic:\n pm_client(arena& a) : my_arena(a) {}\n virtual ~pm_client() {}\n\n unsigned priority_level() {\n return my_arena.priority_level();\n }\n\n void set_top_priority(bool b) {\n my_arena.set_top_priority(b);\n }\n\n int min_workers() const {\n return my_min_workers;\n }\n\n int max_workers() const {\n return my_max_workers;\n }\n\n int update_request(int mandatory_delta, int workers_delta) {\n auto min_max_workers = my_arena.update_request(mandatory_delta, workers_delta);\n int delta = min_max_workers.second - my_max_workers;\n set_workers(min_max_workers.first, min_max_workers.second);\n return delta;\n }\n\n virtual void register_thread() = 0;\n\n virtual void unregister_thread() = 0;\n\n\nprotected:\n void set_workers(int mn_w, int mx_w) {\n __TBB_ASSERT(mn_w >= 0, nullptr);\n __TBB_ASSERT(mx_w >= 0, nullptr);\n my_min_workers = mn_w;\n my_max_workers = mx_w;\n }\n\n arena& my_arena;\n int my_min_workers{0};\n int my_max_workers{0};\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_pm_client_H\n" }, { "path": "src/tbb/src/tbb/private_server.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/mutex.h\"\n\n#include \"rml_tbb.h\"\n#include \"rml_thread_monitor.h\"\n\n#include \"scheduler_common.h\"\n#include \"governor.h\"\n#include \"misc.h\"\n\n#include \n\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\nnamespace rml {\n\nusing rml::internal::thread_monitor;\ntypedef thread_monitor::handle_type thread_handle;\n\nclass private_server;\n\nclass private_worker: no_copy {\nprivate:\n //! State in finite-state machine that controls the worker.\n /** State diagram:\n init --> starting --> normal\n | | |\n | V |\n \\------> quit <------/\n */\n enum state_t {\n //! *this is initialized\n st_init,\n //! *this has associated thread that is starting up.\n st_starting,\n //! Associated thread is doing normal life sequence.\n st_normal,\n //! Associated thread has ended normal life sequence and promises to never touch *this again.\n st_quit\n };\n std::atomic my_state;\n\n //! Associated server\n private_server& my_server;\n\n //! Associated client\n tbb_client& my_client;\n\n //! index used for avoiding the 64K aliasing problem\n const std::size_t my_index;\n\n //! Monitor for sleeping when there is no work to do.\n /** The invariant that holds for sleeping workers is:\n \"my_slack<=0 && my_state==st_normal && I am on server's list of asleep threads\" */\n thread_monitor my_thread_monitor;\n\n //! Handle of the OS thread associated with this worker\n thread_handle my_handle;\n\n //! Link for list of workers that are sleeping or have no associated thread.\n private_worker* my_next;\n\n friend class private_server;\n\n //! Actions executed by the associated thread\n void run() noexcept;\n\n //! Wake up associated thread (or launch a thread if there is none)\n void wake_or_launch();\n\n //! Called by a thread (usually not the associated thread) to commence termination.\n void start_shutdown();\n\n static __RML_DECL_THREAD_ROUTINE thread_routine( void* arg );\n\n static void release_handle(thread_handle my_handle, bool join);\n\nprotected:\n private_worker( private_server& server, tbb_client& client, const std::size_t i ) :\n my_state(st_init), my_server(server), my_client(client), my_index(i),\n my_handle(), my_next()\n {}\n};\n\nstatic const std::size_t cache_line_size = tbb::detail::max_nfs_size;\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Suppress overzealous compiler warnings about uninstantiable class\n // #pragma warning(push)\n // #pragma warning(disable:4510 4610)\n#endif\nclass padded_private_worker: public private_worker {\n char pad[cache_line_size - sizeof(private_worker)%cache_line_size];\npublic:\n padded_private_worker( private_server& server, tbb_client& client, const std::size_t i )\n : private_worker(server,client,i) { suppress_unused_warning(pad); }\n};\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning(pop)\n#endif\n\nclass private_server: public tbb_server, no_copy {\nprivate:\n tbb_client& my_client;\n //! Maximum number of threads to be created.\n /** Threads are created lazily, so maximum might not actually be reached. */\n const tbb_client::size_type my_n_thread;\n\n //! Stack size for each thread. */\n const std::size_t my_stack_size;\n\n //! Number of jobs that could use their associated thread minus number of active threads.\n /** If negative, indicates oversubscription.\n If positive, indicates that more threads should run.\n Can be lowered asynchronously, but must be raised only while holding my_asleep_list_mutex,\n because raising it impacts the invariant for sleeping threads. */\n std::atomic my_slack;\n\n //! Counter used to determine when to delete this.\n std::atomic my_ref_count;\n\n padded_private_worker* my_thread_array;\n\n //! List of workers that are asleep or committed to sleeping until notified by another thread.\n std::atomic my_asleep_list_root;\n\n //! Protects my_asleep_list_root\n typedef mutex asleep_list_mutex_type;\n asleep_list_mutex_type my_asleep_list_mutex;\n\n#if TBB_USE_ASSERT\n std::atomic my_net_slack_requests;\n#endif /* TBB_USE_ASSERT */\n\n //! Wake up to two sleeping workers, if there are any sleeping.\n /** The call is used to propagate a chain reaction where each thread wakes up two threads,\n which in turn each wake up two threads, etc. */\n void propagate_chain_reaction() {\n // First test of a double-check idiom. Second test is inside wake_some(0).\n if( my_asleep_list_root.load(std::memory_order_relaxed) )\n wake_some(0);\n }\n\n //! Try to add t to list of sleeping workers\n bool try_insert_in_asleep_list( private_worker& t );\n\n //! Equivalent of adding additional_slack to my_slack and waking up to 2 threads if my_slack permits.\n void wake_some( int additional_slack );\n\n ~private_server() override;\n\n void remove_server_ref() {\n if( --my_ref_count==0 ) {\n my_client.acknowledge_close_connection();\n this->~private_server();\n tbb::cache_aligned_allocator().deallocate( this, 1 );\n }\n }\n\n friend class private_worker;\npublic:\n private_server( tbb_client& client );\n\n version_type version() const override {\n return 0;\n }\n\n void request_close_connection( bool /*exiting*/ ) override {\n for( std::size_t i=0; i=2 && !__MINGW64__\n// ensure that stack is properly aligned for TBB threads\n__attribute__((force_align_arg_pointer))\n#endif\n__RML_DECL_THREAD_ROUTINE private_worker::thread_routine( void* arg ) {\n private_worker* self = static_cast(arg);\n AVOID_64K_ALIASING( self->my_index );\n self->run();\n // return 0 instead of nullptr due to the difference in the type __RML_DECL_THREAD_ROUTINE on various OSs\n return 0;\n}\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // #pragma warning(pop)\n#endif\n\nvoid private_worker::release_handle(thread_handle handle, bool join) {\n if (join)\n thread_monitor::join(handle);\n else\n thread_monitor::detach_thread(handle);\n}\n\nvoid private_worker::start_shutdown() {\n __TBB_ASSERT(my_state.load(std::memory_order_relaxed) != st_quit, \"The quit state is expected to be set only once\");\n\n // `acq` to acquire my_handle\n // `rel` to release market state\n state_t prev_state = my_state.exchange(st_quit, std::memory_order_acq_rel);\n\n if (prev_state == st_init) {\n // Perform action that otherwise would be performed by associated thread when it quits.\n my_server.remove_server_ref();\n } else {\n __TBB_ASSERT(prev_state == st_normal || prev_state == st_starting, nullptr);\n // May have invalidated invariant for sleeping, so wake up the thread.\n // Note that the notify() here occurs without maintaining invariants for my_slack.\n // It does not matter, because my_state==st_quit overrides checking of my_slack.\n my_thread_monitor.notify();\n // Do not need release handle in st_init state,\n // because in this case the thread wasn't started yet.\n // For st_starting release is done at launch site.\n if (prev_state == st_normal)\n release_handle(my_handle, governor::does_client_join_workers(my_client));\n }\n}\n\nvoid private_worker::run() noexcept {\n my_server.propagate_chain_reaction();\n\n // Transiting to st_normal here would require setting my_handle,\n // which would create race with the launching thread and\n // complications in handle management on Windows.\n\n ::rml::job& j = *my_client.create_one_job();\n // memory_order_seq_cst to be strictly ordered after thread_monitor::wait on the next iteration\n while( my_state.load(std::memory_order_seq_cst)!=st_quit ) {\n if( my_server.my_slack.load(std::memory_order_acquire)>=0 ) {\n my_client.process(j);\n } else if( my_server.try_insert_in_asleep_list(*this) ) {\n my_thread_monitor.wait();\n __TBB_ASSERT(my_state.load(std::memory_order_relaxed) == st_quit || !my_next, \"Thread monitor missed a spurious wakeup?\" );\n my_server.propagate_chain_reaction();\n }\n }\n my_client.cleanup(j);\n\n ++my_server.my_slack;\n my_server.remove_server_ref();\n}\n\ninline void private_worker::wake_or_launch() {\n state_t state = my_state.load(std::memory_order_relaxed);\n\n switch (state) {\n case st_starting:\n __TBB_fallthrough;\n case st_normal:\n __TBB_ASSERT(!my_next, \"Should not wake a thread while it's still in asleep list\");\n my_thread_monitor.notify();\n break;\n case st_init:\n if (my_state.compare_exchange_strong(state, st_starting)) {\n // after this point, remove_server_ref() must be done by created thread\n#if __TBB_USE_WINAPI\n // Win thread_monitor::launch is designed on the assumption that the workers thread id go from 1 to Hard limit set by TBB market::global_market\n const std::size_t worker_idx = my_server.my_n_thread - this->my_index; \n my_handle = thread_monitor::launch(thread_routine, this, my_server.my_stack_size, &worker_idx);\n#elif __TBB_USE_POSIX\n {\n affinity_helper fpa;\n fpa.protect_affinity_mask( /*restore_process_mask=*/true);\n my_handle = thread_monitor::launch(thread_routine, this, my_server.my_stack_size);\n // Implicit destruction of fpa resets original affinity mask.\n }\n#endif /* __TBB_USE_POSIX */\n state = st_starting;\n if (!my_state.compare_exchange_strong(state, st_normal)) {\n // Do shutdown during startup. my_handle can't be released\n // by start_shutdown, because my_handle value might be not set yet\n // at time of transition from st_starting to st_quit.\n __TBB_ASSERT(state == st_quit, nullptr);\n release_handle(my_handle, governor::does_client_join_workers(my_client));\n }\n }\n break;\n default:\n __TBB_ASSERT(state == st_quit, nullptr);\n }\n}\n\n//------------------------------------------------------------------------\n// Methods of private_server\n//------------------------------------------------------------------------\nprivate_server::private_server( tbb_client& client ) :\n my_client(client),\n my_n_thread(client.max_job_count()),\n my_stack_size(client.min_stack_size()),\n my_slack(0),\n my_ref_count(my_n_thread+1),\n my_thread_array(nullptr),\n my_asleep_list_root(nullptr)\n#if TBB_USE_ASSERT\n , my_net_slack_requests(0)\n#endif /* TBB_USE_ASSERT */\n{\n my_thread_array = tbb::cache_aligned_allocator().allocate( my_n_thread );\n for( std::size_t i=0; imy_next = my_asleep_list_root.load(std::memory_order_relaxed);\n my_asleep_list_root.store(t, std::memory_order_relaxed);\n }\n}\n\nprivate_server::~private_server() {\n __TBB_ASSERT( my_net_slack_requests==0, nullptr);\n for( std::size_t i=my_n_thread; i--; )\n my_thread_array[i].~padded_private_worker();\n tbb::cache_aligned_allocator().deallocate( my_thread_array, my_n_thread );\n tbb::detail::poison_pointer( my_thread_array );\n}\n\ninline bool private_server::try_insert_in_asleep_list( private_worker& t ) {\n asleep_list_mutex_type::scoped_lock lock;\n if( !lock.try_acquire(my_asleep_list_mutex) )\n return false;\n // Contribute to slack under lock so that if another takes that unit of slack,\n // it sees us sleeping on the list and wakes us up.\n auto expected = my_slack.load(std::memory_order_relaxed);\n while (expected < 0) {\n if (my_slack.compare_exchange_strong(expected, expected + 1)) {\n t.my_next = my_asleep_list_root.load(std::memory_order_relaxed);\n my_asleep_list_root.store(&t, std::memory_order_relaxed);\n return true;\n }\n }\n\n return false;\n}\n\nvoid private_server::wake_some( int additional_slack ) {\n __TBB_ASSERT( additional_slack>=0, nullptr );\n private_worker* wakee[2];\n private_worker**w = wakee;\n\n if (additional_slack) {\n // Contribute our unused slack to my_slack.\n my_slack += additional_slack;\n }\n\n int allotted_slack = 0;\n while (allotted_slack < 2) {\n // Chain reaction; Try to claim unit of slack\n int old = my_slack.load(std::memory_order_relaxed);\n do {\n if (old <= 0) goto done;\n } while (!my_slack.compare_exchange_strong(old, old - 1));\n ++allotted_slack;\n }\ndone:\n\n if (allotted_slack) {\n asleep_list_mutex_type::scoped_lock lock(my_asleep_list_mutex);\n auto root = my_asleep_list_root.load(std::memory_order_relaxed);\n while( root && wmy_next;\n }\n my_asleep_list_root.store(root, std::memory_order_relaxed);\n if(allotted_slack) {\n // Contribute our unused slack to my_slack.\n my_slack += allotted_slack;\n }\n }\n while( w>wakee ) {\n private_worker* ww = *--w;\n ww->my_next = nullptr;\n ww->wake_or_launch();\n }\n}\n\nvoid private_server::adjust_job_count_estimate( int delta ) {\n#if TBB_USE_ASSERT\n my_net_slack_requests+=delta;\n#endif /* TBB_USE_ASSERT */\n if( delta<0 ) {\n my_slack+=delta;\n } else if( delta>0 ) {\n wake_some( delta );\n }\n}\n\n//! Factory method called from task.cpp to create a private_server.\ntbb_server* make_private_server( tbb_client& client ) {\n return new( tbb::cache_aligned_allocator().allocate(1) ) private_server(client);\n}\n\n} // namespace rml\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n" }, { "path": "src/tbb/src/tbb/profiling.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n\n#include \"main.h\"\n#include \"itt_notify.h\"\n\n#include \"oneapi/tbb/profiling.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if __TBB_USE_ITT_NOTIFY\nbool ITT_Present;\nstatic std::atomic ITT_InitializationDone;\n\nstatic __itt_domain *tbb_domains[d1::ITT_NUM_DOMAINS] = {};\n\nstruct resource_string {\n const char *str;\n __itt_string_handle *itt_str_handle;\n};\n\n//\n// populate resource strings\n//\n#define TBB_STRING_RESOURCE( index_name, str ) { str, nullptr },\nstatic resource_string strings_for_itt[] = {\n #include \"oneapi/tbb/detail/_string_resource.h\"\n { \"num_resource_strings\", nullptr }\n};\n#undef TBB_STRING_RESOURCE\n\nstatic __itt_string_handle* ITT_get_string_handle(std::uintptr_t idx) {\n __TBB_ASSERT(idx < NUM_STRINGS, \"string handle out of valid range\");\n return idx < NUM_STRINGS ? strings_for_itt[idx].itt_str_handle : nullptr;\n}\n\nstatic void ITT_init_domains() {\n tbb_domains[d1::ITT_DOMAIN_MAIN] = __itt_domain_create( _T(\"tbb\") );\n tbb_domains[d1::ITT_DOMAIN_MAIN]->flags = 1;\n tbb_domains[d1::ITT_DOMAIN_FLOW] = __itt_domain_create( _T(\"tbb.flow\") );\n tbb_domains[d1::ITT_DOMAIN_FLOW]->flags = 1;\n tbb_domains[d1::ITT_DOMAIN_ALGO] = __itt_domain_create( _T(\"tbb.algorithm\") );\n tbb_domains[d1::ITT_DOMAIN_ALGO]->flags = 1;\n}\n\nstatic void ITT_init_strings() {\n for ( std::uintptr_t i = 0; i < NUM_STRINGS; ++i ) {\n#if _WIN32||_WIN64\n strings_for_itt[i].itt_str_handle = __itt_string_handle_createA( strings_for_itt[i].str );\n#else\n strings_for_itt[i].itt_str_handle = __itt_string_handle_create( strings_for_itt[i].str );\n#endif\n }\n}\n\nstatic void ITT_init() {\n ITT_init_domains();\n ITT_init_strings();\n}\n\n/** Thread-unsafe lazy one-time initialization of tools interop.\n Used by both dummy handlers and general TBB one-time initialization routine. **/\nvoid ITT_DoUnsafeOneTimeInitialization () {\n // Double check ITT_InitializationDone is necessary because the first check\n // in ITT_DoOneTimeInitialization is not guarded with the __TBB_InitOnce lock.\n if ( !ITT_InitializationDone ) {\n ITT_Present = (__TBB_load_ittnotify()!=0);\n if (ITT_Present) ITT_init();\n ITT_InitializationDone = true;\n }\n}\n\n/** Thread-safe lazy one-time initialization of tools interop.\n Used by dummy handlers only. **/\nextern \"C\"\nvoid ITT_DoOneTimeInitialization() {\n if ( !ITT_InitializationDone ) {\n __TBB_InitOnce::lock();\n ITT_DoUnsafeOneTimeInitialization();\n __TBB_InitOnce::unlock();\n }\n}\n\nvoid create_itt_sync(void* ptr, const tchar* objtype, const tchar* objname) {\n ITT_SYNC_CREATE(ptr, objtype, objname);\n}\n\nvoid call_itt_notify(int t, void *ptr) {\n switch (t) {\n case 0: ITT_NOTIFY(sync_prepare, ptr); break;\n case 1: ITT_NOTIFY(sync_cancel, ptr); break;\n case 2: ITT_NOTIFY(sync_acquired, ptr); break;\n case 3: ITT_NOTIFY(sync_releasing, ptr); break;\n case 4: ITT_NOTIFY(sync_destroy, ptr); break;\n }\n}\n\nvoid itt_set_sync_name(void* obj, const tchar* name) {\n __itt_sync_rename(obj, name);\n}\n\nconst __itt_id itt_null_id = { 0, 0, 0 };\n\nstatic inline __itt_domain* get_itt_domain(d1::itt_domain_enum idx) {\n if (tbb_domains[idx] == nullptr) {\n ITT_DoOneTimeInitialization();\n }\n return tbb_domains[idx];\n}\n\nstatic inline void itt_id_make(__itt_id* id, void* addr, unsigned long long extra) {\n *id = __itt_id_make(addr, extra);\n}\n\nstatic inline void itt_id_create(const __itt_domain* domain, __itt_id id) {\n __itt_id_create(domain, id);\n}\n\nvoid itt_make_task_group(d1::itt_domain_enum domain, void* group, unsigned long long group_extra,\n void* parent, unsigned long long parent_extra, string_resource_index name_index) {\n if (__itt_domain* d = get_itt_domain(domain)) {\n __itt_id group_id = itt_null_id;\n __itt_id parent_id = itt_null_id;\n itt_id_make(&group_id, group, group_extra);\n itt_id_create(d, group_id);\n if (parent) {\n itt_id_make(&parent_id, parent, parent_extra);\n }\n __itt_string_handle* n = ITT_get_string_handle(name_index);\n __itt_task_group(d, group_id, parent_id, n);\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_metadata_str_add(d1::itt_domain_enum domain, void *addr, unsigned long long addr_extra,\n string_resource_index key, const char *value ) {\n if ( __itt_domain *d = get_itt_domain( domain ) ) {\n __itt_id id = itt_null_id;\n itt_id_make( &id, addr, addr_extra );\n __itt_string_handle *k = ITT_get_string_handle(key);\n size_t value_length = strlen( value );\n#if _WIN32||_WIN64\n __itt_metadata_str_addA(d, id, k, value, value_length);\n#else\n __itt_metadata_str_add(d, id, k, value, value_length);\n#endif\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_metadata_ptr_add(d1::itt_domain_enum domain, void *addr, unsigned long long addr_extra,\n string_resource_index key, void *value ) {\n if ( __itt_domain *d = get_itt_domain( domain ) ) {\n __itt_id id = itt_null_id;\n itt_id_make( &id, addr, addr_extra );\n __itt_string_handle *k = ITT_get_string_handle(key);\n#if __TBB_x86_32\n __itt_metadata_add(d, id, k, __itt_metadata_u32, 1, value);\n#else\n __itt_metadata_add(d, id, k, __itt_metadata_u64, 1, value);\n#endif\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_relation_add(d1::itt_domain_enum domain, void *addr0, unsigned long long addr0_extra,\n itt_relation relation, void *addr1, unsigned long long addr1_extra ) {\n if ( __itt_domain *d = get_itt_domain( domain ) ) {\n __itt_id id0 = itt_null_id;\n __itt_id id1 = itt_null_id;\n itt_id_make( &id0, addr0, addr0_extra );\n itt_id_make( &id1, addr1, addr1_extra );\n __itt_relation_add( d, id0, (__itt_relation)relation, id1 );\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_task_begin(d1::itt_domain_enum domain, void* task, unsigned long long task_extra,\n void* parent, unsigned long long parent_extra, string_resource_index name_index) {\n if (__itt_domain* d = get_itt_domain(domain)) {\n __itt_id task_id = itt_null_id;\n __itt_id parent_id = itt_null_id;\n if (task) {\n itt_id_make(&task_id, task, task_extra);\n }\n if (parent) {\n itt_id_make(&parent_id, parent, parent_extra);\n }\n __itt_string_handle* n = ITT_get_string_handle(name_index);\n __itt_task_begin(d, task_id, parent_id, n);\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_task_end(d1::itt_domain_enum domain) {\n if (__itt_domain* d = get_itt_domain(domain)) {\n __itt_task_end(d);\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_region_begin(d1::itt_domain_enum domain, void *region, unsigned long long region_extra,\n void *parent, unsigned long long parent_extra, string_resource_index /* name_index */ ) {\n if ( __itt_domain *d = get_itt_domain( domain ) ) {\n __itt_id region_id = itt_null_id;\n __itt_id parent_id = itt_null_id;\n itt_id_make( ®ion_id, region, region_extra );\n if ( parent ) {\n itt_id_make( &parent_id, parent, parent_extra );\n }\n __itt_region_begin( d, region_id, parent_id, nullptr );\n }\n}\n\nvoid __TBB_EXPORTED_FUNC itt_region_end(d1::itt_domain_enum domain, void *region, unsigned long long region_extra ) {\n if ( __itt_domain *d = get_itt_domain( domain ) ) {\n __itt_id region_id = itt_null_id;\n itt_id_make( ®ion_id, region, region_extra );\n __itt_region_end( d, region_id );\n }\n}\n\n#else\nvoid create_itt_sync(void* /*ptr*/, const tchar* /*objtype*/, const tchar* /*objname*/) {}\nvoid call_itt_notify(int /*t*/, void* /*ptr*/) {}\nvoid itt_set_sync_name(void* /*obj*/, const tchar* /*name*/) {}\nvoid itt_make_task_group(d1::itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/,\n void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/) {}\nvoid itt_metadata_str_add(d1::itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/,\n string_resource_index /*key*/, const char* /*value*/ ) { }\nvoid itt_metadata_ptr_add(d1::itt_domain_enum /*domain*/, void * /*addr*/, unsigned long long /*addr_extra*/,\n string_resource_index /*key*/, void * /*value*/ ) {}\nvoid itt_relation_add(d1::itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/,\n itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) { }\nvoid itt_task_begin(d1::itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/,\n void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/ ) { }\nvoid itt_task_end(d1::itt_domain_enum /*domain*/ ) { }\nvoid itt_region_begin(d1::itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/,\n void* /*parent*/, unsigned long long /*parent_extra*/, string_resource_index /*name_index*/ ) { }\nvoid itt_region_end(d1::itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) { }\n#endif /* __TBB_USE_ITT_NOTIFY */\n\nconst tchar\n *SyncType_Scheduler = _T(\"%Constant\")\n ;\nconst tchar\n *SyncObj_ContextsList = _T(\"TBB Scheduler\")\n ;\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/queuing_rw_mutex.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n/** Before making any changes in the implementation, please emulate algorithmic changes\n with SPIN tool using /tools/spin_models/ReaderWriterMutex.pml.\n There could be some code looking as \"can be restructured\" but its structure does matter! */\n\n#include \"oneapi/tbb/queuing_rw_mutex.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"itt_notify.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Workaround for overzealous compiler warnings\n // #pragma warning (push)\n // #pragma warning (disable: 4311 4312)\n#endif\n\n//! A view of a T* with additional functionality for twiddling low-order bits.\ntemplate\nclass tricky_atomic_pointer {\npublic:\n using word = uintptr_t;\n\n static T* fetch_add( std::atomic& location, word addend, std::memory_order memory_order ) {\n return reinterpret_cast(location.fetch_add(addend, memory_order));\n }\n\n static T* exchange( std::atomic& location, T* value, std::memory_order memory_order ) {\n return reinterpret_cast(location.exchange(reinterpret_cast(value), memory_order));\n }\n\n static T* compare_exchange_strong( std::atomic& obj, const T* expected, const T* desired, std::memory_order memory_order ) {\n word expd = reinterpret_cast(expected);\n obj.compare_exchange_strong(expd, reinterpret_cast(desired), memory_order);\n return reinterpret_cast(expd);\n }\n\n static void store( std::atomic& location, const T* value, std::memory_order memory_order ) {\n location.store(reinterpret_cast(value), memory_order);\n }\n\n static T* load( std::atomic& location, std::memory_order memory_order ) {\n return reinterpret_cast(location.load(memory_order));\n }\n\n static void spin_wait_while_eq(const std::atomic& location, const T* value) {\n tbb::detail::d0::spin_wait_while_eq(location, reinterpret_cast(value) );\n }\n\n T* & ref;\n tricky_atomic_pointer( T*& original ) : ref(original) {};\n tricky_atomic_pointer(const tricky_atomic_pointer&) = delete;\n tricky_atomic_pointer& operator=(const tricky_atomic_pointer&) = delete;\n T* operator&( const word operand2 ) const {\n return reinterpret_cast( reinterpret_cast(ref) & operand2 );\n }\n T* operator|( const word operand2 ) const {\n return reinterpret_cast( reinterpret_cast(ref) | operand2 );\n }\n};\n\nusing tricky_pointer = tricky_atomic_pointer;\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Workaround for overzealous compiler warnings\n // #pragma warning (pop)\n#endif\n\n//! Flag bits in a state_t that specify information about a locking request.\nenum state_t_flags : unsigned char {\n STATE_NONE = 0,\n STATE_WRITER = 1<<0,\n STATE_READER = 1<<1,\n STATE_READER_UNBLOCKNEXT = 1<<2,\n STATE_ACTIVEREADER = 1<<3,\n STATE_UPGRADE_REQUESTED = 1<<4,\n STATE_UPGRADE_WAITING = 1<<5,\n STATE_UPGRADE_LOSER = 1<<6,\n STATE_COMBINED_WAITINGREADER = STATE_READER | STATE_READER_UNBLOCKNEXT,\n STATE_COMBINED_READER = STATE_COMBINED_WAITINGREADER | STATE_ACTIVEREADER,\n STATE_COMBINED_UPGRADING = STATE_UPGRADE_WAITING | STATE_UPGRADE_LOSER\n};\n\nstatic const unsigned char RELEASED = 0;\nstatic const unsigned char ACQUIRED = 1;\n\nstruct queuing_rw_mutex_impl {\n //! Try to acquire the internal lock\n /** Returns true if lock was successfully acquired. */\n static bool try_acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s)\n {\n auto expected = RELEASED;\n return s.my_internal_lock.compare_exchange_strong(expected, ACQUIRED);\n }\n\n //! Acquire the internal lock\n static void acquire_internal_lock(d1::queuing_rw_mutex::scoped_lock& s)\n {\n // Usually, we would use the test-test-and-set idiom here, with exponential backoff.\n // But so far, experiments indicate there is no value in doing so here.\n while( !try_acquire_internal_lock(s) ) {\n machine_pause(1);\n }\n }\n\n //! Release the internal lock\n static void release_internal_lock(d1::queuing_rw_mutex::scoped_lock& s)\n {\n s.my_internal_lock.store(RELEASED, std::memory_order_release);\n }\n\n //! Wait for internal lock to be released\n static void wait_for_release_of_internal_lock(d1::queuing_rw_mutex::scoped_lock& s)\n {\n spin_wait_until_eq(s.my_internal_lock, RELEASED);\n }\n\n //! A helper function\n static void unblock_or_wait_on_internal_lock(d1::queuing_rw_mutex::scoped_lock& s, uintptr_t flag ) {\n if( flag ) {\n wait_for_release_of_internal_lock(s);\n }\n else {\n release_internal_lock(s);\n }\n }\n\n //! Mask for low order bit of a pointer.\n static const tricky_pointer::word FLAG = 0x1;\n\n static uintptr_t get_flag( d1::queuing_rw_mutex::scoped_lock* ptr ) {\n return reinterpret_cast(ptr) & FLAG;\n }\n\n //------------------------------------------------------------------------\n // Methods of queuing_rw_mutex::scoped_lock\n //------------------------------------------------------------------------\n\n //! A method to acquire queuing_rw_mutex lock\n static void acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write)\n {\n __TBB_ASSERT( !s.my_mutex, \"scoped_lock is already holding a mutex\");\n\n // Must set all fields before the exchange, because once the\n // exchange executes, *this becomes accessible to other threads.\n s.my_mutex = &m;\n s.my_prev.store(0U, std::memory_order_relaxed);\n s.my_next.store(0U, std::memory_order_relaxed);\n s.my_going.store(0U, std::memory_order_relaxed);\n s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_READER), std::memory_order_relaxed);\n s.my_internal_lock.store(RELEASED, std::memory_order_relaxed);\n\n\n // The CAS must have release semantics, because we are\n // \"sending\" the fields initialized above to other actors.\n // We need acquire semantics, because we are acquiring the predecessor (or mutex if no predecessor)\n queuing_rw_mutex::scoped_lock* predecessor = m.q_tail.exchange(&s, std::memory_order_acq_rel);\n\n if( write ) { // Acquiring for write\n\n if( predecessor ) {\n ITT_NOTIFY(sync_prepare, s.my_mutex);\n predecessor = tricky_pointer(predecessor) & ~FLAG;\n __TBB_ASSERT( !predecessor->my_next, \"the predecessor has another successor!\");\n tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release);\n // We are acquiring the mutex\n spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire);\n }\n\n } else { // Acquiring for read\n #if __TBB_USE_ITT_NOTIFY\n bool sync_prepare_done = false;\n #endif\n if( predecessor ) {\n unsigned char pred_state{};\n __TBB_ASSERT( !s.my_prev.load(std::memory_order_relaxed), \"the predecessor is already set\" );\n if( tricky_pointer(predecessor) & FLAG ) {\n /* this is only possible if predecessor is an upgrading reader and it signals us to wait */\n pred_state = STATE_UPGRADE_WAITING;\n predecessor = tricky_pointer(predecessor) & ~FLAG;\n } else {\n // Load predecessor->my_state now, because once predecessor->my_next becomes\n // non-null, we must assume that *predecessor might be destroyed.\n pred_state = predecessor->my_state.load(std::memory_order_relaxed);\n if (pred_state == STATE_READER) {\n // Notify the previous reader to unblock us.\n predecessor->my_state.compare_exchange_strong(pred_state, STATE_READER_UNBLOCKNEXT, std::memory_order_relaxed);\n }\n if (pred_state == STATE_ACTIVEREADER) { // either we initially read it or CAS failed\n // Active reader means that the predecessor already acquired the mutex and cannot notify us.\n // Therefore, we need to acquire the mutex ourselves by re-reading predecessor state.\n (void)predecessor->my_state.load(std::memory_order_acquire);\n }\n }\n tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);\n __TBB_ASSERT( !( tricky_pointer(predecessor) & FLAG ), \"use of corrupted pointer!\" );\n __TBB_ASSERT( !predecessor->my_next.load(std::memory_order_relaxed), \"the predecessor has another successor!\");\n tricky_pointer::store(predecessor->my_next, &s, std::memory_order_release);\n if( pred_state != STATE_ACTIVEREADER ) {\n #if __TBB_USE_ITT_NOTIFY\n sync_prepare_done = true;\n ITT_NOTIFY(sync_prepare, s.my_mutex);\n #endif\n // We are acquiring the mutex\n spin_wait_until_eq(s.my_going, 1U, std::memory_order_acquire);\n }\n }\n\n // The protected state must have been acquired here before it can be further released to any other reader(s):\n unsigned char old_state = STATE_READER;\n // When this reader is signaled by previous actor it acquires the mutex.\n // We need to build happens-before relation with all other coming readers that will read our ACTIVEREADER\n // without blocking on my_going. Therefore, we need to publish ACTIVEREADER with release semantics.\n // On fail it is relaxed, because we will build happens-before on my_going.\n s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memory_order_relaxed);\n if( old_state!=STATE_READER ) {\n#if __TBB_USE_ITT_NOTIFY\n if( !sync_prepare_done )\n ITT_NOTIFY(sync_prepare, s.my_mutex);\n#endif\n // Failed to become active reader -> need to unblock the next waiting reader first\n __TBB_ASSERT( s.my_state.load(std::memory_order_relaxed)==STATE_READER_UNBLOCKNEXT, \"unexpected state\" );\n spin_wait_while_eq(s.my_next, 0U, std::memory_order_acquire);\n /* my_state should be changed before unblocking the next otherwise it might finish\n and another thread can get our old state and left blocked */\n s.my_state.store(STATE_ACTIVEREADER, std::memory_order_relaxed);\n tricky_pointer::load(s.my_next, std::memory_order_relaxed)->my_going.store(1U, std::memory_order_release);\n }\n __TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, \"unlocked reader is active reader\");\n }\n\n ITT_NOTIFY(sync_acquired, s.my_mutex);\n }\n\n //! A method to acquire queuing_rw_mutex if it is free\n static bool try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write)\n {\n __TBB_ASSERT( !s.my_mutex, \"scoped_lock is already holding a mutex\");\n\n if( m.q_tail.load(std::memory_order_relaxed) )\n return false; // Someone already took the lock\n\n // Must set all fields before the exchange, because once the\n // exchange executes, *this becomes accessible to other threads.\n s.my_prev.store(0U, std::memory_order_relaxed);\n s.my_next.store(0U, std::memory_order_relaxed);\n s.my_going.store(0U, std::memory_order_relaxed); // TODO: remove dead assignment?\n s.my_state.store(d1::queuing_rw_mutex::scoped_lock::state_t(write ? STATE_WRITER : STATE_ACTIVEREADER), std::memory_order_relaxed);\n s.my_internal_lock.store(RELEASED, std::memory_order_relaxed);\n\n // The CAS must have release semantics, because we are\n // \"sending\" the fields initialized above to other actors.\n // We need acquire semantics, because we are acquiring the mutex\n d1::queuing_rw_mutex::scoped_lock* expected = nullptr;\n if (!m.q_tail.compare_exchange_strong(expected, &s, std::memory_order_acq_rel))\n return false; // Someone already took the lock\n s.my_mutex = &m;\n ITT_NOTIFY(sync_acquired, s.my_mutex);\n return true;\n }\n\n //! A method to release queuing_rw_mutex lock\n static void release(d1::queuing_rw_mutex::scoped_lock& s) {\n __TBB_ASSERT(s.my_mutex!=nullptr, \"no lock acquired\");\n\n ITT_NOTIFY(sync_releasing, s.my_mutex);\n\n if( s.my_state.load(std::memory_order_relaxed) == STATE_WRITER ) { // Acquired for write\n\n // The logic below is the same as \"writerUnlock\", but elides\n // \"return\" from the middle of the routine.\n // In the statement below, acquire semantics of reading my_next is required\n // so that following operations with fields of my_next are safe.\n d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n if( !next ) {\n d1::queuing_rw_mutex::scoped_lock* expected = &s;\n // Release mutex on success otherwise wait for successor publication\n if( s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr,\n std::memory_order_release, std::memory_order_relaxed) )\n {\n // this was the only item in the queue, and the queue is now empty.\n goto done;\n }\n spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed);\n next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n }\n next->my_going.store(2U, std::memory_order_relaxed); // protect next queue node from being destroyed too early\n // If the next is STATE_UPGRADE_WAITING, it is expected to acquire all other released readers via release\n // sequence in next->my_state. In that case, we need to preserve release sequence in next->my_state\n // contributed by other reader. So, there are two approaches not to break the release sequence:\n // 1. Use read-modify-write (exchange) operation to store with release the UPGRADE_LOSER state;\n // 2. Acquire the release sequence and store the sequence and UPGRADE_LOSER state.\n // The second approach seems better on x86 because it does not involve interlocked operations.\n // Therefore, we read next->my_state with acquire while it is not required for else branch to get the \n // release sequence.\n if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING ) {\n // the next waiting for upgrade means this writer was upgraded before.\n acquire_internal_lock(s);\n // Responsibility transition, the one who reads uncorrupted my_prev will do release.\n // Guarantee that above store of 2 into next->my_going happens-before resetting of next->my_prev\n d1::queuing_rw_mutex::scoped_lock* tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release);\n // Pass the release sequence that we acquired with the above load of next->my_state.\n next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release);\n // We are releasing the mutex\n next->my_going.store(1U, std::memory_order_release);\n unblock_or_wait_on_internal_lock(s, get_flag(tmp));\n } else {\n // next->state cannot be STATE_UPGRADE_REQUESTED\n __TBB_ASSERT( next->my_state.load(std::memory_order_relaxed) & (STATE_COMBINED_WAITINGREADER | STATE_WRITER), \"unexpected state\" );\n __TBB_ASSERT( !( next->my_prev.load(std::memory_order_relaxed) & FLAG ), \"use of corrupted pointer!\" );\n // Guarantee that above store of 2 into next->my_going happens-before resetting of next->my_prev\n tricky_pointer::store(next->my_prev, nullptr, std::memory_order_release);\n // We are releasing the mutex\n next->my_going.store(1U, std::memory_order_release);\n }\n\n } else { // Acquired for read\n // The basic idea it to build happens-before relation with left and right readers via prev and next. In addition,\n // the first reader should acquire the left (prev) signal and propagate to right (next). To simplify, we always\n // build happens-before relation between left and right (left is happened before right).\n queuing_rw_mutex::scoped_lock *tmp = nullptr;\n retry:\n // Addition to the original paper: Mark my_prev as in use\n queuing_rw_mutex::scoped_lock *predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire);\n\n if( predecessor ) {\n if( !(try_acquire_internal_lock(*predecessor)) )\n {\n // Failed to acquire the lock on predecessor. The predecessor either unlinks or upgrades.\n // In the second case, it could or could not know my \"in use\" flag - need to check\n // Responsibility transition, the one who reads uncorrupted my_prev will do release.\n tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor) | FLAG, predecessor, std::memory_order_acquire);\n if( !(tricky_pointer(tmp) & FLAG) ) {\n __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed) != (tricky_pointer(predecessor) | FLAG), nullptr);\n // Now owner of predecessor is waiting for _us_ to release its lock\n release_internal_lock(*predecessor);\n }\n // else the \"in use\" flag is back -> the predecessor didn't get it and will release itself; nothing to do\n\n tmp = nullptr;\n goto retry;\n }\n __TBB_ASSERT(predecessor && predecessor->my_internal_lock.load(std::memory_order_relaxed)==ACQUIRED, \"predecessor's lock is not acquired\");\n tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);\n acquire_internal_lock(s);\n\n tricky_pointer::store(predecessor->my_next, nullptr, std::memory_order_release);\n\n d1::queuing_rw_mutex::scoped_lock* expected = &s;\n if( !tricky_pointer::load(s.my_next, std::memory_order_acquire) && !s.my_mutex->q_tail.compare_exchange_strong(expected, predecessor, std::memory_order_release) ) {\n spin_wait_while_eq( s.my_next, 0U, std::memory_order_acquire );\n }\n __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), \"use of corrupted pointer\" );\n\n // my_next is acquired either with load or spin_wait.\n if(d1::queuing_rw_mutex::scoped_lock *const l_next = tricky_pointer::load(s.my_next, std::memory_order_relaxed) ) { // I->next != nil, TODO: rename to next after clearing up and adapting the n in the comment two lines below\n // Equivalent to I->next->prev = I->prev but protected against (prev[n]&FLAG)!=0\n tmp = tricky_pointer::exchange(l_next->my_prev, predecessor, std::memory_order_release);\n // I->prev->next = I->next;\n __TBB_ASSERT(tricky_pointer::load(s.my_prev, std::memory_order_relaxed)==predecessor, nullptr);\n predecessor->my_next.store(s.my_next.load(std::memory_order_relaxed), std::memory_order_release);\n }\n // Safe to release in the order opposite to acquiring which makes the code simpler\n release_internal_lock(*predecessor);\n\n } else { // No predecessor when we looked\n acquire_internal_lock(s); // \"exclusiveLock(&I->EL)\"\n d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n if( !next ) {\n d1::queuing_rw_mutex::scoped_lock* expected = &s;\n // Release mutex on success otherwise wait for successor publication\n if( !s.my_mutex->q_tail.compare_exchange_strong(expected, nullptr,\n std::memory_order_release, std::memory_order_relaxed) )\n {\n spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed );\n next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n } else {\n goto unlock_self;\n }\n }\n next->my_going.store(2U, std::memory_order_relaxed);\n // Responsibility transition, the one who reads uncorrupted my_prev will do release.\n tmp = tricky_pointer::exchange(next->my_prev, nullptr, std::memory_order_release);\n next->my_going.store(1U, std::memory_order_release);\n }\n unlock_self:\n unblock_or_wait_on_internal_lock(s, get_flag(tmp));\n }\n done:\n // Lifetime synchronization, no need to build happens-before relation\n spin_wait_while_eq( s.my_going, 2U, std::memory_order_relaxed );\n\n s.initialize();\n }\n\n static bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) {\n if ( s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER ) return true; // Already a reader\n\n ITT_NOTIFY(sync_releasing, s.my_mutex);\n d1::queuing_rw_mutex::scoped_lock* next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n if( !next ) {\n s.my_state.store(STATE_READER, std::memory_order_seq_cst);\n // the following load of q_tail must not be reordered with setting STATE_READER above\n if( &s == s.my_mutex->q_tail.load(std::memory_order_seq_cst) ) {\n unsigned char old_state = STATE_READER;\n // When this reader is signaled by previous actor it acquires the mutex.\n // We need to build happens-before relation with all other coming readers that will read our ACTIVEREADER\n // without blocking on my_going. Therefore, we need to publish ACTIVEREADER with release semantics.\n // On fail it is relaxed, because we will build happens-before on my_going.\n s.my_state.compare_exchange_strong(old_state, STATE_ACTIVEREADER, std::memory_order_release, std::memory_order_relaxed);\n if( old_state==STATE_READER )\n return true; // Downgrade completed\n }\n /* wait for the next to register */\n spin_wait_while_eq(s.my_next, 0U, std::memory_order_relaxed);\n next = tricky_pointer::load(s.my_next, std::memory_order_acquire);\n }\n\n __TBB_ASSERT( next, \"still no successor at this point!\" );\n if( next->my_state.load(std::memory_order_relaxed) & STATE_COMBINED_WAITINGREADER )\n next->my_going.store(1U, std::memory_order_release);\n // If the next is STATE_UPGRADE_WAITING, it is expected to acquire all other released readers via release\n // sequence in next->my_state. In that case, we need to preserve release sequence in next->my_state\n // contributed by other reader. So, there are two approaches not to break the release sequence:\n // 1. Use read-modify-write (exchange) operation to store with release the UPGRADE_LOSER state;\n // 2. Acquire the release sequence and store the sequence and UPGRADE_LOSER state.\n // The second approach seems better on x86 because it does not involve interlocked operations.\n // Therefore, we read next->my_state with acquire while it is not required for else branch to get the \n // release sequence.\n else if( next->my_state.load(std::memory_order_acquire)==STATE_UPGRADE_WAITING )\n // the next waiting for upgrade means this writer was upgraded before.\n // To safe release sequence on next->my_state read it with acquire\n next->my_state.store(STATE_UPGRADE_LOSER, std::memory_order_release);\n s.my_state.store(STATE_ACTIVEREADER, std::memory_order_release);\n return true;\n }\n\n static bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) {\n if (s.my_state.load(std::memory_order_relaxed) == STATE_WRITER) {\n // Already a writer\n return true;\n }\n\n __TBB_ASSERT(s.my_state.load(std::memory_order_relaxed) == STATE_ACTIVEREADER, \"only active reader can be updated\");\n\n queuing_rw_mutex::scoped_lock* tmp{};\n queuing_rw_mutex::scoped_lock* me = &s;\n\n ITT_NOTIFY(sync_releasing, s.my_mutex);\n // Publish ourselves into my_state that other UPGRADE_WAITING actors can acquire our state.\n s.my_state.store(STATE_UPGRADE_REQUESTED, std::memory_order_release);\n requested:\n __TBB_ASSERT( !(s.my_next.load(std::memory_order_relaxed) & FLAG), \"use of corrupted pointer!\" );\n acquire_internal_lock(s);\n d1::queuing_rw_mutex::scoped_lock* expected = &s;\n if( !s.my_mutex->q_tail.compare_exchange_strong(expected, tricky_pointer(me)|FLAG, std::memory_order_acq_rel) ) {\n spin_wait_while_eq( s.my_next, 0U, std::memory_order_relaxed );\n queuing_rw_mutex::scoped_lock * next;\n next = tricky_pointer::fetch_add(s.my_next, FLAG, std::memory_order_acquire);\n // While we were READER the next READER might reach STATE_UPGRADE_WAITING state.\n // Therefore, it did not build happens before relation with us and we need to acquire the \n // next->my_state to build the happens before relation ourselves\n unsigned short n_state = next->my_state.load(std::memory_order_acquire);\n /* the next reader can be blocked by our state. the best thing to do is to unblock it */\n if( n_state & STATE_COMBINED_WAITINGREADER )\n next->my_going.store(1U, std::memory_order_release);\n // Responsibility transition, the one who reads uncorrupted my_prev will do release.\n tmp = tricky_pointer::exchange(next->my_prev, &s, std::memory_order_release);\n unblock_or_wait_on_internal_lock(s, get_flag(tmp));\n if( n_state & (STATE_COMBINED_READER | STATE_UPGRADE_REQUESTED) ) {\n // save next|FLAG for simplicity of following comparisons\n tmp = tricky_pointer(next)|FLAG;\n for( atomic_backoff b; tricky_pointer::load(s.my_next, std::memory_order_relaxed)==tmp; b.pause() ) {\n if( s.my_state.load(std::memory_order_acquire) & STATE_COMBINED_UPGRADING ) {\n if( tricky_pointer::load(s.my_next, std::memory_order_acquire)==tmp )\n tricky_pointer::store(s.my_next, next, std::memory_order_relaxed);\n goto waiting;\n }\n }\n __TBB_ASSERT(tricky_pointer::load(s.my_next, std::memory_order_relaxed) != (tricky_pointer(next)|FLAG), nullptr);\n goto requested;\n } else {\n __TBB_ASSERT( n_state & (STATE_WRITER | STATE_UPGRADE_WAITING), \"unexpected state\");\n __TBB_ASSERT( (tricky_pointer(next)|FLAG) == tricky_pointer::load(s.my_next, std::memory_order_relaxed), nullptr);\n tricky_pointer::store(s.my_next, next, std::memory_order_relaxed);\n }\n } else {\n /* We are in the tail; whoever comes next is blocked by q_tail&FLAG */\n release_internal_lock(s);\n } // if( this != my_mutex->q_tail... )\n {\n unsigned char old_state = STATE_UPGRADE_REQUESTED;\n // If we reach STATE_UPGRADE_WAITING state we do not build happens-before relation with READER on\n // left. We delegate this responsibility to READER on left when it try upgrading. Therefore, we are releasing\n // on success.\n // Otherwise, on fail, we already acquired the next->my_state.\n s.my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release, std::memory_order_relaxed);\n }\n waiting:\n __TBB_ASSERT( !( s.my_next.load(std::memory_order_relaxed) & FLAG ), \"use of corrupted pointer!\" );\n __TBB_ASSERT( s.my_state & STATE_COMBINED_UPGRADING, \"wrong state at upgrade waiting_retry\" );\n __TBB_ASSERT( me==&s, nullptr );\n ITT_NOTIFY(sync_prepare, s.my_mutex);\n /* if no one was blocked by the \"corrupted\" q_tail, turn it back */\n expected = tricky_pointer(me)|FLAG;\n s.my_mutex->q_tail.compare_exchange_strong(expected, &s, std::memory_order_release);\n queuing_rw_mutex::scoped_lock * predecessor;\n // Mark my_prev as 'in use' to prevent predecessor from releasing\n predecessor = tricky_pointer::fetch_add(s.my_prev, FLAG, std::memory_order_acquire);\n if( predecessor ) {\n bool success = try_acquire_internal_lock(*predecessor);\n {\n // While the predecessor pointer (my_prev) is in use (FLAG is set), we can safely update the node`s state.\n // Corrupted pointer transitions responsibility to release the predecessor`s node on us.\n unsigned char old_state = STATE_UPGRADE_REQUESTED;\n // Try to build happens before with the upgrading READER on left. If fail, the predecessor state is not\n // important for us because it will acquire our state.\n predecessor->my_state.compare_exchange_strong(old_state, STATE_UPGRADE_WAITING, std::memory_order_release,\n std::memory_order_relaxed);\n }\n if( !success ) {\n // Responsibility transition, the one who reads uncorrupted my_prev will do release.\n tmp = tricky_pointer::compare_exchange_strong(s.my_prev, tricky_pointer(predecessor)|FLAG, predecessor, std::memory_order_acquire);\n if( tricky_pointer(tmp) & FLAG ) {\n tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor);\n predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed);\n } else {\n // TODO: spin_wait condition seems never reachable\n tricky_pointer::spin_wait_while_eq(s.my_prev, tricky_pointer(predecessor)|FLAG);\n release_internal_lock(*predecessor);\n }\n } else {\n tricky_pointer::store(s.my_prev, predecessor, std::memory_order_relaxed);\n release_internal_lock(*predecessor);\n tricky_pointer::spin_wait_while_eq(s.my_prev, predecessor);\n predecessor = tricky_pointer::load(s.my_prev, std::memory_order_relaxed);\n }\n if( predecessor )\n goto waiting;\n } else {\n tricky_pointer::store(s.my_prev, nullptr, std::memory_order_relaxed);\n }\n __TBB_ASSERT( !predecessor && !s.my_prev, nullptr );\n\n // additional lifetime issue prevention checks\n // wait for the successor to finish working with my fields\n wait_for_release_of_internal_lock(s);\n // now wait for the predecessor to finish working with my fields\n spin_wait_while_eq( s.my_going, 2U );\n\n bool result = ( s.my_state != STATE_UPGRADE_LOSER );\n s.my_state.store(STATE_WRITER, std::memory_order_relaxed);\n s.my_going.store(1U, std::memory_order_relaxed);\n\n ITT_NOTIFY(sync_acquired, s.my_mutex);\n return result;\n }\n\n static bool is_writer(const d1::queuing_rw_mutex::scoped_lock& m) {\n return m.my_state.load(std::memory_order_relaxed) == STATE_WRITER;\n }\n\n static void construct(d1::queuing_rw_mutex& m) {\n suppress_unused_warning(m);\n ITT_SYNC_CREATE(&m, _T(\"tbb::queuing_rw_mutex\"), _T(\"\"));\n }\n};\n\nvoid __TBB_EXPORTED_FUNC acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) {\n queuing_rw_mutex_impl::acquire(m, s, write);\n}\n\nbool __TBB_EXPORTED_FUNC try_acquire(d1::queuing_rw_mutex& m, d1::queuing_rw_mutex::scoped_lock& s, bool write) {\n return queuing_rw_mutex_impl::try_acquire(m, s, write);\n}\n\nvoid __TBB_EXPORTED_FUNC release(d1::queuing_rw_mutex::scoped_lock& s) {\n queuing_rw_mutex_impl::release(s);\n}\n\nbool __TBB_EXPORTED_FUNC upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock& s) {\n return queuing_rw_mutex_impl::upgrade_to_writer(s);\n}\n\nbool __TBB_EXPORTED_FUNC is_writer(const d1::queuing_rw_mutex::scoped_lock& s) {\n return queuing_rw_mutex_impl::is_writer(s);\n}\n\nbool __TBB_EXPORTED_FUNC downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock& s) {\n return queuing_rw_mutex_impl::downgrade_to_reader(s);\n}\n\nvoid __TBB_EXPORTED_FUNC construct(d1::queuing_rw_mutex& m) {\n queuing_rw_mutex_impl::construct(m);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/rml_base.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// Header guard and namespace names follow rml conventions.\n\n#ifndef __RML_rml_base_H\n#define __RML_rml_base_H\n\n#include \n\n#if _WIN32||_WIN64\n#include \n#endif /* _WIN32||_WIN64 */\n\n#ifdef RML_PURE_VIRTUAL_HANDLER\n#define RML_PURE(T) {RML_PURE_VIRTUAL_HANDLER(); return (T)0;}\n#else\n#define RML_PURE(T) = 0;\n#endif\n\nnamespace rml {\n\nclass server;\n\nclass versioned_object {\npublic:\n //! A version number\n typedef unsigned version_type;\n\n virtual ~versioned_object() {}\n\n //! Get version of this object\n /** The version number is incremented when a incompatible change is introduced.\n The version number is invariant for the lifetime of the object. */\n virtual version_type version() const RML_PURE(version_type)\n\n};\n\n//! Represents a client's job for an execution context.\n/** A job object is constructed by the client.\n Not derived from versioned_object because version is same as for client. */\nclass job {\n friend class server;\n};\n\n//! Information that client provides to server when asking for a server.\n/** The instance must endure at least until acknowledge_close_connection is called. */\nclass client: public versioned_object {\npublic:\n //! Typedef for convenience of derived classes in other namespaces.\n typedef ::rml::job job;\n\n //! Index of a job in a job pool\n typedef unsigned size_type;\n\n //! Maximum number of threads that client can exploit profitably if nothing else is running on the machine.\n /** The returned value should remain invariant for the lifetime of the connection. [idempotent] */\n virtual size_type max_job_count() const RML_PURE(size_type)\n\n //! Minimum stack size for each job. 0 means to use default stack size. [idempotent]\n virtual std::size_t min_stack_size() const RML_PURE(std::size_t)\n\n //! Server calls this routine when it needs client to create a job object.\n virtual job* create_one_job() RML_PURE(job*)\n\n //! Acknowledge that all jobs have been cleaned up.\n /** Called by server in response to request_close_connection\n after cleanup(job) has been called for each job. */\n virtual void acknowledge_close_connection() RML_PURE(void)\n\n //! Inform client that server is done with *this.\n /** Client should destroy the job.\n Not necessarily called by execution context represented by *this.\n Never called while any other thread is working on the job. */\n virtual void cleanup( job& ) RML_PURE(void)\n\n // In general, we should not add new virtual methods, because that would\n // break derived classes. Think about reserving some vtable slots.\n};\n\n// Information that server provides to client.\n// Virtual functions are routines provided by the server for the client to call.\nclass server: public versioned_object {\npublic:\n //! Typedef for convenience of derived classes.\n typedef ::rml::job job;\n\n#if _WIN32||_WIN64\n typedef void* execution_resource_t;\n#endif\n\n //! Request that connection to server be closed.\n /** Causes each job associated with the client to have its cleanup method called,\n possibly by a thread different than the thread that created the job.\n This method can return before all cleanup methods return.\n Actions that have to wait after all cleanup methods return should be part of\n client::acknowledge_close_connection.\n Pass true as exiting if request_close_connection() is called because exit() is\n called. In that case, it is the client's responsibility to make sure all threads\n are terminated. In all other cases, pass false. */\n virtual void request_close_connection( bool exiting = false ) = 0;\n\n //! Called by client thread when it reaches a point where it cannot make progress until other threads do.\n virtual void yield() = 0;\n\n //! Called by client to indicate a change in the number of non-RML threads that are running.\n /** This is a performance hint to the RML to adjust how many threads it should let run\n concurrently. The delta is the change in the number of non-RML threads that are running.\n For example, a value of 1 means the client has started running another thread, and a value\n of -1 indicates that the client has blocked or terminated one of its threads. */\n virtual void independent_thread_number_changed( int delta ) = 0;\n\n //! Default level of concurrency for which RML strives when there are no non-RML threads running.\n /** Normally, the value is the hardware concurrency minus one.\n The \"minus one\" accounts for the thread created by main(). */\n virtual unsigned default_concurrency() const = 0;\n};\n\nclass factory {\npublic:\n //! status results\n enum status_type {\n st_success=0,\n st_connection_exists,\n st_not_found,\n st_incompatible\n };\n\nprotected:\n //! Pointer to routine that waits for server to indicate when client can close itself.\n status_type (*my_wait_to_close_routine)( factory& );\n\npublic:\n //! Library handle for use by RML.\n#if _WIN32||_WIN64\n HMODULE library_handle;\n#else\n void* library_handle;\n#endif /* _WIN32||_WIN64 */\n\n //! Special marker to keep dll from being unloaded prematurely\n static const std::size_t c_dont_unload = 1;\n};\n\n//! Typedef for callback functions to print server info\ntypedef void (*server_info_callback_t)( void* arg, const char* server_info );\n\n} // namespace rml\n\n#endif /* __RML_rml_base_H */\n" }, { "path": "src/tbb/src/tbb/rml_tbb.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_assert.h\"\n\n#include \"rml_tbb.h\"\n#include \"dynamic_link.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\nnamespace rml {\n\n#define MAKE_SERVER(x) DLD(__TBB_make_rml_server,x)\n#define GET_INFO(x) DLD(__TBB_call_with_my_server_info,x)\n#define SERVER tbb_server\n#define CLIENT tbb_client\n#define FACTORY tbb_factory\n\n#if __TBB_WEAK_SYMBOLS_PRESENT\n #pragma weak __TBB_make_rml_server\n #pragma weak __TBB_call_with_my_server_info\n extern \"C\" {\n ::rml::factory::status_type __TBB_make_rml_server( rml::tbb_factory& f, rml::tbb_server*& server, rml::tbb_client& client );\n void __TBB_call_with_my_server_info( ::rml::server_info_callback_t cb, void* arg );\n }\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\n#if TBB_USE_DEBUG\n#define DEBUG_SUFFIX \"_debug\"\n#else\n#define DEBUG_SUFFIX\n#endif /* TBB_USE_DEBUG */\n\n// RML_SERVER_NAME is the name of the RML server library.\n#if _WIN32 || _WIN64\n#define RML_SERVER_NAME \"irml\" DEBUG_SUFFIX \".dll\"\n#elif __APPLE__\n#define RML_SERVER_NAME \"libirml\" DEBUG_SUFFIX \".1.dylib\"\n#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX\n#define RML_SERVER_NAME \"libirml\" DEBUG_SUFFIX \".so\"\n#elif __unix__\n#define RML_SERVER_NAME \"libirml\" DEBUG_SUFFIX \".so.1\"\n#else\n#error Unknown OS\n#endif\n\nconst ::rml::versioned_object::version_type CLIENT_VERSION = 2;\n\n#if __TBB_WEAK_SYMBOLS_PRESENT\n #pragma weak __RML_open_factory\n #pragma weak __RML_close_factory\n extern \"C\" {\n ::rml::factory::status_type __RML_open_factory ( ::rml::factory&, ::rml::versioned_object::version_type&, ::rml::versioned_object::version_type );\n void __RML_close_factory( ::rml::factory& f );\n }\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\n::rml::factory::status_type FACTORY::open() {\n // Failure of following assertion indicates that factory is already open, or not zero-inited.\n __TBB_ASSERT_EX( !library_handle, nullptr);\n status_type (*open_factory_routine)( factory&, version_type&, version_type );\n dynamic_link_descriptor server_link_table[4] = {\n DLD(__RML_open_factory,open_factory_routine),\n MAKE_SERVER(my_make_server_routine),\n DLD(__RML_close_factory,my_wait_to_close_routine),\n GET_INFO(my_call_with_server_info_routine),\n };\n status_type result;\n if ( dynamic_link( RML_SERVER_NAME, server_link_table, 4, &library_handle ) ) {\n version_type server_version;\n result = (*open_factory_routine)( *this, server_version, CLIENT_VERSION );\n // server_version can be checked here for incompatibility if necessary.\n } else {\n library_handle = nullptr;\n result = st_not_found;\n }\n return result;\n}\n\nvoid FACTORY::close() {\n if ( library_handle )\n (*my_wait_to_close_routine)(*this);\n if ( (size_t)library_handle>FACTORY::c_dont_unload ) {\n dynamic_unlink(library_handle);\n library_handle = nullptr;\n }\n}\n\n::rml::factory::status_type FACTORY::make_server( SERVER*& s, CLIENT& c) {\n // Failure of following assertion means that factory was not successfully opened.\n __TBB_ASSERT_EX( my_make_server_routine, nullptr);\n return (*my_make_server_routine)(*this,s,c);\n}\n\n} // namespace rml\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/rml_tbb.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// Header guard and namespace names follow TBB conventions.\n\n#ifndef __TBB_rml_tbb_H\n#define __TBB_rml_tbb_H\n\n#include \"oneapi/tbb/version.h\"\n#include \"rml_base.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\nnamespace rml {\n\n//------------------------------------------------------------------------\n// Classes instantiated by the server\n//------------------------------------------------------------------------\n\n//! Represents a set of oneTBB worker threads provided by the server.\nclass tbb_server: public ::rml::server {\npublic:\n //! Inform server of adjustments in the number of workers that the client can profitably use.\n virtual void adjust_job_count_estimate( int delta ) = 0;\n\n#if _WIN32 || _WIN64\n //! Inform server of a oneTBB external thread.\n virtual void register_external_thread( execution_resource_t& v ) = 0;\n\n //! Inform server that the oneTBB external thread is done with its work.\n virtual void unregister_external_thread( execution_resource_t v ) = 0;\n#endif /* _WIN32||_WIN64 */\n};\n\n//------------------------------------------------------------------------\n// Classes instantiated by the client\n//------------------------------------------------------------------------\n\nclass tbb_client: public ::rml::client {\npublic:\n //! Defined by TBB to steal a task and execute it. \n /** Called by server when it wants an execution context to do some TBB work.\n The method should return when it is okay for the thread to yield indefinitely. */\n virtual void process( job& ) RML_PURE(void)\n};\n\n/** Client must ensure that instance is zero-inited, typically by being a file-scope object. */\nclass tbb_factory: public ::rml::factory {\n\n //! Pointer to routine that creates an RML server.\n status_type (*my_make_server_routine)( tbb_factory&, tbb_server*&, tbb_client& );\n\n //! Pointer to routine that calls callback function with server version info.\n void (*my_call_with_server_info_routine)( ::rml::server_info_callback_t cb, void* arg );\n\npublic:\n typedef ::rml::versioned_object::version_type version_type;\n typedef tbb_client client_type;\n typedef tbb_server server_type;\n\n //! Open factory.\n /** Dynamically links against RML library. \n Returns st_success, st_incompatible, or st_not_found. */\n status_type open();\n\n //! Factory method to be called by client to create a server object.\n /** Factory must be open. \n Returns st_success, or st_incompatible . */\n status_type make_server( server_type*&, client_type& );\n\n //! Close factory\n void close();\n};\n\n} // namespace rml\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /*__TBB_rml_tbb_H */\n" }, { "path": "src/tbb/src/tbb/rml_thread_monitor.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// All platform-specific threading support is encapsulated here. */\n\n#ifndef __RML_thread_monitor_H\n#define __RML_thread_monitor_H\n\n#if __TBB_USE_WINAPI\n#include \n#include \n#include //_alloca\n#include \"misc.h\" // support for processor groups\n#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\n#include \n#endif\n#elif __TBB_USE_POSIX\n#include \n#include \n#include \n#include \n#else\n#error Unsupported platform\n#endif\n#include \n\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n\n#include \"itt_notify.h\"\n#include \"semaphore.h\"\n\n// All platform-specific threading support is in this header.\n\n#if (_WIN32||_WIN64)&&!__TBB_ipf\n// Deal with 64K aliasing. The formula for \"offset\" is a Fibonacci hash function,\n// which has the desirable feature of spreading out the offsets fairly evenly\n// without knowing the total number of offsets, and furthermore unlikely to\n// accidentally cancel out other 64K aliasing schemes that Microsoft might implement later.\n// See Knuth Vol 3. \"Theorem S\" for details on Fibonacci hashing.\n// The second statement is really does need \"volatile\", otherwise the compiler might remove the _alloca.\n#define AVOID_64K_ALIASING(idx) \\\n std::size_t offset = (idx+1) * 40503U % (1U<<16); \\\n void* volatile sink_for_alloca = _alloca(offset); \\\n __TBB_ASSERT_EX(sink_for_alloca, \"_alloca failed\");\n#else\n// Linux thread allocators avoid 64K aliasing.\n#define AVOID_64K_ALIASING(idx) tbb::detail::suppress_unused_warning(idx)\n#endif /* _WIN32||_WIN64 */\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n// Forward declaration: throws std::runtime_error with what() returning error_code description prefixed with aux_info\nvoid handle_perror(int error_code, const char* aux_info);\n\nnamespace rml {\nnamespace internal {\n\n#if __TBB_USE_ITT_NOTIFY\nstatic const ::tbb::detail::r1::tchar *SyncType_RML = _T(\"%Constant\");\nstatic const ::tbb::detail::r1::tchar *SyncObj_ThreadMonitor = _T(\"RML Thr Monitor\");\n#endif /* __TBB_USE_ITT_NOTIFY */\n\n//! Monitor with limited two-phase commit form of wait.\n/** At most one thread should wait on an instance at a time. */\nclass thread_monitor {\npublic:\n thread_monitor() {\n ITT_SYNC_CREATE(&my_sema, SyncType_RML, SyncObj_ThreadMonitor);\n }\n ~thread_monitor() {}\n\n //! Notify waiting thread\n /** Can be called by any thread. */\n void notify();\n\n //! Wait for notification\n void wait();\n\n#if __TBB_USE_WINAPI\n typedef HANDLE handle_type;\n\n #define __RML_DECL_THREAD_ROUTINE unsigned WINAPI\n typedef unsigned (WINAPI *thread_routine_type)(void*);\n\n //! Launch a thread\n static handle_type launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size, const size_t* worker_index = nullptr );\n\n#elif __TBB_USE_POSIX\n typedef pthread_t handle_type;\n\n #define __RML_DECL_THREAD_ROUTINE void*\n typedef void*(*thread_routine_type)(void*);\n\n //! Launch a thread\n static handle_type launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size );\n#endif /* __TBB_USE_POSIX */\n\n //! Join thread\n static void join(handle_type handle);\n\n //! Detach thread\n static void detach_thread(handle_type handle);\nprivate:\n // The protection from double notification of the binary semaphore\n std::atomic my_notified{ false };\n binary_semaphore my_sema;\n#if __TBB_USE_POSIX\n static void check( int error_code, const char* routine );\n#endif\n};\n\n#if __TBB_USE_WINAPI\n\n#ifndef STACK_SIZE_PARAM_IS_A_RESERVATION\n#define STACK_SIZE_PARAM_IS_A_RESERVATION 0x00010000\n#endif\n\n// _beginthreadex API is not available in Windows 8 Store* applications, so use std::thread instead\n#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\ninline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_function, void* arg, std::size_t, const std::size_t*) {\n//TODO: check that exception thrown from std::thread is not swallowed silently\n std::thread* thread_tmp=new std::thread(thread_function, arg);\n return thread_tmp->native_handle();\n}\n#else\ninline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_routine, void* arg, std::size_t stack_size, const std::size_t* worker_index ) {\n unsigned thread_id;\n int number_of_processor_groups = ( worker_index ) ? NumberOfProcessorGroups() : 0;\n unsigned create_flags = ( number_of_processor_groups > 1 ) ? CREATE_SUSPENDED : 0;\n HANDLE h = (HANDLE)_beginthreadex( nullptr, unsigned(stack_size), thread_routine, arg, STACK_SIZE_PARAM_IS_A_RESERVATION | create_flags, &thread_id );\n if( !h ) {\n handle_perror(0, \"thread_monitor::launch: _beginthreadex failed\\n\");\n }\n if ( number_of_processor_groups > 1 ) {\n MoveThreadIntoProcessorGroup( h, FindProcessorGroupIndex( static_cast(*worker_index) ) );\n ResumeThread( h );\n }\n return h;\n}\n#endif //__TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00)\n\nvoid thread_monitor::join(handle_type handle) {\n#if TBB_USE_ASSERT\n DWORD res =\n#endif\n WaitForSingleObjectEx(handle, INFINITE, FALSE);\n __TBB_ASSERT( res==WAIT_OBJECT_0, nullptr);\n#if TBB_USE_ASSERT\n BOOL val =\n#endif\n CloseHandle(handle);\n __TBB_ASSERT( val, nullptr);\n}\n\nvoid thread_monitor::detach_thread(handle_type handle) {\n#if TBB_USE_ASSERT\n BOOL val =\n#endif\n CloseHandle(handle);\n __TBB_ASSERT( val, nullptr);\n}\n\n#endif /* __TBB_USE_WINAPI */\n\n#if __TBB_USE_POSIX\ninline void thread_monitor::check( int error_code, const char* routine ) {\n if( error_code ) {\n handle_perror(error_code, routine);\n }\n}\n\ninline thread_monitor::handle_type thread_monitor::launch( void* (*thread_routine)(void*), void* arg, std::size_t stack_size ) {\n // FIXME - consider more graceful recovery than just exiting if a thread cannot be launched.\n // Note that there are some tricky situations to deal with, such that the thread is already\n // grabbed as part of an OpenMP team.\n pthread_attr_t s;\n check(pthread_attr_init( &s ), \"pthread_attr_init has failed\");\n if( stack_size>0 )\n check(pthread_attr_setstacksize( &s, stack_size ), \"pthread_attr_setstack_size has failed\" );\n\n // pthread_create(2) can spuriously fail with EAGAIN. We retry\n // max_num_tries times with progressively longer wait times.\n pthread_t handle;\n const int max_num_tries = 20;\n int error = EAGAIN;\n\n for (int i = 0; i < max_num_tries && error == EAGAIN; i++) {\n if (i != 0) {\n // Wait i milliseconds\n struct timespec ts = {0, i * 1000 * 1000};\n nanosleep(&ts, NULL);\n }\n error = pthread_create(&handle, &s, thread_routine, arg);\n }\n\n if (error)\n handle_perror(error, \"pthread_create has failed\");\n\n check( pthread_attr_destroy( &s ), \"pthread_attr_destroy has failed\" );\n return handle;\n}\n\nvoid thread_monitor::join(handle_type handle) {\n check(pthread_join(handle, nullptr), \"pthread_join has failed\");\n}\n\nvoid thread_monitor::detach_thread(handle_type handle) {\n check(pthread_detach(handle), \"pthread_detach has failed\");\n}\n#endif /* __TBB_USE_POSIX */\n\ninline void thread_monitor::notify() {\n // Check that the semaphore is not notified twice\n if (!my_notified.exchange(true, std::memory_order_release)) {\n my_sema.V();\n }\n}\n\ninline void thread_monitor::wait() {\n my_sema.P();\n // memory_order_seq_cst is required here to be ordered with\n // further load checking shutdown state\n my_notified.store(false, std::memory_order_seq_cst);\n}\n\n} // namespace internal\n} // namespace rml\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __RML_thread_monitor_H */\n" }, { "path": "src/tbb/src/tbb/rtm_mutex.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_rtm_mutex.h\"\n#include \"itt_notify.h\"\n#include \"governor.h\"\n#include \"misc.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n\nstruct rtm_mutex_impl {\n // maximum number of times to retry\n // TODO: experiment on retry values.\n static constexpr int retry_threshold = 10;\n using transaction_result_type = decltype(begin_transaction());\n\n //! Release speculative mutex\n static void release(d1::rtm_mutex::scoped_lock& s) {\n switch(s.m_transaction_state) {\n case d1::rtm_mutex::rtm_state::rtm_transacting:\n __TBB_ASSERT(is_in_transaction(), \"m_transaction_state && not speculating\");\n end_transaction();\n s.m_mutex = nullptr;\n break;\n case d1::rtm_mutex::rtm_state::rtm_real:\n s.m_mutex->unlock();\n s.m_mutex = nullptr;\n break;\n case d1::rtm_mutex::rtm_state::rtm_none:\n __TBB_ASSERT(false, \"mutex is not locked, but in release\");\n break;\n default:\n __TBB_ASSERT(false, \"invalid m_transaction_state\");\n }\n s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_none;\n }\n\n //! Acquire lock on the given mutex.\n static void acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) {\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, \"scoped_lock already in transaction\");\n if(governor::speculation_enabled()) {\n int num_retries = 0;\n transaction_result_type abort_code = 0;\n do {\n if(m.m_flag.load(std::memory_order_acquire)) {\n if(only_speculate) return;\n spin_wait_while_eq(m.m_flag, true);\n }\n // _xbegin returns -1 on success or the abort code, so capture it\n if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin))\n {\n // started speculation\n if(m.m_flag.load(std::memory_order_relaxed)) {\n abort_transaction();\n }\n s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_transacting;\n // Don not wrap the following assignment to a function,\n // because it can abort the transaction in debug. Need mutex for release().\n s.m_mutex = &m;\n return; // successfully started speculation\n }\n ++num_retries;\n } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold));\n }\n\n if(only_speculate) return;\n s.m_mutex = &m;\n s.m_mutex->lock();\n s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real;\n }\n\n //! Try to acquire lock on the given mutex.\n static bool try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) {\n acquire(m, s, /*only_speculate=*/true);\n if (s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_transacting) {\n return true;\n }\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, nullptr);\n // transacting acquire failed. try_lock the real mutex\n if (m.try_lock()) {\n s.m_mutex = &m;\n s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real;\n return true;\n }\n return false;\n }\n};\n\nvoid __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) {\n rtm_mutex_impl::acquire(m, s, only_speculate);\n}\nbool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) {\n return rtm_mutex_impl::try_acquire(m, s);\n}\nvoid __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock& s) {\n rtm_mutex_impl::release(s);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n" }, { "path": "src/tbb/src/tbb/rtm_rw_mutex.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_rtm_rw_mutex.h\"\n#include \"itt_notify.h\"\n#include \"governor.h\"\n#include \"misc.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct rtm_rw_mutex_impl {\n // maximum number of times to retry\n // TODO: experiment on retry values.\n static constexpr int retry_threshold_read = 10;\n static constexpr int retry_threshold_write = 10;\n using transaction_result_type = decltype(begin_transaction());\n\n //! Release speculative mutex\n static void release(d1::rtm_rw_mutex::scoped_lock& s) {\n switch(s.m_transaction_state) {\n case d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer:\n case d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader:\n __TBB_ASSERT(is_in_transaction(), \"m_transaction_state && not speculating\");\n end_transaction();\n s.m_mutex = nullptr;\n break;\n case d1::rtm_rw_mutex::rtm_type::rtm_real_reader:\n __TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), \"write_flag set but read lock acquired\");\n s.m_mutex->unlock_shared();\n s.m_mutex = nullptr;\n break;\n case d1::rtm_rw_mutex::rtm_type::rtm_real_writer:\n __TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), \"write_flag unset but write lock acquired\");\n s.m_mutex->write_flag.store(false, std::memory_order_relaxed);\n s.m_mutex->unlock();\n s.m_mutex = nullptr;\n break;\n case d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex:\n __TBB_ASSERT(false, \"rtm_not_in_mutex, but in release\");\n break;\n default:\n __TBB_ASSERT(false, \"invalid m_transaction_state\");\n }\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex;\n }\n\n //! Acquire write lock on the given mutex.\n static void acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) {\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, \"scoped_lock already in transaction\");\n if(governor::speculation_enabled()) {\n int num_retries = 0;\n transaction_result_type abort_code = 0;\n do {\n if(m.m_state.load(std::memory_order_acquire)) {\n if(only_speculate) return;\n spin_wait_until_eq(m.m_state, d1::rtm_rw_mutex::state_type(0));\n }\n // _xbegin returns -1 on success or the abort code, so capture it\n if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin))\n {\n // started speculation\n if(m.m_state.load(std::memory_order_relaxed)) { // add spin_rw_mutex to read-set.\n // reader or writer grabbed the lock, so abort.\n abort_transaction();\n }\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer;\n // Don not wrap the following assignment to a function,\n // because it can abort the transaction in debug. Need mutex for release().\n s.m_mutex = &m;\n return; // successfully started speculation\n }\n ++num_retries;\n } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write));\n }\n\n if(only_speculate) return;\n s.m_mutex = &m; // should apply a real try_lock...\n s.m_mutex->lock(); // kill transactional writers\n __TBB_ASSERT(!m.write_flag.load(std::memory_order_relaxed), \"After acquire for write, write_flag already true\");\n m.write_flag.store(true, std::memory_order_relaxed); // kill transactional readers\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;\n }\n\n //! Acquire read lock on given mutex.\n // only_speculate : true if we are doing a try_acquire. If true and we fail to speculate, don't\n // really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex. If\n // the lock is already held by a writer, just return.\n static void acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) {\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, \"scoped_lock already in transaction\");\n if(governor::speculation_enabled()) {\n int num_retries = 0;\n transaction_result_type abort_code = 0;\n do {\n // if in try_acquire, and lock is held as writer, don't attempt to speculate.\n if(m.write_flag.load(std::memory_order_acquire)) {\n if(only_speculate) return;\n spin_wait_while_eq(m.write_flag, true);\n }\n // _xbegin returns -1 on success or the abort code, so capture it\n if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin))\n {\n // started speculation\n if(m.write_flag.load(std::memory_order_relaxed)) { // add write_flag to read-set.\n abort_transaction(); // writer grabbed the lock, so abort.\n }\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader;\n // Don not wrap the following assignment to a function,\n // because it can abort the transaction in debug. Need mutex for release().\n s.m_mutex = &m;\n return; // successfully started speculation\n }\n // fallback path\n // retry only if there is any hope of getting into a transaction soon\n // Retry in the following cases (from Section 8.3.5 of\n // Intel(R) Architecture Instruction Set Extensions Programming Reference):\n // 1. abort caused by XABORT instruction (bit 0 of EAX register is set)\n // 2. the transaction may succeed on a retry (bit 1 of EAX register is set)\n // 3. if another logical processor conflicted with a memory address\n // that was part of the transaction that aborted (bit 2 of EAX register is set)\n // That is, retry if (abort_code & 0x7) is non-zero\n ++num_retries;\n } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read));\n }\n\n if(only_speculate) return;\n s.m_mutex = &m;\n s.m_mutex->lock_shared();\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;\n }\n\n //! Upgrade reader to become a writer.\n /** Returns whether the upgrade happened without releasing and re-acquiring the lock */\n static bool upgrade(d1::rtm_rw_mutex::scoped_lock& s) {\n switch(s.m_transaction_state) {\n case d1::rtm_rw_mutex::rtm_type::rtm_real_reader: {\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;\n bool no_release = s.m_mutex->upgrade();\n __TBB_ASSERT(!s.m_mutex->write_flag.load(std::memory_order_relaxed), \"After upgrade, write_flag already true\");\n s.m_mutex->write_flag.store(true, std::memory_order_relaxed);\n return no_release;\n }\n case d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader: {\n d1::rtm_rw_mutex& m = *s.m_mutex;\n if(m.m_state.load(std::memory_order_acquire)) { // add spin_rw_mutex to read-set.\n // Real reader or writer holds the lock; so commit the read and re-acquire for write.\n release(s);\n acquire_writer(m, s, false);\n return false;\n } else\n {\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer;\n return true;\n }\n }\n default:\n __TBB_ASSERT(false, \"Invalid state for upgrade\");\n return false;\n }\n }\n\n //! Downgrade writer to a reader.\n static bool downgrade(d1::rtm_rw_mutex::scoped_lock& s) {\n switch (s.m_transaction_state) {\n case d1::rtm_rw_mutex::rtm_type::rtm_real_writer:\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;\n __TBB_ASSERT(s.m_mutex->write_flag.load(std::memory_order_relaxed), \"Before downgrade write_flag not true\");\n s.m_mutex->write_flag.store(false, std::memory_order_relaxed);\n s.m_mutex->downgrade();\n return true;\n case d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer:\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader;\n return true;\n default:\n __TBB_ASSERT(false, \"Invalid state for downgrade\");\n return false;\n }\n }\n\n //! Try to acquire write lock on the given mutex.\n // There may be reader(s) which acquired the spin_rw_mutex, as well as possibly\n // transactional reader(s). If this is the case, the acquire will fail, and assigning\n // write_flag will kill the transactors. So we only assign write_flag if we have successfully\n // acquired the lock.\n static bool try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {\n acquire_writer(m, s, /*only_speculate=*/true);\n if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_writer) {\n return true;\n }\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr);\n // transacting write acquire failed. try_lock the real mutex\n if (m.try_lock()) {\n s.m_mutex = &m;\n // only shoot down readers if we're not transacting ourselves\n __TBB_ASSERT(!m.write_flag.load(std::memory_order_relaxed), \"After try_acquire_writer, write_flag already true\");\n m.write_flag.store(true, std::memory_order_relaxed);\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_writer;\n return true;\n }\n return false;\n }\n\n //! Try to acquire read lock on the given mutex.\n static bool try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {\n // speculatively acquire the lock. If this fails, do try_lock_shared on the spin_rw_mutex.\n acquire_reader(m, s, /*only_speculate=*/true);\n if (s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_transacting_reader) {\n return true;\n }\n __TBB_ASSERT(s.m_transaction_state == d1::rtm_rw_mutex::rtm_type::rtm_not_in_mutex, nullptr);\n // transacting read acquire failed. try_lock_shared the real mutex\n if (m.try_lock_shared()) {\n s.m_mutex = &m;\n s.m_transaction_state = d1::rtm_rw_mutex::rtm_type::rtm_real_reader;\n return true;\n }\n return false;\n }\n};\n\nvoid __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) {\n rtm_rw_mutex_impl::acquire_writer(m, s, only_speculate);\n}\n//! Internal acquire read lock.\n// only_speculate == true if we're doing a try_lock, else false.\nvoid __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s, bool only_speculate) {\n rtm_rw_mutex_impl::acquire_reader(m, s, only_speculate);\n}\n//! Internal upgrade reader to become a writer.\nbool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock& s) {\n return rtm_rw_mutex_impl::upgrade(s);\n}\n//! Internal downgrade writer to become a reader.\nbool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock& s) {\n return rtm_rw_mutex_impl::downgrade(s);\n}\n//! Internal try_acquire write lock.\nbool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {\n return rtm_rw_mutex_impl::try_acquire_writer(m, s);\n}\n//! Internal try_acquire read lock.\nbool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex& m, d1::rtm_rw_mutex::scoped_lock& s) {\n return rtm_rw_mutex_impl::try_acquire_reader(m, s);\n}\n//! Internal release lock.\nvoid __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock& s) {\n rtm_rw_mutex_impl::release(s);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n\n" }, { "path": "src/tbb/src/tbb/scheduler_common.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_scheduler_common_H\n#define _TBB_scheduler_common_H\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/detail/_machine.h\"\n#include \"oneapi/tbb/task_group.h\"\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"itt_notify.h\"\n#include \"co_context.h\"\n#include \"misc.h\"\n#include \"governor.h\"\n\n#ifndef __TBB_SCHEDULER_MUTEX_TYPE\n#define __TBB_SCHEDULER_MUTEX_TYPE tbb::spin_mutex\n#endif\n// TODO: add conditional inclusion based on specified type\n#include \"oneapi/tbb/spin_mutex.h\"\n#include \"oneapi/tbb/mutex.h\"\n\n#if TBB_USE_ASSERT\n#include \n#endif\n\n#include \n#include \n#include // unique_ptr\n#include \n\n//! Mutex type for global locks in the scheduler\nusing scheduler_mutex_type = __TBB_SCHEDULER_MUTEX_TYPE;\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n // Workaround for overzealous compiler warnings\n // These particular warnings are so ubiquitous that no attempt is made to narrow\n // the scope of the warnings.\n // #pragma warning (disable: 4100 4127 4312 4244 4267 4706)\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass arena;\nclass mail_inbox;\nclass mail_outbox;\nclass market;\nclass observer_proxy;\n\nenum task_stream_accessor_type { front_accessor = 0, back_nonnull_accessor };\ntemplate class task_stream;\n\nusing isolation_type = std::intptr_t;\nconstexpr isolation_type no_isolation = 0;\n\nstruct cache_aligned_deleter {\n template \n void operator() (T* ptr) const {\n ptr->~T();\n cache_aligned_deallocate(ptr);\n }\n};\n\ntemplate \nusing cache_aligned_unique_ptr = std::unique_ptr;\n\ntemplate \ncache_aligned_unique_ptr make_cache_aligned_unique(Args&& ...args) {\n return cache_aligned_unique_ptr(new (cache_aligned_allocate(sizeof(T))) T(std::forward(args)...));\n}\n\n//------------------------------------------------------------------------\n// Extended execute data\n//------------------------------------------------------------------------\n\n//! Execute data used on a task dispatcher side, reflects a current execution state\nstruct execution_data_ext : d1::execution_data {\n task_dispatcher* task_disp{};\n isolation_type isolation{};\n d1::wait_context* wait_ctx{};\n};\n\n//------------------------------------------------------------------------\n// Task accessor\n//------------------------------------------------------------------------\n\n//! Interpretation of reserved task fields inside a task dispatcher\nstruct task_accessor {\n static constexpr std::uint64_t proxy_task_trait = 1;\n static constexpr std::uint64_t resume_task_trait = 2;\n static d1::task_group_context*& context(d1::task& t) {\n task_group_context** tgc = reinterpret_cast(&t.m_reserved[0]);\n return *tgc;\n }\n static isolation_type& isolation(d1::task& t) {\n isolation_type* tag = reinterpret_cast(&t.m_reserved[2]);\n return *tag;\n }\n static void set_proxy_trait(d1::task& t) {\n // TODO: refactor proxy tasks not to work on uninitialized memory.\n //__TBB_ASSERT((t.m_version_and_traits & proxy_task_trait) == 0, nullptr);\n t.m_version_and_traits |= proxy_task_trait;\n }\n static bool is_proxy_task(d1::task& t) {\n return (t.m_version_and_traits & proxy_task_trait) != 0;\n }\n static void set_resume_trait(d1::task& t) {\n __TBB_ASSERT((t.m_version_and_traits & resume_task_trait) == 0, nullptr);\n t.m_version_and_traits |= resume_task_trait;\n }\n static bool is_resume_task(d1::task& t) {\n return (t.m_version_and_traits & resume_task_trait) != 0;\n }\n};\n\n//------------------------------------------------------------------------\n//! Extended variant of the standard offsetof macro\n/** The standard offsetof macro is not sufficient for TBB as it can be used for\n POD-types only. The constant 0x1000 (not nullptr) is necessary to appease GCC. **/\n#define __TBB_offsetof(class_name, member_name) \\\n ((ptrdiff_t)&(reinterpret_cast(0x1000)->member_name) - 0x1000)\n\n//! Returns address of the object containing a member with the given name and address\n#define __TBB_get_object_ref(class_name, member_name, member_addr) \\\n (*reinterpret_cast((char*)member_addr - __TBB_offsetof(class_name, member_name)))\n\n//! Helper class for tracking floating point context and task group context switches\n/** Assuming presence of an itt collector, in addition to keeping track of floating\n point context, this class emits itt events to indicate begin and end of task group\n context execution **/\ntemplate \nclass context_guard_helper {\n const d1::task_group_context* curr_ctx;\n d1::cpu_ctl_env guard_cpu_ctl_env;\n d1::cpu_ctl_env curr_cpu_ctl_env;\npublic:\n context_guard_helper() : curr_ctx(nullptr) {\n guard_cpu_ctl_env.get_env();\n curr_cpu_ctl_env = guard_cpu_ctl_env;\n }\n ~context_guard_helper() {\n if (curr_cpu_ctl_env != guard_cpu_ctl_env)\n guard_cpu_ctl_env.set_env();\n if (report_tasks && curr_ctx)\n ITT_TASK_END;\n }\n // The function is called from bypass dispatch loop on the hot path.\n // Consider performance issues when refactoring.\n void set_ctx(const d1::task_group_context* ctx) {\n if (!ctx)\n return;\n const d1::cpu_ctl_env* ctl = reinterpret_cast(&ctx->my_cpu_ctl_env);\n // Compare the FPU settings directly because the context can be reused between parallel algorithms.\n if (*ctl != curr_cpu_ctl_env) {\n curr_cpu_ctl_env = *ctl;\n curr_cpu_ctl_env.set_env();\n }\n if (report_tasks && ctx != curr_ctx) {\n // if task group context was active, report end of current execution frame.\n if (curr_ctx)\n ITT_TASK_END;\n // reporting begin of new task group context execution frame.\n // using address of task group context object to group tasks (parent).\n // id of task execution frame is nullptr and reserved for future use.\n ITT_TASK_BEGIN(ctx, ctx->my_name, nullptr);\n curr_ctx = ctx;\n }\n }\n#if _WIN64\n void restore_default() {\n if (curr_cpu_ctl_env != guard_cpu_ctl_env) {\n guard_cpu_ctl_env.set_env();\n curr_cpu_ctl_env = guard_cpu_ctl_env;\n }\n }\n#endif // _WIN64\n};\n\n#if (_WIN32 || _WIN64 || __unix__ || __APPLE__) && (__TBB_x86_32 || __TBB_x86_64)\n#if _MSC_VER\n#pragma intrinsic(__rdtsc)\n#endif\ninline std::uint64_t machine_time_stamp() {\n#if __INTEL_COMPILER\n return _rdtsc();\n#elif _MSC_VER\n return __rdtsc();\n#else\n std::uint32_t hi, lo;\n __asm__ __volatile__(\"rdtsc\" : \"=d\"(hi), \"=a\"(lo));\n return (std::uint64_t(hi) << 32) | lo;\n#endif\n}\n\ninline void prolonged_pause_impl() {\n // Assumption based on practice: 1000-2000 ticks seems to be a suitable invariant for the\n // majority of platforms. Currently, skip platforms that define __TBB_STEALING_PAUSE\n // because these platforms require very careful tuning.\n std::uint64_t prev = machine_time_stamp();\n const std::uint64_t finish = prev + 1000;\n atomic_backoff backoff;\n do {\n backoff.bounded_pause();\n std::uint64_t curr = machine_time_stamp();\n if (curr <= prev)\n // Possibly, the current logical thread is moved to another hardware thread or overflow is occurred.\n break;\n prev = curr;\n } while (prev < finish);\n}\n#else\ninline void prolonged_pause_impl() {\n#ifdef __TBB_ipf\n static const long PauseTime = 1500;\n#else\n static const long PauseTime = 80;\n#endif\n // TODO IDEA: Update PauseTime adaptively?\n machine_pause(PauseTime);\n}\n#endif\n\ninline void prolonged_pause() {\n#if __TBB_WAITPKG_INTRINSICS_PRESENT\n if (governor::wait_package_enabled()) {\n std::uint64_t time_stamp = machine_time_stamp();\n // _tpause function directs the processor to enter an implementation-dependent optimized state\n // until the Time Stamp Counter reaches or exceeds the value specified in second parameter.\n // Constant \"1000\" is ticks to wait for.\n // TODO : Modify this parameter based on empirical study of benchmarks.\n // First parameter 0 selects between a lower power (cleared) or faster wakeup (set) optimized state.\n _tpause(0, time_stamp + 1000);\n }\n else\n#endif\n prolonged_pause_impl();\n}\n\n// TODO: investigate possibility to work with number of CPU cycles\n// because for different configurations this number of pauses + yields\n// will be calculated in different amount of CPU cycles\n// for example use rdtsc for it\nclass stealing_loop_backoff {\n const int my_pause_threshold;\n const int my_yield_threshold;\n int my_pause_count;\n int my_yield_count;\npublic:\n // my_yield_threshold = 100 is an experimental value. Ideally, once we start calling __TBB_Yield(),\n // the time spent spinning before calling out_of_work() should be approximately\n // the time it takes for a thread to be woken up. Doing so would guarantee that we do\n // no worse than 2x the optimal spin time. Or perhaps a time-slice quantum is the right amount.\n stealing_loop_backoff(int num_workers, int yields_multiplier)\n : my_pause_threshold{ 2 * (num_workers + 1) }\n , my_yield_threshold{100 * yields_multiplier}\n , my_pause_count{}\n , my_yield_count{}\n {}\n bool pause() {\n prolonged_pause();\n if (my_pause_count++ >= my_pause_threshold) {\n my_pause_count = my_pause_threshold;\n d0::yield();\n if (my_yield_count++ >= my_yield_threshold) {\n my_yield_count = my_yield_threshold;\n return true;\n }\n }\n return false;\n }\n void reset_wait() {\n my_pause_count = my_yield_count = 0;\n }\n};\n\n//------------------------------------------------------------------------\n// Exception support\n//------------------------------------------------------------------------\n//! Task group state change propagation global epoch\n/** Together with generic_scheduler::my_context_state_propagation_epoch forms\n cross-thread signaling mechanism that allows to avoid locking at the hot path\n of normal execution flow.\n\n When a descendant task group context is registered or unregistered, the global\n and local epochs are compared. If they differ, a state change is being propagated,\n and thus registration/deregistration routines take slower branch that may block\n (at most one thread of the pool can be blocked at any moment). Otherwise the\n control path is lock-free and fast. **/\nextern std::atomic the_context_state_propagation_epoch;\n\n//! Mutex guarding state change propagation across task groups forest.\n/** Also protects modification of related data structures. **/\ntypedef scheduler_mutex_type context_state_propagation_mutex_type;\nextern context_state_propagation_mutex_type the_context_state_propagation_mutex;\n\nclass tbb_exception_ptr {\n std::exception_ptr my_ptr;\npublic:\n static tbb_exception_ptr* allocate() noexcept;\n\n //! Destroys this objects\n /** Note that objects of this type can be created only by the allocate() method. **/\n void destroy() noexcept;\n\n //! Throws the contained exception .\n void throw_self();\n\nprivate:\n tbb_exception_ptr(const std::exception_ptr& src) : my_ptr(src) {}\n}; // class tbb_exception_ptr\n\n//------------------------------------------------------------------------\n// Debugging support\n//------------------------------------------------------------------------\n\n#if TBB_USE_ASSERT\nstatic const std::uintptr_t venom = tbb::detail::select_size_t_constant<0xDEADBEEFU, 0xDDEEAADDDEADBEEFULL>::value;\n\ninline void poison_value(std::uintptr_t& val) { val = venom; }\n\ninline void poison_value(std::atomic& val) { val.store(venom, std::memory_order_relaxed); }\n\n/** Expected to be used in assertions only, thus no empty form is defined. **/\ninline bool is_alive(std::uintptr_t v) { return v != venom; }\n\n/** Logically, this method should be a member of class task.\n But we do not want to publish it, so it is here instead. */\ninline void assert_task_valid(const d1::task* t) {\n assert_pointer_valid(t);\n}\n#else /* !TBB_USE_ASSERT */\n\n/** In contrast to debug version poison_value() is a macro here because\n the variable used as its argument may be undefined in release builds. **/\n#define poison_value(g) ((void)0)\n\ninline void assert_task_valid(const d1::task*) {}\n\n#endif /* !TBB_USE_ASSERT */\n\nstruct suspend_point_type {\n#if __TBB_RESUMABLE_TASKS\n //! The arena related to this task_dispatcher\n arena* m_arena{ nullptr };\n //! The random for the resume task\n FastRandom m_random;\n //! The flag is raised when the original owner should return to this task dispatcher.\n std::atomic m_is_owner_recalled{ false };\n //! Inicates if the resume task should be placed to the critical task stream.\n bool m_is_critical{ false };\n //! Associated coroutine\n co_context m_co_context;\n //! Supend point before resume\n suspend_point_type* m_prev_suspend_point{nullptr};\n\n // Possible state transitions:\n // A -> S -> N -> A\n // A -> N -> S -> N -> A\n enum class stack_state {\n active, // some thread is working with this stack\n suspended, // no thread is working with this stack\n notified // some thread tried to resume this stack\n };\n\n //! The flag required to protect suspend finish and resume call\n std::atomic m_stack_state{stack_state::active};\n\n void resume(suspend_point_type* sp) {\n __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) != stack_state::suspended, \"The stack is expected to be active\");\n\n sp->m_prev_suspend_point = this;\n\n // Do not access sp after resume\n m_co_context.resume(sp->m_co_context);\n __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) != stack_state::active, nullptr);\n\n finilize_resume();\n }\n\n void finilize_resume() {\n m_stack_state.store(stack_state::active, std::memory_order_relaxed);\n // Set the suspended state for the stack that we left. If the state is already notified, it means that\n // someone already tried to resume our previous stack but failed. So, we need to resume it.\n // m_prev_suspend_point might be nullptr when destroying co_context based on threads\n if (m_prev_suspend_point && m_prev_suspend_point->m_stack_state.exchange(stack_state::suspended) == stack_state::notified) {\n r1::resume(m_prev_suspend_point);\n }\n m_prev_suspend_point = nullptr;\n }\n\n bool try_notify_resume() {\n // Check that stack is already suspended. Return false if not yet.\n return m_stack_state.exchange(stack_state::notified) == stack_state::suspended;\n }\n\n void recall_owner() {\n __TBB_ASSERT(m_stack_state.load(std::memory_order_relaxed) == stack_state::suspended, nullptr);\n m_stack_state.store(stack_state::notified, std::memory_order_relaxed);\n m_is_owner_recalled.store(true, std::memory_order_release);\n }\n\n struct resume_task final : public d1::task {\n task_dispatcher& m_target;\n explicit resume_task(task_dispatcher& target) : m_target(target) {\n task_accessor::set_resume_trait(*this);\n }\n d1::task* execute(d1::execution_data& ed) override;\n d1::task* cancel(d1::execution_data&) override {\n __TBB_ASSERT(false, \"The resume task cannot be canceled\");\n return nullptr;\n }\n } m_resume_task;\n\n suspend_point_type(arena* a, std::size_t stack_size, task_dispatcher& target);\n#endif /*__TBB_RESUMABLE_TASKS */\n};\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// structure was padded due to alignment specifier\n// #pragma warning( push )\n// #pragma warning( disable: 4324 )\n#endif\n\nclass alignas (max_nfs_size) task_dispatcher {\npublic:\n // TODO: reconsider low level design to better organize dependencies and files.\n friend class thread_data;\n friend class arena_slot;\n friend class nested_arena_context;\n friend class delegated_task;\n friend struct base_waiter;\n\n //! The list of possible post resume actions.\n enum class post_resume_action {\n invalid,\n register_waiter,\n cleanup,\n notify,\n none\n };\n\n //! The data of the current thread attached to this task_dispatcher\n thread_data* m_thread_data{ nullptr };\n\n //! The current execution data\n execution_data_ext m_execute_data_ext;\n\n //! Properties\n struct properties {\n bool outermost{ true };\n bool fifo_tasks_allowed{ true };\n bool critical_task_allowed{ true };\n } m_properties;\n\n //! Position in the call stack when stealing is still allowed.\n std::uintptr_t m_stealing_threshold{};\n\n //! Suspend point (null if this task dispatcher has been never suspended)\n suspend_point_type* m_suspend_point{ nullptr };\n\n //! Used to improve scalability of d1::wait_context by using per thread reference_counter\n std::unordered_map, std::equal_to,\n tbb_allocator>\n >\n m_reference_vertex_map;\n\n //! Attempt to get a task from the mailbox.\n /** Gets a task only if it has not been executed by its sender or a thief\n that has stolen it from the sender's task pool. Otherwise returns nullptr.\n This method is intended to be used only by the thread extracting the proxy\n from its mailbox. (In contrast to local task pool, mailbox can be read only\n by its owner). **/\n d1::task* get_mailbox_task(mail_inbox& my_inbox, execution_data_ext& ed, isolation_type isolation);\n\n d1::task* get_critical_task(d1::task*, execution_data_ext&, isolation_type, bool);\n\n template \n d1::task* receive_or_steal_task(thread_data& tls, execution_data_ext& ed, Waiter& waiter,\n isolation_type isolation, bool outermost, bool criticality_absence);\n\n template \n d1::task* local_wait_for_all(d1::task * t, Waiter& waiter);\n\n task_dispatcher(const task_dispatcher&) = delete;\n\n bool can_steal();\npublic:\n task_dispatcher(arena* a);\n\n ~task_dispatcher() {\n if (m_suspend_point) {\n m_suspend_point->~suspend_point_type();\n cache_aligned_deallocate(m_suspend_point);\n }\n\n for (auto& elem : m_reference_vertex_map) {\n d1::reference_vertex*& node = elem.second;\n node->~reference_vertex();\n cache_aligned_deallocate(node);\n poison_pointer(node);\n }\n\n poison_pointer(m_thread_data);\n poison_pointer(m_suspend_point);\n }\n\n template \n d1::task* local_wait_for_all(d1::task* t, Waiter& waiter);\n\n bool allow_fifo_task(bool new_state) {\n bool old_state = m_properties.fifo_tasks_allowed;\n m_properties.fifo_tasks_allowed = new_state;\n return old_state;\n }\n\n isolation_type set_isolation(isolation_type isolation) {\n isolation_type prev = m_execute_data_ext.isolation;\n m_execute_data_ext.isolation = isolation;\n return prev;\n }\n\n thread_data& get_thread_data() {\n __TBB_ASSERT(m_thread_data, nullptr);\n return *m_thread_data;\n }\n\n static void execute_and_wait(d1::task* t, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx);\n\n void set_stealing_threshold(std::uintptr_t stealing_threshold) {\n bool assert_condition = (stealing_threshold == 0 && m_stealing_threshold != 0) ||\n (stealing_threshold != 0 && m_stealing_threshold == 0);\n __TBB_ASSERT_EX( assert_condition, nullptr );\n m_stealing_threshold = stealing_threshold;\n }\n\n d1::task* get_inbox_or_critical_task(execution_data_ext&, mail_inbox&, isolation_type, bool);\n d1::task* get_stream_or_critical_task(execution_data_ext&, arena&, task_stream&,\n unsigned& /*hint_for_stream*/, isolation_type,\n bool /*critical_allowed*/);\n d1::task* steal_or_get_critical(execution_data_ext&, arena&, unsigned /*arena_index*/, FastRandom&,\n isolation_type, bool /*critical_allowed*/);\n\n#if __TBB_RESUMABLE_TASKS\n /* [[noreturn]] */ void co_local_wait_for_all() noexcept;\n void suspend(suspend_callback_type suspend_callback, void* user_callback);\n void internal_suspend();\n void do_post_resume_action();\n\n bool resume(task_dispatcher& target);\n suspend_point_type* get_suspend_point();\n void init_suspend_point(arena* a, std::size_t stack_size);\n friend void internal_resume(suspend_point_type*);\n void recall_point();\n#endif /* __TBB_RESUMABLE_TASKS */\n};\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// #pragma warning( pop )\n#endif\n\ninline std::uintptr_t calculate_stealing_threshold(std::uintptr_t base, std::size_t stack_size) {\n __TBB_ASSERT(stack_size != 0, \"Stack size cannot be zero\");\n __TBB_ASSERT(base > stack_size / 2, \"Stack anchor calculation overflow\");\n return base - stack_size / 2;\n}\n\nstruct task_group_context_impl {\n static void destroy(d1::task_group_context&);\n static void initialize(d1::task_group_context&);\n static void register_with(d1::task_group_context&, thread_data*);\n static void bind_to_impl(d1::task_group_context&, thread_data*);\n static void bind_to(d1::task_group_context&, thread_data*);\n static void propagate_task_group_state(d1::task_group_context&, std::atomic d1::task_group_context::*, d1::task_group_context&, uint32_t);\n static bool cancel_group_execution(d1::task_group_context&);\n static bool is_group_execution_cancelled(const d1::task_group_context&);\n static void reset(d1::task_group_context&);\n static void capture_fp_settings(d1::task_group_context&);\n static void copy_fp_settings(d1::task_group_context& ctx, const d1::task_group_context& src);\n};\n\n\n//! Forward declaration for scheduler entities\nbool gcc_rethrow_exception_broken();\nvoid fix_broken_rethrow();\n//! Forward declaration: throws std::runtime_error with what() returning error_code description prefixed with aux_info\nvoid handle_perror(int error_code, const char* aux_info);\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_scheduler_common_H */\n" }, { "path": "src/tbb/src/tbb/semaphore.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"semaphore.h\"\n#if __TBB_USE_SRWLOCK\n#include \"dynamic_link.h\" // Refers to src/tbb, not include/tbb\n#include \"tbb_misc.h\"\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n// TODO: For new win UI port, we can use SRWLock API without dynamic_link etc.\n#if __TBB_USE_SRWLOCK\n\nstatic std::atomic concmon_module_inited;\n\nvoid WINAPI init_binsem_using_event( SRWLOCK* h_ )\n{\n srwl_or_handle* shptr = (srwl_or_handle*) h_;\n shptr->h = CreateEventEx( nullptr, nullptr, 0, EVENT_ALL_ACCESS|SEMAPHORE_ALL_ACCESS );\n}\n\nvoid WINAPI acquire_binsem_using_event( SRWLOCK* h_ )\n{\n srwl_or_handle* shptr = (srwl_or_handle*) h_;\n WaitForSingleObjectEx( shptr->h, INFINITE, FALSE );\n}\n\nvoid WINAPI release_binsem_using_event( SRWLOCK* h_ )\n{\n srwl_or_handle* shptr = (srwl_or_handle*) h_;\n SetEvent( shptr->h );\n}\n\nstatic void (WINAPI *__TBB_init_binsem)( SRWLOCK* ) = (void (WINAPI *)(SRWLOCK*))&init_binsem_using_event;\nstatic void (WINAPI *__TBB_acquire_binsem)( SRWLOCK* ) = (void (WINAPI *)(SRWLOCK*))&acquire_binsem_using_event;\nstatic void (WINAPI *__TBB_release_binsem)( SRWLOCK* ) = (void (WINAPI *)(SRWLOCK*))&release_binsem_using_event;\n\n//! Table describing the how to link the handlers.\nstatic const dynamic_link_descriptor SRWLLinkTable[] = {\n DLD(InitializeSRWLock, __TBB_init_binsem),\n DLD(AcquireSRWLockExclusive, __TBB_acquire_binsem),\n DLD(ReleaseSRWLockExclusive, __TBB_release_binsem)\n};\n\ninline void init_concmon_module()\n{\n __TBB_ASSERT( (uintptr_t)__TBB_init_binsem==(uintptr_t)&init_binsem_using_event, nullptr);\n if( dynamic_link( \"Kernel32.dll\", SRWLLinkTable, sizeof(SRWLLinkTable)/sizeof(dynamic_link_descriptor) ) ) {\n __TBB_ASSERT( (uintptr_t)__TBB_init_binsem!=(uintptr_t)&init_binsem_using_event, nullptr);\n __TBB_ASSERT( (uintptr_t)__TBB_acquire_binsem!=(uintptr_t)&acquire_binsem_using_event, nullptr);\n __TBB_ASSERT( (uintptr_t)__TBB_release_binsem!=(uintptr_t)&release_binsem_using_event, nullptr);\n }\n}\n\nbinary_semaphore::binary_semaphore() {\n atomic_do_once( &init_concmon_module, concmon_module_inited );\n\n __TBB_init_binsem( &my_sem.lock );\n if( (uintptr_t)__TBB_init_binsem!=(uintptr_t)&init_binsem_using_event )\n P();\n}\n\nbinary_semaphore::~binary_semaphore() {\n if( (uintptr_t)__TBB_init_binsem==(uintptr_t)&init_binsem_using_event )\n CloseHandle( my_sem.h );\n}\n\nvoid binary_semaphore::P() { __TBB_acquire_binsem( &my_sem.lock ); }\n\nvoid binary_semaphore::V() { __TBB_release_binsem( &my_sem.lock ); }\n\n#endif /* __TBB_USE_SRWLOCK */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/semaphore.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_semaphore_H\n#define __TBB_semaphore_H\n\n#include \"oneapi/tbb/detail/_utils.h\"\n\n#if _WIN32||_WIN64\n#include \n#elif __APPLE__\n#include \n#else\n#include \n#ifdef TBB_USE_DEBUG\n#include \n#endif\n#endif /*_WIN32||_WIN64*/\n\n#include \n\n#if __unix__\n#if defined(__has_include)\n#define __TBB_has_include __has_include\n#else\n#define __TBB_has_include(x) 0\n#endif\n\n/* Futex definitions */\n#include \n#if defined(__linux__) || __TBB_has_include()\n#include \n#endif\n\n#if defined(SYS_futex)\n\n/* This section is included for Linux and some other systems that may support futexes.*/\n\n#define __TBB_USE_FUTEX 1\n\n/*\nIf available, use typical headers where futex API is defined. While Linux and OpenBSD\nare known to provide such headers, other systems might have them as well.\n*/\n#if defined(__linux__) || __TBB_has_include()\n#include \n#elif defined(__OpenBSD__) || __TBB_has_include()\n#include \n#endif\n\n#include \n#include \n\n/*\nSome systems might not define the macros or use different names. In such case we expect\nthe actual parameter values to match Linux: 0 for wait, 1 for wake.\n*/\n#if defined(FUTEX_WAIT_PRIVATE)\n#define __TBB_FUTEX_WAIT FUTEX_WAIT_PRIVATE\n#elif defined(FUTEX_WAIT)\n#define __TBB_FUTEX_WAIT FUTEX_WAIT\n#else\n#define __TBB_FUTEX_WAIT 0\n#endif\n\n#if defined(FUTEX_WAKE_PRIVATE)\n#define __TBB_FUTEX_WAKE FUTEX_WAKE_PRIVATE\n#elif defined(FUTEX_WAKE)\n#define __TBB_FUTEX_WAKE FUTEX_WAKE\n#else\n#define __TBB_FUTEX_WAKE 1\n#endif\n\n#endif // SYS_futex\n#endif // __unix__\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// Futex implementation\n////////////////////////////////////////////////////////////////////////////////////////////////////\n\n#if __TBB_USE_FUTEX\n\nstatic inline int futex_wait( void *futex, int comparand ) {\n int r = ::syscall(SYS_futex, futex, __TBB_FUTEX_WAIT, comparand, nullptr, nullptr, 0);\n#if TBB_USE_ASSERT\n int e = errno;\n __TBB_ASSERT(r == 0 || r == EWOULDBLOCK || (r == -1 && (e == EAGAIN || e == EINTR)), \"futex_wait failed.\");\n#endif /* TBB_USE_ASSERT */\n return r;\n}\n\nstatic inline int futex_wakeup_one( void *futex ) {\n int r = ::syscall(SYS_futex, futex, __TBB_FUTEX_WAKE, 1, nullptr, nullptr, 0);\n __TBB_ASSERT(r == 0 || r == 1, \"futex_wakeup_one: more than one thread woken up?\");\n return r;\n}\n\n// Additional possible methods that are not required right now\n// static inline int futex_wakeup_all( void *futex ) {\n// int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,INT_MAX,nullptr,nullptr,0 );\n// __TBB_ASSERT( r>=0, \"futex_wakeup_all: error in waking up threads\" );\n// return r;\n// }\n\n#endif // __TBB_USE_FUTEX\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n#if _WIN32||_WIN64\ntypedef LONG sem_count_t;\n//! Edsger Dijkstra's counting semaphore\nclass semaphore : no_copy {\n static const int max_semaphore_cnt = MAXLONG;\npublic:\n //! ctor\n semaphore(size_t start_cnt_ = 0) {init_semaphore(start_cnt_);}\n //! dtor\n ~semaphore() {CloseHandle( sem );}\n //! wait/acquire\n void P() {WaitForSingleObjectEx( sem, INFINITE, FALSE );}\n //! post/release\n void V() {ReleaseSemaphore( sem, 1, nullptr);}\nprivate:\n HANDLE sem;\n void init_semaphore(size_t start_cnt_) {\n sem = CreateSemaphoreEx( nullptr, LONG(start_cnt_), max_semaphore_cnt, nullptr, 0, SEMAPHORE_ALL_ACCESS );\n }\n};\n#elif __APPLE__\n//! Edsger Dijkstra's counting semaphore\nclass semaphore : no_copy {\npublic:\n //! ctor\n semaphore(int start_cnt_ = 0) { my_sem = dispatch_semaphore_create(start_cnt_); }\n //! dtor\n ~semaphore() { dispatch_release(my_sem); }\n //! wait/acquire\n void P() {\n std::intptr_t ret = dispatch_semaphore_wait(my_sem, DISPATCH_TIME_FOREVER);\n __TBB_ASSERT_EX(ret == 0, \"dispatch_semaphore_wait() failed\");\n }\n //! post/release\n void V() { dispatch_semaphore_signal(my_sem); }\nprivate:\n dispatch_semaphore_t my_sem;\n};\n#else /* Linux/Unix */\ntypedef uint32_t sem_count_t;\n//! Edsger Dijkstra's counting semaphore\nclass semaphore : no_copy {\npublic:\n //! ctor\n semaphore(int start_cnt_ = 0 ) { init_semaphore( start_cnt_ ); }\n\n //! dtor\n ~semaphore() {\n int ret = sem_destroy( &sem );\n __TBB_ASSERT_EX( !ret, nullptr);\n }\n //! wait/acquire\n void P() {\n while( sem_wait( &sem )!=0 )\n __TBB_ASSERT( errno==EINTR, nullptr);\n }\n //! post/release\n void V() { sem_post( &sem ); }\nprivate:\n sem_t sem;\n void init_semaphore(int start_cnt_) {\n int ret = sem_init( &sem, /*shared among threads*/ 0, start_cnt_ );\n __TBB_ASSERT_EX( !ret, nullptr);\n }\n};\n#endif /* _WIN32||_WIN64 */\n\n\n//! for performance reasons, we want specialized binary_semaphore\n#if _WIN32||_WIN64\n#if !__TBB_USE_SRWLOCK\n//! binary_semaphore for concurrent_monitor\nclass binary_semaphore : no_copy {\npublic:\n //! ctor\n binary_semaphore() { my_sem = CreateEventEx( nullptr, nullptr, 0, EVENT_ALL_ACCESS ); }\n //! dtor\n ~binary_semaphore() { CloseHandle( my_sem ); }\n //! wait/acquire\n void P() { WaitForSingleObjectEx( my_sem, INFINITE, FALSE ); }\n //! post/release\n void V() { SetEvent( my_sem ); }\nprivate:\n HANDLE my_sem;\n};\n#else /* __TBB_USE_SRWLOCK */\n\nunion srwl_or_handle {\n SRWLOCK lock;\n HANDLE h;\n};\n\n//! binary_semaphore for concurrent_monitor\nclass binary_semaphore : no_copy {\npublic:\n //! ctor\n binary_semaphore();\n //! dtor\n ~binary_semaphore();\n //! wait/acquire\n void P();\n //! post/release\n void V();\nprivate:\n srwl_or_handle my_sem;\n};\n#endif /* !__TBB_USE_SRWLOCK */\n#elif __APPLE__\n//! binary_semaphore for concurrent monitor\nusing binary_semaphore = semaphore;\n#else /* Linux/Unix */\n\n#if __TBB_USE_FUTEX\nclass binary_semaphore : no_copy {\n// The implementation is equivalent to the \"Mutex, Take 3\" one\n// in the paper \"Futexes Are Tricky\" by Ulrich Drepper\npublic:\n //! ctor\n binary_semaphore() { my_sem = 1; }\n //! dtor\n ~binary_semaphore() {}\n //! wait/acquire\n void P() {\n int s = 0;\n if( !my_sem.compare_exchange_strong( s, 1 ) ) {\n if( s!=2 )\n s = my_sem.exchange( 2 );\n while( s!=0 ) { // This loop deals with spurious wakeup\n futex_wait( &my_sem, 2 );\n s = my_sem.exchange( 2 );\n }\n }\n }\n //! post/release\n void V() {\n __TBB_ASSERT( my_sem.load(std::memory_order_relaxed)>=1, \"multiple V()'s in a row?\" );\n if( my_sem.exchange( 0 )==2 )\n futex_wakeup_one( &my_sem );\n }\nprivate:\n std::atomic my_sem; // 0 - open; 1 - closed, no waits; 2 - closed, possible waits\n};\n#else\ntypedef uint32_t sem_count_t;\n//! binary_semaphore for concurrent monitor\nclass binary_semaphore : no_copy {\npublic:\n //! ctor\n binary_semaphore() {\n int ret = sem_init( &my_sem, /*shared among threads*/ 0, 0 );\n __TBB_ASSERT_EX( !ret, nullptr);\n }\n //! dtor\n ~binary_semaphore() {\n int ret = sem_destroy( &my_sem );\n __TBB_ASSERT_EX( !ret, nullptr);\n }\n //! wait/acquire\n void P() {\n while( sem_wait( &my_sem )!=0 )\n __TBB_ASSERT( errno==EINTR, nullptr);\n }\n //! post/release\n void V() { sem_post( &my_sem ); }\nprivate:\n sem_t my_sem;\n};\n#endif /* __TBB_USE_FUTEX */\n#endif /* _WIN32||_WIN64 */\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_semaphore_H */\n" }, { "path": "src/tbb/src/tbb/small_object_pool.cpp", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/detail/_small_object_pool.h\"\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"governor.h\"\n#include \"thread_data.h\"\n#include \"task_dispatcher.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nsmall_object_pool_impl::small_object* const small_object_pool_impl::dead_public_list =\n reinterpret_cast(1);\n\nvoid* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& allocator, std::size_t number_of_bytes, const d1::execution_data& ed) {\n auto& tls = static_cast(ed).task_disp->get_thread_data();\n auto pool = tls.my_small_object_pool;\n return pool->allocate_impl(allocator, number_of_bytes);\n}\n\nvoid* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& allocator, std::size_t number_of_bytes) {\n // TODO: optimize if the allocator contains a valid pool.\n auto tls = governor::get_thread_data();\n auto pool = tls->my_small_object_pool;\n return pool->allocate_impl(allocator, number_of_bytes);\n}\n\nvoid* small_object_pool_impl::allocate_impl(d1::small_object_pool*& allocator, std::size_t number_of_bytes)\n{\n small_object* obj{nullptr};\n\n if (number_of_bytes <= small_object_size) {\n if (m_private_list) {\n obj = m_private_list;\n m_private_list = m_private_list->next;\n } else if (m_public_list.load(std::memory_order_relaxed)) {\n // No fence required for read of my_public_list above, because std::atomic::exchange() has a fence.\n obj = m_public_list.exchange(nullptr);\n __TBB_ASSERT( obj, \"another thread emptied the my_public_list\" );\n m_private_list = obj->next;\n } else {\n obj = new (cache_aligned_allocate(small_object_size)) small_object{nullptr};\n ++m_private_counter;\n }\n } else {\n obj = new (cache_aligned_allocate(number_of_bytes)) small_object{nullptr};\n }\n allocator = this;\n\n // Return uninitialized memory for further construction on user side.\n obj->~small_object();\n return obj;\n}\n\nvoid __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& allocator, void* ptr, std::size_t number_of_bytes) {\n auto pool = static_cast(&allocator);\n auto tls = governor::get_thread_data();\n pool->deallocate_impl(ptr, number_of_bytes, *tls);\n}\n\nvoid __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& allocator, void* ptr, std::size_t number_of_bytes, const d1::execution_data& ed) {\n auto& tls = static_cast(ed).task_disp->get_thread_data();\n auto pool = static_cast(&allocator);\n pool->deallocate_impl(ptr, number_of_bytes, tls);\n}\n\nvoid small_object_pool_impl::deallocate_impl(void* ptr, std::size_t number_of_bytes, thread_data& td) {\n __TBB_ASSERT(ptr != nullptr, \"pointer to deallocate should not be null\");\n __TBB_ASSERT(number_of_bytes >= sizeof(small_object), \"number of bytes should be at least sizeof(small_object)\");\n\n if (number_of_bytes <= small_object_size) {\n auto obj = new (ptr) small_object{nullptr};\n if (td.my_small_object_pool == this) {\n obj->next = m_private_list;\n m_private_list = obj;\n } else {\n auto old_public_list = m_public_list.load(std::memory_order_relaxed);\n\n for (;;) {\n if (old_public_list == dead_public_list) {\n obj->~small_object();\n cache_aligned_deallocate(obj);\n if (++m_public_counter == 0)\n {\n this->~small_object_pool_impl();\n cache_aligned_deallocate(this);\n }\n break;\n }\n obj->next = old_public_list;\n if (m_public_list.compare_exchange_strong(old_public_list, obj)) {\n break;\n }\n }\n }\n } else {\n cache_aligned_deallocate(ptr);\n }\n}\n\nstd::int64_t small_object_pool_impl::cleanup_list(small_object* list)\n{\n std::int64_t removed_count{};\n\n while (list) {\n small_object* current = list;\n list = list->next;\n current->~small_object();\n cache_aligned_deallocate(current);\n ++removed_count;\n }\n return removed_count;\n}\n\nvoid small_object_pool_impl::destroy()\n{\n // clean up private list and subtract the removed count from private counter\n m_private_counter -= cleanup_list(m_private_list);\n // Grab public list and place dead mark\n small_object* public_list = m_public_list.exchange(dead_public_list);\n // clean up public list and subtract from private (intentionally) counter\n m_private_counter -= cleanup_list(public_list);\n __TBB_ASSERT(m_private_counter >= 0, \"Private counter may not be less than 0\");\n // Equivalent to fetch_sub(m_private_counter) - m_private_counter. But we need to do it\n // atomically with operator-= not to access m_private_counter after the subtraction.\n auto new_value = m_public_counter -= m_private_counter;\n // check if this method is responsible to clean up the resources\n if (new_value == 0) {\n this->~small_object_pool_impl();\n cache_aligned_deallocate(this);\n }\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/small_object_pool_impl.h", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_small_object_pool_impl_H\n#define __TBB_small_object_pool_impl_H\n\n#include \"oneapi/tbb/detail/_small_object_pool.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n\n#include \n#include \n#include \n\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass thread_data;\n\nclass small_object_pool_impl : public d1::small_object_pool\n{\n static constexpr std::size_t small_object_size = 256;\n struct small_object {\n small_object* next;\n };\n static small_object* const dead_public_list;\npublic:\n void* allocate_impl(small_object_pool*& allocator, std::size_t number_of_bytes);\n void deallocate_impl(void* ptr, std::size_t number_of_bytes, thread_data& td);\n void destroy();\nprivate:\n static std::int64_t cleanup_list(small_object* list);\n ~small_object_pool_impl() = default;\nprivate:\n alignas(max_nfs_size) small_object* m_private_list;\n std::int64_t m_private_counter{};\n alignas(max_nfs_size) std::atomic m_public_list;\n std::atomic m_public_counter{};\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* __TBB_small_object_pool_impl_H */\n" }, { "path": "src/tbb/src/tbb/task.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// Do not include task.h directly. Use scheduler_common.h instead\n#include \"scheduler_common.h\"\n#include \"governor.h\"\n#include \"arena.h\"\n#include \"thread_data.h\"\n#include \"task_dispatcher.h\"\n#include \"waiters.h\"\n#include \"itt_notify.h\"\n\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/partitioner.h\"\n#include \"oneapi/tbb/task.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// resumable tasks\n//------------------------------------------------------------------------\n#if __TBB_RESUMABLE_TASKS\n\nvoid suspend(suspend_callback_type suspend_callback, void* user_callback) {\n thread_data& td = *governor::get_thread_data();\n td.my_task_dispatcher->suspend(suspend_callback, user_callback);\n // Do not access td after suspend.\n}\n\nvoid resume(suspend_point_type* sp) {\n assert_pointers_valid(sp, sp->m_arena);\n task_dispatcher& task_disp = sp->m_resume_task.m_target;\n\n if (sp->try_notify_resume()) {\n // TODO: remove this work-around\n // Prolong the arena's lifetime while all coroutines are alive\n // (otherwise the arena can be destroyed while some tasks are suspended).\n arena& a = *sp->m_arena;\n a.my_references += arena::ref_worker;\n\n if (task_disp.m_properties.critical_task_allowed) {\n // The target is not in the process of executing critical task, so the resume task is not critical.\n a.my_resume_task_stream.push(&sp->m_resume_task, random_lane_selector(sp->m_random));\n } else {\n #if __TBB_PREVIEW_CRITICAL_TASKS\n // The target is in the process of executing critical task, so the resume task is critical.\n a.my_critical_task_stream.push(&sp->m_resume_task, random_lane_selector(sp->m_random));\n #endif\n }\n // Do not access target after that point.\n a.advertise_new_work();\n // Release our reference to my_arena.\n a.on_thread_leaving(arena::ref_worker);\n }\n\n}\n\nsuspend_point_type* current_suspend_point() {\n thread_data& td = *governor::get_thread_data();\n return td.my_task_dispatcher->get_suspend_point();\n}\n\ntask_dispatcher& create_coroutine(thread_data& td) {\n // We may have some task dispatchers cached\n task_dispatcher* task_disp = td.my_arena->my_co_cache.pop();\n if (!task_disp) {\n void* ptr = cache_aligned_allocate(sizeof(task_dispatcher));\n task_disp = new(ptr) task_dispatcher(td.my_arena);\n task_disp->init_suspend_point(td.my_arena, td.my_arena->my_threading_control->worker_stack_size());\n }\n // Prolong the arena's lifetime until all coroutines is alive\n // (otherwise the arena can be destroyed while some tasks are suspended).\n // TODO: consider behavior if there are more than 4K external references.\n td.my_arena->my_references += arena::ref_external;\n return *task_disp;\n}\n\nvoid task_dispatcher::internal_suspend() {\n __TBB_ASSERT(m_thread_data != nullptr, nullptr);\n\n arena_slot* slot = m_thread_data->my_arena_slot;\n __TBB_ASSERT(slot != nullptr, nullptr);\n\n task_dispatcher& default_task_disp = slot->default_task_dispatcher();\n // TODO: simplify the next line, e.g. is_task_dispatcher_recalled( task_dispatcher& )\n bool is_recalled = default_task_disp.get_suspend_point()->m_is_owner_recalled.load(std::memory_order_acquire);\n task_dispatcher& target = is_recalled ? default_task_disp : create_coroutine(*m_thread_data);\n\n resume(target);\n\n if (m_properties.outermost) {\n recall_point();\n }\n}\n\nvoid task_dispatcher::suspend(suspend_callback_type suspend_callback, void* user_callback) {\n __TBB_ASSERT(suspend_callback != nullptr, nullptr);\n __TBB_ASSERT(user_callback != nullptr, nullptr);\n suspend_callback(user_callback, get_suspend_point());\n\n __TBB_ASSERT(m_thread_data != nullptr, nullptr);\n __TBB_ASSERT(m_thread_data->my_post_resume_action == post_resume_action::none, nullptr);\n __TBB_ASSERT(m_thread_data->my_post_resume_arg == nullptr, nullptr);\n internal_suspend();\n}\n\nbool task_dispatcher::resume(task_dispatcher& target) {\n // Do not create non-trivial objects on the stack of this function. They might never be destroyed\n {\n thread_data* td = m_thread_data;\n __TBB_ASSERT(&target != this, \"We cannot resume to ourself\");\n __TBB_ASSERT(td != nullptr, \"This task dispatcher must be attach to a thread data\");\n __TBB_ASSERT(td->my_task_dispatcher == this, \"Thread data must be attached to this task dispatcher\");\n\n // Change the task dispatcher\n td->detach_task_dispatcher();\n td->attach_task_dispatcher(target);\n }\n __TBB_ASSERT(m_suspend_point != nullptr, \"Suspend point must be created\");\n __TBB_ASSERT(target.m_suspend_point != nullptr, \"Suspend point must be created\");\n // Swap to the target coroutine.\n\n m_suspend_point->resume(target.m_suspend_point);\n // Pay attention that m_thread_data can be changed after resume\n if (m_thread_data) {\n thread_data* td = m_thread_data;\n __TBB_ASSERT(td != nullptr, \"This task dispatcher must be attach to a thread data\");\n __TBB_ASSERT(td->my_task_dispatcher == this, \"Thread data must be attached to this task dispatcher\");\n do_post_resume_action();\n\n // Remove the recall flag if the thread in its original task dispatcher\n arena_slot* slot = td->my_arena_slot;\n __TBB_ASSERT(slot != nullptr, nullptr);\n if (this == slot->my_default_task_dispatcher) {\n __TBB_ASSERT(m_suspend_point != nullptr, nullptr);\n m_suspend_point->m_is_owner_recalled.store(false, std::memory_order_relaxed);\n }\n return true;\n }\n return false;\n}\n\nvoid task_dispatcher::do_post_resume_action() {\n thread_data* td = m_thread_data;\n switch (td->my_post_resume_action) {\n case post_resume_action::register_waiter:\n {\n __TBB_ASSERT(td->my_post_resume_arg, \"The post resume action must have an argument\");\n static_cast(td->my_post_resume_arg)->notify();\n break;\n }\n case post_resume_action::cleanup:\n {\n __TBB_ASSERT(td->my_post_resume_arg, \"The post resume action must have an argument\");\n task_dispatcher* to_cleanup = static_cast(td->my_post_resume_arg);\n // Release coroutine's reference to my_arena\n td->my_arena->on_thread_leaving(arena::ref_external);\n // Cache the coroutine for possible later re-usage\n td->my_arena->my_co_cache.push(to_cleanup);\n break;\n }\n case post_resume_action::notify:\n {\n __TBB_ASSERT(td->my_post_resume_arg, \"The post resume action must have an argument\");\n suspend_point_type* sp = static_cast(td->my_post_resume_arg);\n sp->recall_owner();\n // Do not access sp because it can be destroyed after recall\n\n auto is_our_suspend_point = [sp] (market_context ctx) {\n return std::uintptr_t(sp) == ctx.my_uniq_addr;\n };\n td->my_arena->get_waiting_threads_monitor().notify(is_our_suspend_point);\n break;\n }\n default:\n __TBB_ASSERT(td->my_post_resume_action == post_resume_action::none, \"Unknown post resume action\");\n __TBB_ASSERT(td->my_post_resume_arg == nullptr, \"The post resume argument should not be set\");\n }\n td->clear_post_resume_action();\n}\n\n#else\n\nvoid suspend(suspend_callback_type, void*) {\n __TBB_ASSERT_RELEASE(false, \"Resumable tasks are unsupported on this platform\");\n}\n\nvoid resume(suspend_point_type*) {\n __TBB_ASSERT_RELEASE(false, \"Resumable tasks are unsupported on this platform\");\n}\n\nsuspend_point_type* current_suspend_point() {\n __TBB_ASSERT_RELEASE(false, \"Resumable tasks are unsupported on this platform\");\n return nullptr;\n}\n\n#endif /* __TBB_RESUMABLE_TASKS */\n\nvoid notify_waiters(std::uintptr_t wait_ctx_addr) {\n auto is_related_wait_ctx = [&] (market_context context) {\n return wait_ctx_addr == context.my_uniq_addr;\n };\n\n governor::get_thread_data()->my_arena->get_waiting_threads_monitor().notify(is_related_wait_ctx);\n}\n\nd1::wait_tree_vertex_interface* get_thread_reference_vertex(d1::wait_tree_vertex_interface* top_wait_context) {\n __TBB_ASSERT(top_wait_context, nullptr);\n auto& dispatcher = *governor::get_thread_data()->my_task_dispatcher;\n\n d1::reference_vertex* ref_counter{nullptr};\n auto& reference_map = dispatcher.m_reference_vertex_map;\n auto pos = reference_map.find(top_wait_context);\n if (pos != reference_map.end()) {\n ref_counter = pos->second;\n } else {\n constexpr std::size_t max_reference_vertex_map_size = 1000;\n if (reference_map.size() > max_reference_vertex_map_size) {\n // TODO: Research the possibility of using better approach for a clean-up\n for (auto it = reference_map.begin(); it != reference_map.end();) {\n if (it->second->get_num_child() == 0) {\n it->second->~reference_vertex();\n cache_aligned_deallocate(it->second);\n it = reference_map.erase(it);\n } else {\n ++it;\n }\n }\n }\n\n reference_map[top_wait_context] = ref_counter =\n new (cache_aligned_allocate(sizeof(d1::reference_vertex))) d1::reference_vertex(top_wait_context, 0);\n }\n\n return ref_counter;\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/task_dispatcher.cpp", "content": "/*\n Copyright (c) 2020-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"task_dispatcher.h\"\n#include \"waiters.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstatic inline void spawn_and_notify(d1::task& t, arena_slot* slot, arena* a) {\n slot->spawn(t);\n a->advertise_new_work();\n // TODO: TBB_REVAMP_TODO slot->assert_task_pool_valid();\n}\n\nvoid __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx) {\n thread_data* tls = governor::get_thread_data();\n task_group_context_impl::bind_to(ctx, tls);\n arena* a = tls->my_arena;\n arena_slot* slot = tls->my_arena_slot;\n // Capture current context\n task_accessor::context(t) = &ctx;\n // Mark isolation\n task_accessor::isolation(t) = tls->my_task_dispatcher->m_execute_data_ext.isolation;\n spawn_and_notify(t, slot, a);\n}\n\nvoid __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id) {\n thread_data* tls = governor::get_thread_data();\n task_group_context_impl::bind_to(ctx, tls);\n arena* a = tls->my_arena;\n arena_slot* slot = tls->my_arena_slot;\n execution_data_ext& ed = tls->my_task_dispatcher->m_execute_data_ext;\n\n // Capture context\n task_accessor::context(t) = &ctx;\n // Mark isolation\n task_accessor::isolation(t) = ed.isolation;\n\n if ( id != d1::no_slot && id != tls->my_arena_index && id < a->my_num_slots) {\n // Allocate proxy task\n d1::small_object_allocator alloc{};\n auto proxy = alloc.new_object(static_cast(ed));\n // Mark as a proxy\n task_accessor::set_proxy_trait(*proxy);\n // Mark isolation for the proxy task\n task_accessor::isolation(*proxy) = ed.isolation;\n // Deallocation hint (tls) from the task allocator\n proxy->allocator = alloc;\n proxy->slot = id;\n proxy->outbox = &a->mailbox(id);\n // Mark proxy as present in both locations (sender's task pool and destination mailbox)\n proxy->task_and_tag = intptr_t(&t) | task_proxy::location_mask;\n // Mail the proxy - after this point t may be destroyed by another thread at any moment.\n proxy->outbox->push(proxy);\n // Spawn proxy to the local task pool\n spawn_and_notify(*proxy, slot, a);\n } else {\n spawn_and_notify(t, slot, a);\n }\n}\n\nvoid __TBB_EXPORTED_FUNC submit(d1::task& t, d1::task_group_context& ctx, arena* a, std::uintptr_t as_critical) {\n suppress_unused_warning(as_critical);\n assert_pointer_valid(a);\n thread_data& tls = *governor::get_thread_data();\n\n // TODO revamp: for each use case investigate neccesity to make this call\n task_group_context_impl::bind_to(ctx, &tls);\n task_accessor::context(t) = &ctx;\n // TODO revamp: consider respecting task isolation if this call is being made by external thread\n task_accessor::isolation(t) = tls.my_task_dispatcher->m_execute_data_ext.isolation;\n\n // TODO: consider code refactoring when lane selection mechanism is unified.\n\n if ( tls.is_attached_to(a) ) {\n arena_slot* slot = tls.my_arena_slot;\n#if __TBB_PREVIEW_CRITICAL_TASKS\n if( as_critical ) {\n a->my_critical_task_stream.push( &t, subsequent_lane_selector(slot->critical_hint()) );\n } else\n#endif\n {\n slot->spawn(t);\n }\n } else {\n random_lane_selector lane_selector{tls.my_random};\n#if !__TBB_PREVIEW_CRITICAL_TASKS\n suppress_unused_warning(as_critical);\n#else\n if ( as_critical ) {\n a->my_critical_task_stream.push( &t, lane_selector );\n } else\n#endif\n {\n // Avoid joining the arena the thread is not currently in.\n a->my_fifo_task_stream.push( &t, lane_selector );\n }\n }\n // It is assumed that some thread will explicitly wait in the arena the task is submitted\n // into. Therefore, no need to utilize mandatory concurrency here.\n a->advertise_new_work();\n}\n\nvoid __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) {\n task_accessor::context(t) = &t_ctx;\n task_dispatcher::execute_and_wait(&t, wait_ctx, w_ctx);\n}\n\nvoid __TBB_EXPORTED_FUNC wait(d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) {\n // Enter the task dispatch loop without a task\n task_dispatcher::execute_and_wait(nullptr, wait_ctx, w_ctx);\n}\n\nd1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data* ed) {\n if (ed) {\n const execution_data_ext* ed_ext = static_cast(ed);\n assert_pointers_valid(ed_ext->task_disp, ed_ext->task_disp->m_thread_data);\n return ed_ext->task_disp->m_thread_data->my_arena_index;\n } else {\n thread_data* td = governor::get_thread_data_if_initialized();\n return td ? td->my_arena_index : d1::slot_id(-1);\n }\n}\n\nd1::task_group_context* __TBB_EXPORTED_FUNC current_context() {\n thread_data* td = governor::get_thread_data();\n assert_pointers_valid(td, td->my_task_dispatcher);\n\n task_dispatcher* task_disp = td->my_task_dispatcher;\n if (task_disp->m_properties.outermost) {\n // No one task is executed, so no execute_data.\n return nullptr;\n } else {\n return td->my_task_dispatcher->m_execute_data_ext.context;\n }\n}\n\nvoid task_dispatcher::execute_and_wait(d1::task* t, d1::wait_context& wait_ctx, d1::task_group_context& w_ctx) {\n // Get an associated task dispatcher\n thread_data* tls = governor::get_thread_data();\n __TBB_ASSERT(tls->my_task_dispatcher != nullptr, nullptr);\n task_dispatcher& local_td = *tls->my_task_dispatcher;\n\n // TODO: factor out the binding to execute_and_wait_impl\n if (t) {\n task_group_context_impl::bind_to(*task_accessor::context(*t), tls);\n // Propagate the isolation to the task executed without spawn.\n task_accessor::isolation(*t) = tls->my_task_dispatcher->m_execute_data_ext.isolation;\n }\n\n // Waiting on special object tied to a waiting thread.\n external_waiter waiter{ *tls->my_arena, wait_ctx };\n t = local_td.local_wait_for_all(t, waiter);\n __TBB_ASSERT_EX(t == nullptr, \"External waiter must not leave dispatch loop with a task\");\n\n // The external thread couldn't exit the dispatch loop in an idle state\n if (local_td.m_thread_data->my_inbox.is_idle_state(true)) {\n local_td.m_thread_data->my_inbox.set_is_idle(false);\n }\n\n auto exception = w_ctx.my_exception.load(std::memory_order_acquire);\n if (exception) {\n __TBB_ASSERT(w_ctx.is_group_execution_cancelled(), \"The task group context with an exception should be canceled.\");\n exception->throw_self();\n }\n}\n\n#if __TBB_RESUMABLE_TASKS\n\n#if _WIN32\n/* [[noreturn]] */ void __stdcall co_local_wait_for_all(void* addr) noexcept\n#else\n/* [[noreturn]] */ void co_local_wait_for_all(unsigned hi, unsigned lo) noexcept\n#endif\n{\n#if !_WIN32\n std::uintptr_t addr = lo;\n __TBB_ASSERT(sizeof(addr) == 8 || hi == 0, nullptr);\n addr += std::uintptr_t(std::uint64_t(hi) << 32);\n#endif\n task_dispatcher& task_disp = *reinterpret_cast(addr);\n assert_pointers_valid(task_disp.m_thread_data, task_disp.m_thread_data->my_arena);\n task_disp.set_stealing_threshold(task_disp.m_thread_data->my_arena->calculate_stealing_threshold());\n __TBB_ASSERT(task_disp.can_steal(), nullptr);\n task_disp.co_local_wait_for_all();\n // This code is unreachable\n}\n\n/* [[noreturn]] */ void task_dispatcher::co_local_wait_for_all() noexcept {\n // Do not create non-trivial objects on the stack of this function. They will never be destroyed.\n assert_pointer_valid(m_thread_data);\n\n m_suspend_point->finilize_resume();\n // Basically calls the user callback passed to the tbb::task::suspend function\n do_post_resume_action();\n\n // Endless loop here because coroutine could be reused\n d1::task* resume_task{};\n do {\n arena* a = m_thread_data->my_arena;\n coroutine_waiter waiter(*a);\n resume_task = local_wait_for_all(nullptr, waiter);\n assert_task_valid(resume_task);\n __TBB_ASSERT(this == m_thread_data->my_task_dispatcher, nullptr);\n\n m_thread_data->set_post_resume_action(post_resume_action::cleanup, this);\n\n } while (resume(static_cast(resume_task)->m_target));\n // This code might be unreachable\n}\n\nd1::suspend_point task_dispatcher::get_suspend_point() {\n if (m_suspend_point == nullptr) {\n assert_pointer_valid(m_thread_data);\n // 0 means that we attach this task dispatcher to the current stack\n init_suspend_point(m_thread_data->my_arena, 0);\n }\n assert_pointer_valid(m_suspend_point);\n return m_suspend_point;\n}\nvoid task_dispatcher::init_suspend_point(arena* a, std::size_t stack_size) {\n __TBB_ASSERT(m_suspend_point == nullptr, nullptr);\n m_suspend_point = new(cache_aligned_allocate(sizeof(suspend_point_type)))\n suspend_point_type(a, stack_size, *this);\n}\n#endif /* __TBB_RESUMABLE_TASKS */\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/task_dispatcher.h", "content": "/*\n Copyright (c) 2020-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_task_dispatcher_H\n#define _TBB_task_dispatcher_H\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/global_control.h\"\n\n#include \"scheduler_common.h\"\n#include \"waiters.h\"\n#include \"arena_slot.h\"\n#include \"arena.h\"\n#include \"thread_data.h\"\n#include \"mailbox.h\"\n#include \"itt_notify.h\"\n#include \"concurrent_monitor.h\"\n#include \"threading_control.h\"\n\n#include \n\n#if !__TBB_CPU_CTL_ENV_PRESENT\n#include //\n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\ninline d1::task* get_self_recall_task(arena_slot& slot) {\n suppress_unused_warning(slot);\n d1::task* t = nullptr;\n#if __TBB_RESUMABLE_TASKS\n suspend_point_type* sp = slot.default_task_dispatcher().m_suspend_point;\n if (sp && sp->m_is_owner_recalled.load(std::memory_order_acquire)) {\n t = &sp->m_resume_task;\n __TBB_ASSERT(sp->m_resume_task.m_target.m_thread_data == nullptr, nullptr);\n }\n#endif /* __TBB_RESUMABLE_TASKS */\n return t;\n}\n\n// Defined in exception.cpp\n/*[[noreturn]]*/void do_throw_noexcept(void (*throw_exception)()) noexcept;\n\n//------------------------------------------------------------------------\n// Suspend point\n//------------------------------------------------------------------------\n#if __TBB_RESUMABLE_TASKS\n\ninline d1::task* suspend_point_type::resume_task::execute(d1::execution_data& ed) {\n execution_data_ext& ed_ext = static_cast(ed);\n\n if (ed_ext.wait_ctx) {\n thread_control_monitor::resume_context monitor_node{{std::uintptr_t(ed_ext.wait_ctx), nullptr}, ed_ext, m_target};\n // The wait_ctx is present only in external_waiter. In that case we leave the current stack\n // in the abandoned state to resume when waiting completes.\n thread_data* td = ed_ext.task_disp->m_thread_data;\n td->set_post_resume_action(task_dispatcher::post_resume_action::register_waiter, &monitor_node);\n\n thread_control_monitor& wait_list = td->my_arena->get_waiting_threads_monitor();\n\n if (wait_list.wait([&] { return !ed_ext.wait_ctx->continue_execution(); }, monitor_node)) {\n return nullptr;\n }\n\n td->clear_post_resume_action();\n r1::resume(ed_ext.task_disp->get_suspend_point());\n } else {\n // If wait_ctx is null, it can be only a worker thread on outermost level because\n // coroutine_waiter interrupts bypass loop before the resume_task execution.\n ed_ext.task_disp->m_thread_data->set_post_resume_action(task_dispatcher::post_resume_action::notify,\n ed_ext.task_disp->get_suspend_point());\n }\n // Do not access this task because it might be destroyed\n ed_ext.task_disp->resume(m_target);\n return nullptr;\n}\n\ninline suspend_point_type::suspend_point_type(arena* a, size_t stack_size, task_dispatcher& task_disp)\n : m_arena(a)\n , m_random(this)\n , m_co_context(stack_size, &task_disp)\n , m_resume_task(task_disp)\n{\n assert_pointer_valid(m_arena);\n assert_pointer_valid(m_arena->my_default_ctx);\n task_accessor::context(m_resume_task) = m_arena->my_default_ctx;\n task_accessor::isolation(m_resume_task) = no_isolation;\n // Initialize the itt_caller for the context of the resume task.\n // It will be bound to the stack of the first suspend call.\n task_group_context_impl::bind_to(*task_accessor::context(m_resume_task), task_disp.m_thread_data);\n}\n\n#endif /* __TBB_RESUMABLE_TASKS */\n\n//------------------------------------------------------------------------\n// Task Dispatcher\n//------------------------------------------------------------------------\ninline task_dispatcher::task_dispatcher(arena* a) {\n m_execute_data_ext.context = a->my_default_ctx;\n m_execute_data_ext.task_disp = this;\n}\n\ninline bool task_dispatcher::can_steal() {\n __TBB_ASSERT(m_stealing_threshold != 0, nullptr);\n stack_anchor_type anchor{};\n return reinterpret_cast(&anchor) > m_stealing_threshold;\n}\n\ninline d1::task* task_dispatcher::get_inbox_or_critical_task(\n execution_data_ext& ed, mail_inbox& inbox, isolation_type isolation, bool critical_allowed)\n{\n if (inbox.empty())\n return nullptr;\n d1::task* result = get_critical_task(nullptr, ed, isolation, critical_allowed);\n if (result)\n return result;\n // Check if there are tasks mailed to this thread via task-to-thread affinity mechanism.\n result = get_mailbox_task(inbox, ed, isolation);\n // There is a race with a thread adding a new task (possibly with suitable isolation)\n // to our mailbox, so the below conditions might result in a false positive.\n // Then set_is_idle(false) allows that task to be stolen; it's OK.\n if (isolation != no_isolation && !result && !inbox.empty() && inbox.is_idle_state(true)) {\n // We have proxy tasks in our mailbox but the isolation blocks their execution.\n // So publish the proxy tasks in mailbox to be available for stealing from owner's task pool.\n inbox.set_is_idle( false );\n }\n return result;\n}\n\ninline d1::task* task_dispatcher::get_stream_or_critical_task(\n execution_data_ext& ed, arena& a, task_stream& stream, unsigned& hint,\n isolation_type isolation, bool critical_allowed)\n{\n if (stream.empty())\n return nullptr;\n d1::task* result = get_critical_task(nullptr, ed, isolation, critical_allowed);\n if (result)\n return result;\n return a.get_stream_task(stream, hint);\n}\n\ninline d1::task* task_dispatcher::steal_or_get_critical(\n execution_data_ext& ed, arena& a, unsigned arena_index, FastRandom& random,\n isolation_type isolation, bool critical_allowed)\n{\n if (d1::task* t = a.steal_task(arena_index, random, ed, isolation)) {\n ed.context = task_accessor::context(*t);\n ed.isolation = task_accessor::isolation(*t);\n return get_critical_task(t, ed, isolation, critical_allowed);\n }\n return nullptr;\n}\n\ntemplate \nd1::task* task_dispatcher::receive_or_steal_task(\n thread_data& tls, execution_data_ext& ed, Waiter& waiter, isolation_type isolation,\n bool fifo_allowed, bool critical_allowed)\n{\n __TBB_ASSERT(governor::is_thread_data_set(&tls), nullptr);\n // Task to return\n d1::task* t = nullptr;\n // Get tls data (again)\n arena& a = *tls.my_arena;\n arena_slot& slot = *tls.my_arena_slot;\n unsigned arena_index = tls.my_arena_index;\n mail_inbox& inbox = tls.my_inbox;\n task_stream& resume_stream = a.my_resume_task_stream;\n unsigned& resume_hint = slot.hint_for_resume_stream;\n task_stream& fifo_stream = a.my_fifo_task_stream;\n unsigned& fifo_hint = slot.hint_for_fifo_stream;\n\n waiter.reset_wait();\n // Thread is in idle state now\n inbox.set_is_idle(true);\n\n bool stealing_is_allowed = can_steal();\n\n // Stealing loop mailbox/enqueue/other_slots\n for (;;) {\n __TBB_ASSERT(t == nullptr, nullptr);\n // Check if the resource manager requires our arena to relinquish some threads\n // For the external thread restore idle state to true after dispatch loop\n if (!waiter.continue_execution(slot, t)) {\n __TBB_ASSERT(t == nullptr, nullptr);\n break;\n }\n // Start searching\n if (t != nullptr) {\n // continue_execution returned a task\n }\n else if ((t = get_inbox_or_critical_task(ed, inbox, isolation, critical_allowed))) {\n // Successfully got the task from mailbox or critical task\n }\n else if ((t = get_stream_or_critical_task(ed, a, resume_stream, resume_hint, isolation, critical_allowed))) {\n // Successfully got the resume or critical task\n }\n else if (fifo_allowed && isolation == no_isolation\n && (t = get_stream_or_critical_task(ed, a, fifo_stream, fifo_hint, isolation, critical_allowed))) {\n // Checked if there are tasks in starvation-resistant stream. Only allowed at the outermost dispatch level without isolation.\n }\n else if (stealing_is_allowed\n && (t = steal_or_get_critical(ed, a, arena_index, tls.my_random, isolation, critical_allowed))) {\n // Stole a task from a random arena slot\n }\n else {\n t = get_critical_task(t, ed, isolation, critical_allowed);\n }\n\n if (t != nullptr) {\n ed.context = task_accessor::context(*t);\n ed.isolation = task_accessor::isolation(*t);\n a.my_observers.notify_entry_observers(tls.my_last_observer, tls.my_is_worker);\n break; // Stealing success, end of stealing attempt\n }\n // Nothing to do, pause a little.\n waiter.pause(slot);\n } // end of nonlocal task retrieval loop\n\n __TBB_ASSERT(is_alive(a.my_guard), nullptr);\n if (inbox.is_idle_state(true)) {\n inbox.set_is_idle(false);\n }\n return t;\n}\n\ntemplate \nd1::task* task_dispatcher::local_wait_for_all(d1::task* t, Waiter& waiter ) {\n assert_pointer_valid(m_thread_data);\n __TBB_ASSERT(m_thread_data->my_task_dispatcher == this, nullptr);\n\n // Guard an outer/default execution state\n struct dispatch_loop_guard {\n task_dispatcher& task_disp;\n execution_data_ext old_execute_data_ext;\n properties old_properties;\n bool is_initially_registered;\n\n ~dispatch_loop_guard() {\n task_disp.m_execute_data_ext = old_execute_data_ext;\n task_disp.m_properties = old_properties;\n\n if (!is_initially_registered) {\n task_disp.m_thread_data->my_arena->my_tc_client.get_pm_client()->unregister_thread();\n task_disp.m_thread_data->my_is_registered = false;\n }\n\n __TBB_ASSERT(task_disp.m_thread_data && governor::is_thread_data_set(task_disp.m_thread_data), nullptr);\n __TBB_ASSERT(task_disp.m_thread_data->my_task_dispatcher == &task_disp, nullptr);\n }\n } dl_guard{ *this, m_execute_data_ext, m_properties, m_thread_data->my_is_registered };\n\n // The context guard to track fp setting and itt tasks.\n context_guard_helper context_guard;\n\n // Current isolation context\n const isolation_type isolation = dl_guard.old_execute_data_ext.isolation;\n\n // Critical work inflection point. Once turned false current execution context has taken\n // critical task on the previous stack frame and cannot take more until that critical path is\n // finished.\n bool critical_allowed = dl_guard.old_properties.critical_task_allowed;\n\n // Extended execution data that is used for dispatching.\n // Base version is passed to the task::execute method.\n execution_data_ext& ed = m_execute_data_ext;\n ed.context = t ? task_accessor::context(*t) : nullptr;\n ed.original_slot = m_thread_data->my_arena_index;\n ed.affinity_slot = d1::no_slot;\n ed.task_disp = this;\n ed.wait_ctx = waiter.wait_ctx();\n\n m_properties.outermost = false;\n m_properties.fifo_tasks_allowed = false;\n\n if (!dl_guard.is_initially_registered) {\n m_thread_data->my_arena->my_tc_client.get_pm_client()->register_thread();\n m_thread_data->my_is_registered = true;\n }\n\n t = get_critical_task(t, ed, isolation, critical_allowed);\n if (t && m_thread_data->my_inbox.is_idle_state(true)) {\n // The thread has a work to do. Therefore, marking its inbox as not idle so that\n // affinitized tasks can be stolen from it.\n m_thread_data->my_inbox.set_is_idle(false);\n }\n\n // Infinite exception loop\n for (;;) {\n try {\n // Main execution loop\n do {\n // We assume that bypass tasks are from the same task group.\n context_guard.set_ctx(ed.context);\n // Inner level evaluates tasks coming from nesting loops and those returned\n // by just executed tasks (bypassing spawn or enqueue calls).\n while (t != nullptr) {\n assert_task_valid(t);\n assert_pointer_valid(ed.context);\n __TBB_ASSERT(ed.context->my_state == d1::task_group_context::state::bound ||\n ed.context->my_state == d1::task_group_context::state::isolated, nullptr);\n __TBB_ASSERT(m_thread_data->my_inbox.is_idle_state(false), nullptr);\n __TBB_ASSERT(task_accessor::is_resume_task(*t) || isolation == no_isolation || isolation == ed.isolation, nullptr);\n // Check premature leave\n if (Waiter::postpone_execution(*t)) {\n __TBB_ASSERT(task_accessor::is_resume_task(*t) && dl_guard.old_properties.outermost,\n \"Currently, the bypass loop can be interrupted only for resume task on outermost level\");\n return t;\n }\n // Copy itt_caller to a stack because the context might be destroyed after t->execute.\n void* itt_caller = ed.context->my_itt_caller;\n suppress_unused_warning(itt_caller);\n\n ITT_CALLEE_ENTER(ITTPossible, t, itt_caller);\n\n if (ed.context->is_group_execution_cancelled()) {\n t = t->cancel(ed);\n } else {\n t = t->execute(ed);\n }\n\n ITT_CALLEE_LEAVE(ITTPossible, itt_caller);\n\n // The task affinity in execution data is set for affinitized tasks.\n // So drop it after the task execution.\n ed.affinity_slot = d1::no_slot;\n // Reset task owner id for bypassed task\n ed.original_slot = m_thread_data->my_arena_index;\n t = get_critical_task(t, ed, isolation, critical_allowed);\n }\n __TBB_ASSERT(m_thread_data && governor::is_thread_data_set(m_thread_data), nullptr);\n __TBB_ASSERT(m_thread_data->my_task_dispatcher == this, nullptr);\n // When refactoring, pay attention that m_thread_data can be changed after t->execute()\n __TBB_ASSERT(m_thread_data->my_arena_slot != nullptr, nullptr);\n arena_slot& slot = *m_thread_data->my_arena_slot;\n if (!waiter.continue_execution(slot, t)) {\n break;\n }\n // Retrieve the task from local task pool\n if (t || (slot.is_task_pool_published() && (t = slot.get_task(ed, isolation)))) {\n __TBB_ASSERT(ed.original_slot == m_thread_data->my_arena_index, nullptr);\n ed.context = task_accessor::context(*t);\n ed.isolation = task_accessor::isolation(*t);\n continue;\n }\n // Retrieve the task from global sources\n t = receive_or_steal_task(\n *m_thread_data, ed, waiter, isolation, dl_guard.old_properties.fifo_tasks_allowed,\n critical_allowed\n );\n } while (t != nullptr); // main dispatch loop\n break; // Exit exception loop;\n } catch (...) {\n if (global_control::active_value(global_control::terminate_on_exception) == 1) {\n do_throw_noexcept([] { throw; });\n }\n if (ed.context->cancel_group_execution()) {\n /* We are the first to signal cancellation, so store the exception that caused it. */\n ed.context->my_exception.store(tbb_exception_ptr::allocate(), std::memory_order_release);\n }\n }\n } // Infinite exception loop\n __TBB_ASSERT(t == nullptr, nullptr);\n\n\n#if __TBB_RESUMABLE_TASKS\n if (dl_guard.old_properties.outermost) {\n recall_point();\n }\n#endif /* __TBB_RESUMABLE_TASKS */\n\n return nullptr;\n}\n\n#if __TBB_RESUMABLE_TASKS\ninline void task_dispatcher::recall_point() {\n if (this != &m_thread_data->my_arena_slot->default_task_dispatcher()) {\n __TBB_ASSERT(m_suspend_point != nullptr, nullptr);\n __TBB_ASSERT(m_suspend_point->m_is_owner_recalled.load(std::memory_order_relaxed) == false, nullptr);\n\n m_thread_data->set_post_resume_action(post_resume_action::notify, get_suspend_point());\n internal_suspend();\n\n if (m_thread_data->my_inbox.is_idle_state(true)) {\n m_thread_data->my_inbox.set_is_idle(false);\n }\n }\n}\n#endif /* __TBB_RESUMABLE_TASKS */\n\n#if __TBB_PREVIEW_CRITICAL_TASKS\ninline d1::task* task_dispatcher::get_critical_task(d1::task* t, execution_data_ext& ed, isolation_type isolation, bool critical_allowed) {\n __TBB_ASSERT( critical_allowed || !m_properties.critical_task_allowed, nullptr );\n\n if (!critical_allowed) {\n // The stack is already in the process of critical path execution. Cannot take another\n // critical work until finish with the current one.\n __TBB_ASSERT(!m_properties.critical_task_allowed, nullptr);\n return t;\n }\n\n assert_pointers_valid(m_thread_data, m_thread_data->my_arena, m_thread_data->my_arena_slot);\n thread_data& td = *m_thread_data;\n arena& a = *td.my_arena;\n arena_slot& slot = *td.my_arena_slot;\n\n d1::task* crit_t = a.get_critical_task(slot.hint_for_critical_stream, isolation);\n if (crit_t != nullptr) {\n assert_task_valid(crit_t);\n if (t != nullptr) {\n assert_pointer_valid(ed.context);\n r1::spawn(*t, *ed.context);\n }\n ed.context = task_accessor::context(*crit_t);\n ed.isolation = task_accessor::isolation(*crit_t);\n\n // We cannot execute more than one critical task on the same stack.\n // In other words, we prevent nested critical tasks.\n m_properties.critical_task_allowed = false;\n\n // TODO: add a test that the observer is called when critical task is taken.\n a.my_observers.notify_entry_observers(td.my_last_observer, td.my_is_worker);\n t = crit_t;\n } else {\n // Was unable to find critical work in the queue. Allow inspecting the queue in nested\n // invocations. Handles the case when critical task has been just completed.\n m_properties.critical_task_allowed = true;\n }\n return t;\n}\n#else\ninline d1::task* task_dispatcher::get_critical_task(d1::task* t, execution_data_ext&, isolation_type, bool /*critical_allowed*/) {\n return t;\n}\n#endif\n\ninline d1::task* task_dispatcher::get_mailbox_task(mail_inbox& my_inbox, execution_data_ext& ed, isolation_type isolation) {\n while (task_proxy* const tp = my_inbox.pop(isolation)) {\n if (d1::task* result = tp->extract_task()) {\n ed.original_slot = (unsigned short)(-2);\n ed.affinity_slot = ed.task_disp->m_thread_data->my_arena_index;\n return result;\n }\n // We have exclusive access to the proxy, and can destroy it.\n tp->allocator.delete_object(tp, ed);\n }\n return nullptr;\n}\n\ntemplate \nd1::task* task_dispatcher::local_wait_for_all(d1::task* t, Waiter& waiter) {\n if (governor::is_itt_present()) {\n return local_wait_for_all(t, waiter);\n } else {\n return local_wait_for_all(t, waiter);\n }\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_task_dispatcher_H\n\n" }, { "path": "src/tbb/src/tbb/task_group_context.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/tbb_allocator.h\"\n#include \"oneapi/tbb/task_group.h\"\n#include \"governor.h\"\n#include \"thread_data.h\"\n#include \"scheduler_common.h\"\n#include \"itt_notify.h\"\n#include \"task_dispatcher.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// tbb_exception_ptr\n//------------------------------------------------------------------------\ntbb_exception_ptr* tbb_exception_ptr::allocate() noexcept {\n tbb_exception_ptr* eptr = (tbb_exception_ptr*)allocate_memory(sizeof(tbb_exception_ptr));\n return eptr ? new (eptr) tbb_exception_ptr(std::current_exception()) : nullptr;\n}\n\nvoid tbb_exception_ptr::destroy() noexcept {\n this->~tbb_exception_ptr();\n deallocate_memory(this);\n}\n\nvoid tbb_exception_ptr::throw_self() {\n if (governor::rethrow_exception_broken()) fix_broken_rethrow();\n std::rethrow_exception(my_ptr);\n}\n\n//------------------------------------------------------------------------\n// task_group_context\n//------------------------------------------------------------------------\n\nvoid task_group_context_impl::destroy(d1::task_group_context& ctx) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n\n if (ctx.my_context_list != nullptr) {\n __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::bound, nullptr);\n // The owner can be destroyed at any moment. Access the associate data with caution.\n ctx.my_context_list->remove(ctx.my_node);\n }\n d1::cpu_ctl_env* ctl = reinterpret_cast(&ctx.my_cpu_ctl_env);\n#if _MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER\n suppress_unused_warning(ctl);\n#endif\n ctl->~cpu_ctl_env();\n\n auto exception = ctx.my_exception.load(std::memory_order_relaxed);\n if (exception) {\n exception->destroy();\n }\n ITT_STACK_DESTROY(ctx.my_itt_caller);\n\n poison_pointer(ctx.my_parent);\n poison_pointer(ctx.my_context_list);\n poison_pointer(ctx.my_node.my_next_node);\n poison_pointer(ctx.my_node.my_prev_node);\n poison_pointer(ctx.my_exception);\n poison_pointer(ctx.my_itt_caller);\n\n ctx.my_state.store(d1::task_group_context::state::dead, std::memory_order_release);\n}\n\nvoid task_group_context_impl::initialize(d1::task_group_context& ctx) {\n ITT_TASK_GROUP(&ctx, ctx.my_name, nullptr);\n\n ctx.my_node.my_next_node = &ctx.my_node;\n ctx.my_node.my_prev_node = &ctx.my_node;\n ctx.my_cpu_ctl_env = 0;\n ctx.my_cancellation_requested = 0;\n ctx.my_may_have_children.store(0, std::memory_order_relaxed);\n // Set the created state to bound at the first usage.\n ctx.my_state.store(d1::task_group_context::state::created, std::memory_order_relaxed);\n ctx.my_parent = nullptr;\n ctx.my_context_list = nullptr;\n ctx.my_exception.store(nullptr, std::memory_order_relaxed);\n ctx.my_itt_caller = nullptr;\n\n static_assert(sizeof(d1::cpu_ctl_env) <= sizeof(ctx.my_cpu_ctl_env), \"FPU settings storage does not fit to uint64_t\");\n d1::cpu_ctl_env* ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env;\n if (ctx.my_traits.fp_settings)\n ctl->get_env();\n}\n\nvoid task_group_context_impl::register_with(d1::task_group_context& ctx, thread_data* td) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n __TBB_ASSERT(td, nullptr);\n ctx.my_context_list = td->my_context_list;\n\n ctx.my_context_list->push_front(ctx.my_node);\n}\n\nvoid task_group_context_impl::bind_to_impl(d1::task_group_context& ctx, thread_data* td) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) == d1::task_group_context::state::locked, \"The context can be bound only under the lock.\");\n __TBB_ASSERT(!ctx.my_parent, \"Parent is set before initial binding\");\n\n ctx.my_parent = td->my_task_dispatcher->m_execute_data_ext.context;\n __TBB_ASSERT(ctx.my_parent, nullptr);\n\n // Inherit FPU settings only if the context has not captured FPU settings yet.\n if (!ctx.my_traits.fp_settings)\n copy_fp_settings(ctx, *ctx.my_parent);\n\n // Condition below prevents unnecessary thrashing parent context's cache line\n if (ctx.my_parent->my_may_have_children.load(std::memory_order_relaxed) != d1::task_group_context::may_have_children) {\n ctx.my_parent->my_may_have_children.store(d1::task_group_context::may_have_children, std::memory_order_relaxed); // full fence is below\n }\n if (ctx.my_parent->my_parent) {\n // Even if this context were made accessible for state change propagation\n // (by placing store_with_release(td->my_context_list_state.head.my_next, &ctx.my_node)\n // above), it still could be missed if state propagation from a grand-ancestor\n // was underway concurrently with binding.\n // Speculative propagation from the parent together with epoch counters\n // detecting possibility of such a race allow to avoid taking locks when\n // there is no contention.\n\n // Acquire fence is necessary to prevent reordering subsequent speculative\n // loads of parent state data out of the scope where epoch counters comparison\n // can reliably validate it.\n uintptr_t local_count_snapshot = ctx.my_parent->my_context_list->epoch.load(std::memory_order_acquire);\n // Speculative propagation of parent's state. The speculation will be\n // validated by the epoch counters check further on.\n ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed);\n register_with(ctx, td); // Issues full fence\n\n // If no state propagation was detected by the following condition, the above\n // full fence guarantees that the parent had correct state during speculative\n // propagation before the fence. Otherwise the propagation from parent is\n // repeated under the lock.\n if (local_count_snapshot != the_context_state_propagation_epoch.load(std::memory_order_relaxed)) {\n // Another thread may be propagating state change right now. So resort to lock.\n context_state_propagation_mutex_type::scoped_lock lock(the_context_state_propagation_mutex);\n ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed);\n }\n } else {\n register_with(ctx, td); // Issues full fence\n // As we do not have grand-ancestors, concurrent state propagation (if any)\n // may originate only from the parent context, and thus it is safe to directly\n // copy the state from it.\n ctx.my_cancellation_requested.store(ctx.my_parent->my_cancellation_requested.load(std::memory_order_relaxed), std::memory_order_relaxed);\n }\n}\n\nvoid task_group_context_impl::bind_to(d1::task_group_context& ctx, thread_data* td) {\n d1::task_group_context::state state = ctx.my_state.load(std::memory_order_acquire);\n if (state <= d1::task_group_context::state::locked) {\n if (state == d1::task_group_context::state::created &&\n#if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910\n ((std::atomic::type>&)ctx.my_state).compare_exchange_strong(\n (typename std::underlying_type::type&)state,\n (typename std::underlying_type::type)d1::task_group_context::state::locked)\n#else\n ctx.my_state.compare_exchange_strong(state, d1::task_group_context::state::locked)\n#endif\n ) {\n // If we are in the outermost task dispatch loop of an external thread, then\n // there is nothing to bind this context to, and we skip the binding part\n // treating the context as isolated.\n __TBB_ASSERT(td->my_task_dispatcher->m_execute_data_ext.context != nullptr, nullptr);\n d1::task_group_context::state release_state{};\n if (td->my_task_dispatcher->m_execute_data_ext.context == td->my_arena->my_default_ctx || !ctx.my_traits.bound) {\n if (!ctx.my_traits.fp_settings) {\n copy_fp_settings(ctx, *td->my_arena->my_default_ctx);\n }\n release_state = d1::task_group_context::state::isolated;\n } else {\n bind_to_impl(ctx, td);\n release_state = d1::task_group_context::state::bound;\n }\n ITT_STACK_CREATE(ctx.my_itt_caller);\n ctx.my_state.store(release_state, std::memory_order_release);\n }\n spin_wait_while_eq(ctx.my_state, d1::task_group_context::state::locked);\n }\n __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::created, nullptr);\n __TBB_ASSERT(ctx.my_state.load(std::memory_order_relaxed) != d1::task_group_context::state::locked, nullptr);\n}\n\nvoid task_group_context_impl::propagate_task_group_state(d1::task_group_context& ctx, std::atomic d1::task_group_context::* mptr_state, d1::task_group_context& src, std::uint32_t new_state) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n /* 1. if ((ctx.*mptr_state).load(std::memory_order_relaxed) == new_state):\n Nothing to do, whether descending from \"src\" or not, so no need to scan.\n Hopefully this happens often thanks to earlier invocations.\n This optimization is enabled by LIFO order in the context lists:\n - new contexts are bound to the beginning of lists;\n - descendants are newer than ancestors;\n - earlier invocations are therefore likely to \"paint\" long chains.\n 2. if (&ctx != &src):\n This clause is disjunct from the traversal below, which skips src entirely.\n Note that src.*mptr_state is not necessarily still equal to new_state (another thread may have changed it again).\n Such interference is probably not frequent enough to aim for optimisation by writing new_state again (to make the other thread back down).\n Letting the other thread prevail may also be fairer.\n */\n if ((ctx.*mptr_state).load(std::memory_order_relaxed) != new_state && &ctx != &src) {\n for (d1::task_group_context* ancestor = ctx.my_parent; ancestor != nullptr; ancestor = ancestor->my_parent) {\n if (ancestor == &src) {\n for (d1::task_group_context* c = &ctx; c != ancestor; c = c->my_parent)\n (c->*mptr_state).store(new_state, std::memory_order_relaxed);\n break;\n }\n }\n }\n}\n\nbool task_group_context_impl::cancel_group_execution(d1::task_group_context& ctx) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n __TBB_ASSERT(ctx.my_cancellation_requested.load(std::memory_order_relaxed) <= 1, \"The cancellation state can be either 0 or 1\");\n if (ctx.my_cancellation_requested.load(std::memory_order_relaxed) || ctx.my_cancellation_requested.exchange(1)) {\n // This task group and any descendants have already been canceled.\n // (A newly added descendant would inherit its parent's ctx.my_cancellation_requested,\n // not missing out on any cancellation still being propagated, and a context cannot be uncanceled.)\n return false;\n }\n governor::get_thread_data()->my_arena->my_threading_control->propagate_task_group_state(&d1::task_group_context::my_cancellation_requested, ctx, uint32_t(1));\n return true;\n}\n\nbool task_group_context_impl::is_group_execution_cancelled(const d1::task_group_context& ctx) {\n return ctx.my_cancellation_requested.load(std::memory_order_relaxed) != 0;\n}\n\n// IMPORTANT: It is assumed that this method is not used concurrently!\nvoid task_group_context_impl::reset(d1::task_group_context& ctx) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n //! TODO: Add assertion that this context does not have children\n // No fences are necessary since this context can be accessed from another thread\n // only after stealing happened (which means necessary fences were used).\n\n auto exception = ctx.my_exception.load(std::memory_order_relaxed);\n if (exception) {\n exception->destroy();\n ctx.my_exception.store(nullptr, std::memory_order_relaxed);\n }\n ctx.my_cancellation_requested = 0;\n}\n\n// IMPORTANT: It is assumed that this method is not used concurrently!\nvoid task_group_context_impl::capture_fp_settings(d1::task_group_context& ctx) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n //! TODO: Add assertion that this context does not have children\n // No fences are necessary since this context can be accessed from another thread\n // only after stealing happened (which means necessary fences were used).\n d1::cpu_ctl_env* ctl = reinterpret_cast(&ctx.my_cpu_ctl_env);\n if (!ctx.my_traits.fp_settings) {\n ctl = new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env;\n ctx.my_traits.fp_settings = true;\n }\n ctl->get_env();\n}\n\nvoid task_group_context_impl::copy_fp_settings(d1::task_group_context& ctx, const d1::task_group_context& src) {\n __TBB_ASSERT(!is_poisoned(ctx.my_context_list), nullptr);\n __TBB_ASSERT(!ctx.my_traits.fp_settings, \"The context already has FPU settings.\");\n __TBB_ASSERT(src.my_traits.fp_settings, \"The source context does not have FPU settings.\");\n\n const d1::cpu_ctl_env* src_ctl = reinterpret_cast(&src.my_cpu_ctl_env);\n new (&ctx.my_cpu_ctl_env) d1::cpu_ctl_env(*src_ctl);\n ctx.my_traits.fp_settings = true;\n}\n\n/*\n Comments:\n\n1. The premise of the cancellation support implementation is that cancellations are\n not part of the hot path of the program execution. Therefore all changes in its\n implementation in order to reduce the overhead of the cancellation control flow\n should be done only in ways that do not increase overhead of the normal execution.\n\n In general, contexts are used by all threads and their descendants are created in\n different threads as well. In order to minimize impact of the cross-thread tree\n maintenance (first of all because of the synchronization), the tree of contexts\n is split into pieces, each of which is handled by a single thread. Such pieces\n are represented as lists of contexts, members of which are contexts that were\n bound to their parents in the given thread.\n\n The context tree maintenance and cancellation propagation algorithms are designed\n in such a manner that cross-thread access to a context list will take place only\n when cancellation signal is sent (by user or when an exception happens), and\n synchronization is necessary only then. Thus the normal execution flow (without\n exceptions and cancellation) remains free from any synchronization done on\n behalf of exception handling and cancellation support.\n\n2. Consider parallel cancellations at the different levels of the context tree:\n\n Ctx1 <- Cancelled by Thread1 |- Thread2 started processing\n | |\n Ctx2 |- Thread1 started processing\n | T1 |- Thread2 finishes and syncs up local counters\n Ctx3 <- Cancelled by Thread2 |\n | |- Ctx5 is bound to Ctx2\n Ctx4 |\n T2 |- Thread1 reaches Ctx2\n\n Thread-propagator of each cancellation increments global counter. However the thread\n propagating the cancellation from the outermost context (Thread1) may be the last\n to finish. Which means that the local counters may be synchronized earlier (by Thread2,\n at Time1) than it propagated cancellation into Ctx2 (at time Time2). If a new context\n (Ctx5) is created and bound to Ctx2 between Time1 and Time2, checking its parent only\n (Ctx2) may result in cancellation request being lost.\n\n This issue is solved by doing the whole propagation under the lock.\n\n If we need more concurrency while processing parallel cancellations, we could try\n the following modification of the propagation algorithm:\n\n advance global counter and remember it\n for each thread:\n scan thread's list of contexts\n for each thread:\n sync up its local counter only if the global counter has not been changed\n\n However this version of the algorithm requires more analysis and verification.\n*/\n\nvoid __TBB_EXPORTED_FUNC initialize(d1::task_group_context& ctx) {\n task_group_context_impl::initialize(ctx);\n}\nvoid __TBB_EXPORTED_FUNC destroy(d1::task_group_context& ctx) {\n task_group_context_impl::destroy(ctx);\n}\nvoid __TBB_EXPORTED_FUNC reset(d1::task_group_context& ctx) {\n task_group_context_impl::reset(ctx);\n}\nbool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context& ctx) {\n return task_group_context_impl::cancel_group_execution(ctx);\n}\nbool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context& ctx) {\n return task_group_context_impl::is_group_execution_cancelled(ctx);\n}\nvoid __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context& ctx) {\n task_group_context_impl::capture_fp_settings(ctx);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n" }, { "path": "src/tbb/src/tbb/task_stream.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_task_stream_H\n#define _TBB_task_stream_H\n\n//! This file is a possible future replacement for the task_stream class implemented in\n//! task_stream.h. It refactors the code and extends task_stream capabilities by moving lane\n//! management during operations on caller side. Despite the fact that new implementation should not\n//! affect performance of the original task stream, analysis on this subject was not made at the\n//! time it was developed. In addition, it is not clearly seen at the moment that this container\n//! would be suitable for critical tasks due to linear time complexity on its operations.\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/cache_aligned_allocator.h\"\n#include \"oneapi/tbb/mutex.h\"\n\n#include \"scheduler_common.h\"\n#include \"misc.h\" // for FastRandom\n\n#include \n#include \n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//! Essentially, this is just a pair of a queue and a mutex to protect the queue.\n/** The reason std::pair is not used is that the code would look less clean\n if field names were replaced with 'first' and 'second'. **/\ntemplate< typename T, typename mutex_t >\nstruct alignas(max_nfs_size) queue_and_mutex {\n typedef std::deque< T, cache_aligned_allocator > queue_base_t;\n\n queue_base_t my_queue{};\n mutex_t my_mutex{};\n};\n\nusing population_t = uintptr_t;\nconst population_t one = 1;\n\ninline void set_one_bit( std::atomic& dest, int pos ) {\n __TBB_ASSERT( pos>=0, nullptr);\n __TBB_ASSERT( pos& dest, int pos ) {\n __TBB_ASSERT( pos>=0, nullptr);\n __TBB_ASSERT( pos=0, nullptr);\n __TBB_ASSERT( pos\nclass task_stream_accessor : no_copy {\nprotected:\n using lane_t = queue_and_mutex ;\n d1::task* get_item( lane_t::queue_base_t& queue ) {\n d1::task* result = queue.front();\n queue.pop_front();\n return result;\n }\n};\n\ntemplate<>\nclass task_stream_accessor< back_nonnull_accessor > : no_copy {\nprotected:\n using lane_t = queue_and_mutex ;\n d1::task* get_item( lane_t::queue_base_t& queue ) {\n d1::task* result = nullptr;\n __TBB_ASSERT(!queue.empty(), nullptr);\n // Isolated task can put zeros in queue see look_specific\n do {\n result = queue.back();\n queue.pop_back();\n } while ( !result && !queue.empty() );\n return result;\n }\n};\n\n//! The container for \"fairness-oriented\" aka \"enqueued\" tasks.\ntemplate\nclass task_stream : public task_stream_accessor< accessor > {\n using lane_t = typename task_stream_accessor::lane_t;\n std::atomic population{};\n lane_t* lanes{nullptr};\n unsigned N{};\n\npublic:\n task_stream() = default;\n\n void initialize( unsigned n_lanes ) {\n const unsigned max_lanes = sizeof(population_t) * CHAR_BIT;\n\n N = n_lanes >= max_lanes ? max_lanes : n_lanes > 2 ? 1 << (tbb::detail::log2(n_lanes - 1) + 1) : 2;\n __TBB_ASSERT( N == max_lanes || (N >= n_lanes && ((N - 1) & N) == 0), \"number of lanes miscalculated\" );\n __TBB_ASSERT( N <= sizeof(population_t) * CHAR_BIT, nullptr);\n lanes = static_cast(cache_aligned_allocate(sizeof(lane_t) * N));\n for (unsigned i = 0; i < N; ++i) {\n new (lanes + i) lane_t;\n }\n __TBB_ASSERT( !population.load(std::memory_order_relaxed), nullptr);\n }\n\n ~task_stream() {\n if (lanes) {\n for (unsigned i = 0; i < N; ++i) {\n lanes[i].~lane_t();\n }\n cache_aligned_deallocate(lanes);\n }\n }\n\n //! Push a task into a lane. Lane selection is performed by passed functor.\n template\n void push(d1::task* source, const lane_selector_t& next_lane ) {\n bool succeed = false;\n unsigned lane = 0;\n do {\n lane = next_lane( /*out_of=*/N );\n __TBB_ASSERT( lane < N, \"Incorrect lane index.\" );\n } while( ! (succeed = try_push( source, lane )) );\n }\n\n //! Try finding and popping a task using passed functor for lane selection. Last used lane is\n //! updated inside lane selector.\n template\n d1::task* pop( const lane_selector_t& next_lane ) {\n d1::task* popped = nullptr;\n unsigned lane = 0;\n for (atomic_backoff b; !empty() && !popped; b.pause()) {\n lane = next_lane( /*out_of=*/N);\n __TBB_ASSERT(lane < N, \"Incorrect lane index.\");\n popped = try_pop(lane);\n }\n return popped;\n }\n\n //! Try finding and popping a related task.\n d1::task* pop_specific( unsigned& last_used_lane, isolation_type isolation ) {\n d1::task* result = nullptr;\n // Lane selection is round-robin in backward direction.\n unsigned idx = last_used_lane & (N-1);\n do {\n if( is_bit_set( population.load(std::memory_order_relaxed), idx ) ) {\n lane_t& lane = lanes[idx];\n mutex::scoped_lock lock;\n if( lock.try_acquire(lane.my_mutex) && !lane.my_queue.empty() ) {\n result = look_specific( lane.my_queue, isolation );\n if( lane.my_queue.empty() )\n clear_one_bit( population, idx );\n if( result )\n break;\n }\n }\n idx=(idx-1)&(N-1);\n } while( !empty() && idx != last_used_lane );\n last_used_lane = idx;\n return result;\n }\n\n //! Checks existence of a task.\n bool empty() {\n return !population.load(std::memory_order_relaxed);\n }\n\nprivate:\n //! Returns true on successful push, otherwise - false.\n bool try_push(d1::task* source, unsigned lane_idx ) {\n mutex::scoped_lock lock;\n if( lock.try_acquire( lanes[lane_idx].my_mutex ) ) {\n lanes[lane_idx].my_queue.push_back( source );\n set_one_bit( population, lane_idx ); // TODO: avoid atomic op if the bit is already set\n return true;\n }\n return false;\n }\n\n //! Returns pointer to task on successful pop, otherwise - nullptr.\n d1::task* try_pop( unsigned lane_idx ) {\n if( !is_bit_set( population.load(std::memory_order_relaxed), lane_idx ) )\n return nullptr;\n d1::task* result = nullptr;\n lane_t& lane = lanes[lane_idx];\n mutex::scoped_lock lock;\n if( lock.try_acquire( lane.my_mutex ) && !lane.my_queue.empty() ) {\n result = this->get_item( lane.my_queue );\n if( lane.my_queue.empty() )\n clear_one_bit( population, lane_idx );\n }\n return result;\n }\n\n // TODO: unify '*_specific' logic with 'pop' methods above\n d1::task* look_specific( typename lane_t::queue_base_t& queue, isolation_type isolation ) {\n __TBB_ASSERT( !queue.empty(), nullptr);\n // TODO: add a worst-case performance test and consider an alternative container with better\n // performance for isolation search.\n typename lane_t::queue_base_t::iterator curr = queue.end();\n do {\n // TODO: consider logic from get_task to simplify the code.\n d1::task* result = *--curr;\n if( result && task_accessor::isolation(*result) == isolation ) {\n if( queue.end() - curr == 1 )\n queue.pop_back(); // a little of housekeeping along the way\n else\n *curr = nullptr; // grabbing task with the same isolation\n // TODO: move one of the container's ends instead if the task has been found there\n return result;\n }\n } while( curr != queue.begin() );\n return nullptr;\n }\n\n}; // task_stream\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_task_stream_H */\n" }, { "path": "src/tbb/src/tbb/tbb.rc", "content": "// Copyright (c) 2005-2024 Intel Corporation\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Includes\n//\n#include \n#include \"../../include/oneapi/tbb/version.h\"\n\n/////////////////////////////////////////////////////////////////////////////\n// Neutral resources\n\n#ifdef _WIN32\nLANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL\n#pragma code_page(1252)\n#endif //_WIN32\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Version\n//\n#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH\n#define TBB_VERSION TBB_VERSION_STRING\n\nVS_VERSION_INFO VERSIONINFO\n FILEVERSION TBB_VERNUMBERS\n PRODUCTVERSION TBB_VERNUMBERS\n FILEFLAGSMASK 0x17L\n#ifdef _DEBUG\n FILEFLAGS 0x1L\n#else\n FILEFLAGS 0x0L\n#endif\n FILEOS 0x40004L\n FILETYPE 0x2L\n FILESUBTYPE 0x0L\nBEGIN\n BLOCK \"StringFileInfo\"\n BEGIN\n BLOCK \"000004b0\"\n BEGIN\n VALUE \"CompanyName\", \"Intel Corporation\\0\"\n VALUE \"FileDescription\", \"oneAPI Threading Building Blocks (oneTBB) library\\0\"\n VALUE \"FileVersion\", TBB_VERSION \"\\0\"\n VALUE \"LegalCopyright\", \"Copyright 2005-2024 Intel Corporation. All Rights Reserved.\\0\"\n VALUE \"LegalTrademarks\", \"\\0\"\n#ifndef TBB_USE_DEBUG\n VALUE \"OriginalFilename\", \"tbb12.dll\\0\"\n#else\n VALUE \"OriginalFilename\", \"tbb12_debug.dll\\0\"\n#endif\n VALUE \"ProductName\", \"oneAPI Threading Building Blocks (oneTBB)\\0\"\n VALUE \"ProductVersion\", TBB_VERSION \"\\0\"\n VALUE \"PrivateBuild\", \"\\0\"\n VALUE \"SpecialBuild\", \"\\0\"\n END\n END\n BLOCK \"VarFileInfo\"\n BEGIN\n VALUE \"Translation\", 0x0, 1200\n END\nEND\n" }, { "path": "src/tbb/src/tbb/tcm.h", "content": "/*\n Copyright (c) 2023-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_tcm_H\n#define _TBB_tcm_H\n\n#include \n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// Support for the TCM API return value\n\ntypedef enum _tcm_result_t {\n TCM_RESULT_SUCCESS = 0x0,\n TCM_RESULT_ERROR_INVALID_ARGUMENT = 0x78000004,\n TCM_RESULT_ERROR_UNKNOWN = 0x7ffffffe\n} tcm_result_t;\n\n// Support for permit states\n\nenum tcm_permit_states_t {\n TCM_PERMIT_STATE_VOID,\n TCM_PERMIT_STATE_INACTIVE,\n TCM_PERMIT_STATE_PENDING,\n TCM_PERMIT_STATE_IDLE,\n TCM_PERMIT_STATE_ACTIVE\n};\n\ntypedef uint8_t tcm_permit_state_t;\n\n// Support for permit flags\n\ntypedef struct _tcm_permit_flags_t {\n uint32_t stale : 1;\n uint32_t rigid_concurrency : 1;\n uint32_t exclusive : 1;\n uint32_t request_as_inactive : 1;\n uint32_t reserved : 28;\n} tcm_permit_flags_t;\n\ntypedef struct _tcm_callback_flags_t {\n uint32_t new_concurrency : 1;\n uint32_t new_state : 1;\n uint32_t reserved : 30;\n} tcm_callback_flags_t;\n\n// Support for cpu masks\n\nstruct hwloc_bitmap_s;\ntypedef struct hwloc_bitmap_s* hwloc_bitmap_t;\ntypedef hwloc_bitmap_t tcm_cpu_mask_t;\n\n// Support for ids\n\ntypedef uint64_t tcm_client_id_t;\n\n// Support for permits\n\ntypedef struct _tcm_permit_t {\n uint32_t* concurrencies;\n tcm_cpu_mask_t* cpu_masks;\n uint32_t size;\n tcm_permit_state_t state;\n tcm_permit_flags_t flags;\n} tcm_permit_t;\n\n// Support for permit handle\n\ntypedef struct tcm_permit_rep_t* tcm_permit_handle_t;\n\n// Support for constraints\n\ntypedef int32_t tcm_numa_node_t;\ntypedef int32_t tcm_core_type_t;\n\nconst int8_t tcm_automatic = -1;\nconst int8_t tcm_any = -2;\n\n#define TCM_PERMIT_REQUEST_CONSTRAINTS_INITIALIZER {tcm_automatic, tcm_automatic, NULL, \\\n tcm_automatic, tcm_automatic, tcm_automatic}\n\ntypedef struct _tcm_cpu_constraints_t {\n int32_t min_concurrency;\n int32_t max_concurrency;\n tcm_cpu_mask_t mask;\n tcm_numa_node_t numa_id;\n tcm_core_type_t core_type_id;\n int32_t threads_per_core;\n} tcm_cpu_constraints_t;\n\n// Support for priorities\n\nenum tcm_request_priorities_t {\n TCM_REQUEST_PRIORITY_LOW = (INT32_MAX / 4) * 1,\n TCM_REQUEST_PRIORITY_NORMAL = (INT32_MAX / 4) * 2,\n TCM_REQUEST_PRIORITY_HIGH = (INT32_MAX / 4) * 3\n};\n\ntypedef int32_t tcm_request_priority_t;\n\n// Support for requests\n\n#define TCM_PERMIT_REQUEST_INITIALIZER {tcm_automatic, tcm_automatic, \\\n NULL, 0, TCM_REQUEST_PRIORITY_NORMAL, {}, {}}\n\ntypedef struct _tcm_permit_request_t {\n int32_t min_sw_threads;\n int32_t max_sw_threads;\n tcm_cpu_constraints_t* cpu_constraints;\n uint32_t constraints_size;\n tcm_request_priority_t priority;\n tcm_permit_flags_t flags;\n char reserved[4];\n} tcm_permit_request_t;\n\n// Support for client callback\n\ntypedef tcm_result_t (*tcm_callback_t)(tcm_permit_handle_t p, void* callback_arg, tcm_callback_flags_t);\n\n#if _WIN32\n #define __TCM_EXPORT __declspec(dllexport)\n#else\n #define __TCM_EXPORT\n#endif\n\n\n__TCM_EXPORT tcm_result_t tcmConnect(tcm_callback_t callback,\n tcm_client_id_t *client_id);\n__TCM_EXPORT tcm_result_t tcmDisconnect(tcm_client_id_t client_id);\n\n__TCM_EXPORT tcm_result_t tcmRequestPermit(tcm_client_id_t client_id,\n tcm_permit_request_t request,\n void* callback_arg,\n tcm_permit_handle_t* permit_handle,\n tcm_permit_t* permit);\n\n__TCM_EXPORT tcm_result_t tcmGetPermitData(tcm_permit_handle_t permit_handle,\n tcm_permit_t* permit);\n\n__TCM_EXPORT tcm_result_t tcmReleasePermit(tcm_permit_handle_t permit);\n\n__TCM_EXPORT tcm_result_t tcmIdlePermit(tcm_permit_handle_t permit_handle);\n\n__TCM_EXPORT tcm_result_t tcmDeactivatePermit(tcm_permit_handle_t permit_handle);\n\n__TCM_EXPORT tcm_result_t tcmActivatePermit(tcm_permit_handle_t permit_handle);\n\n__TCM_EXPORT tcm_result_t tcmRegisterThread(tcm_permit_handle_t permit_handle);\n\n__TCM_EXPORT tcm_result_t tcmUnregisterThread();\n\n__TCM_EXPORT tcm_result_t tcmGetVersionInfo(char* buffer, uint32_t buffer_size);\n\n#ifdef __cplusplus\n} // extern \"C\"\n#endif\n\n#endif /* _TBB_tcm_H */\n" }, { "path": "src/tbb/src/tbb/tcm_adaptor.cpp", "content": "/*\n Copyright (c) 2023-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_intrusive_list_node.h\"\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n#include \"oneapi/tbb/task_arena.h\"\n\n#include \"pm_client.h\"\n#include \"dynamic_link.h\"\n#include \"misc.h\"\n#include \"tcm.h\"\n#include \"tcm_adaptor.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nnamespace {\n#if __TBB_WEAK_SYMBOLS_PRESENT\n#pragma weak tcmConnect\n#pragma weak tcmDisconnect\n#pragma weak tcmRequestPermit\n#pragma weak tcmGetPermitData\n#pragma weak tcmReleasePermit\n#pragma weak tcmIdlePermit\n#pragma weak tcmDeactivatePermit\n#pragma weak tcmActivatePermit\n#pragma weak tcmRegisterThread\n#pragma weak tcmUnregisterThread\n#pragma weak tcmGetVersionInfo\n#endif /* __TBB_WEAK_SYMBOLS_PRESENT */\n\ntcm_result_t(*tcm_connect)(tcm_callback_t callback, tcm_client_id_t* client_id){nullptr};\ntcm_result_t(*tcm_disconnect)(tcm_client_id_t client_id){ nullptr };\ntcm_result_t(*tcm_request_permit)(tcm_client_id_t client_id, tcm_permit_request_t request,\n void* callback_arg, tcm_permit_handle_t* permit_handle, tcm_permit_t* permit){nullptr};\ntcm_result_t(*tcm_get_permit_data)(tcm_permit_handle_t permit_handle, tcm_permit_t* permit){nullptr};\ntcm_result_t(*tcm_release_permit)(tcm_permit_handle_t permit){nullptr};\ntcm_result_t(*tcm_idle_permit)(tcm_permit_handle_t permit_handle){nullptr};\ntcm_result_t(*tcm_deactivate_permit)(tcm_permit_handle_t permit_handle){nullptr};\ntcm_result_t(*tcm_activate_permit)(tcm_permit_handle_t permit_handle){nullptr};\ntcm_result_t(*tcm_register_thread)(tcm_permit_handle_t permit_handle){nullptr};\ntcm_result_t(*tcm_unregister_thread)(){nullptr};\ntcm_result_t (*tcm_get_version_info)(char* buffer, uint32_t buffer_size){nullptr};\n\nstatic const dynamic_link_descriptor tcm_link_table[] = {\n DLD(tcmConnect, tcm_connect),\n DLD(tcmDisconnect, tcm_disconnect),\n DLD(tcmRequestPermit, tcm_request_permit),\n DLD(tcmGetPermitData, tcm_get_permit_data),\n DLD(tcmReleasePermit, tcm_release_permit),\n DLD(tcmIdlePermit, tcm_idle_permit),\n DLD(tcmDeactivatePermit, tcm_deactivate_permit),\n DLD(tcmActivatePermit, tcm_activate_permit),\n DLD(tcmRegisterThread, tcm_register_thread),\n DLD(tcmUnregisterThread, tcm_unregister_thread),\n DLD(tcmGetVersionInfo, tcm_get_version_info)\n};\n\n#if TBB_USE_DEBUG\n#define DEBUG_SUFFIX \"_debug\"\n#else\n#define DEBUG_SUFFIX\n#endif /* TBB_USE_DEBUG */\n\n#if _WIN32 || _WIN64\n#define LIBRARY_EXTENSION \".dll\"\n#define LIBRARY_PREFIX\n#elif __unix__\n#define LIBRARY_EXTENSION \".so.1\"\n#define LIBRARY_PREFIX \"lib\"\n#else\n#define LIBRARY_EXTENSION\n#define LIBRARY_PREFIX\n#endif /* __unix__ */\n\n#define TCMLIB_NAME LIBRARY_PREFIX \"tcm\" DEBUG_SUFFIX LIBRARY_EXTENSION\n\nstatic bool tcm_functions_loaded{ false };\n}\n\nclass tcm_client : public pm_client {\n using tcm_client_mutex_type = d1::mutex;\npublic:\n tcm_client(tcm_adaptor& adaptor, arena& a) : pm_client(a), my_tcm_adaptor(adaptor) {}\n\n ~tcm_client() {\n if (my_permit_handle) {\n __TBB_ASSERT(tcm_release_permit, nullptr);\n auto res = tcm_release_permit(my_permit_handle);\n __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr);\n }\n }\n\n int update_concurrency(uint32_t concurrency) {\n return my_arena.update_concurrency(concurrency);\n }\n\n unsigned priority_level() {\n return my_arena.priority_level();\n }\n\n tcm_permit_request_t& permit_request() {\n return my_permit_request;\n }\n\n tcm_permit_handle_t& permit_handle() {\n return my_permit_handle;\n }\n\n void actualize_permit() {\n __TBB_ASSERT(tcm_get_permit_data, nullptr);\n int delta{};\n {\n tcm_client_mutex_type::scoped_lock lock(my_permit_mutex);\n\n uint32_t new_concurrency{};\n tcm_permit_t new_permit{ &new_concurrency, nullptr, 1, TCM_PERMIT_STATE_VOID, {} };\n auto res = tcm_get_permit_data(my_permit_handle, &new_permit);\n __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr);\n\n // The permit has changed during the reading, so the callback will be invoked soon one more time and\n // we can just skip this renegotiation iteration.\n if (!new_permit.flags.stale) {\n // If there is no other demand in TCM, the permit may still have granted concurrency but\n // be in the deactivated state thus we enforce 0 allotment to preserve arena invariants.\n delta = update_concurrency(new_permit.state != TCM_PERMIT_STATE_INACTIVE ? new_concurrency : 0);\n }\n }\n if (delta) {\n my_tcm_adaptor.notify_thread_request(delta);\n }\n }\n\n void request_permit(tcm_client_id_t client_id) {\n __TBB_ASSERT(tcm_request_permit, nullptr);\n\n my_permit_request.max_sw_threads = max_workers();\n my_permit_request.min_sw_threads = my_permit_request.max_sw_threads == 0 ? 0 : min_workers();\n\n if (my_permit_request.constraints_size > 0) {\n my_permit_request.cpu_constraints->min_concurrency = my_permit_request.min_sw_threads;\n my_permit_request.cpu_constraints->max_concurrency = my_permit_request.max_sw_threads;\n }\n\n __TBB_ASSERT(my_permit_request.max_sw_threads >= my_permit_request.min_sw_threads, nullptr);\n\n tcm_result_t res = tcm_request_permit(client_id, my_permit_request, this, &my_permit_handle, nullptr);\n __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr);\n }\n\n void deactivate_permit() {\n __TBB_ASSERT(tcm_deactivate_permit, nullptr);\n tcm_result_t res = tcm_deactivate_permit(my_permit_handle);\n __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr);\n }\n\n void init(tcm_client_id_t client_id, d1::constraints& constraints) {\n __TBB_ASSERT(tcm_request_permit, nullptr);\n __TBB_ASSERT(tcm_deactivate_permit, nullptr);\n\n if (constraints.core_type != d1::task_arena::automatic ||\n constraints.numa_id != d1::task_arena::automatic ||\n constraints.max_threads_per_core != d1::task_arena::automatic)\n {\n my_permit_constraints.max_concurrency = constraints.max_concurrency;\n my_permit_constraints.min_concurrency = 0;\n my_permit_constraints.core_type_id = constraints.core_type;\n my_permit_constraints.numa_id = constraints.numa_id;\n my_permit_constraints.threads_per_core = constraints.max_threads_per_core;\n\n my_permit_request.cpu_constraints = &my_permit_constraints;\n my_permit_request.constraints_size = 1;\n }\n\n my_permit_request.min_sw_threads = 0;\n my_permit_request.max_sw_threads = 0;\n my_permit_request.flags.request_as_inactive = 1;\n\n tcm_result_t res = tcm_request_permit(client_id, my_permit_request, this, &my_permit_handle, nullptr);\n __TBB_ASSERT_EX(res == TCM_RESULT_SUCCESS, nullptr);\n\n my_permit_request.flags.request_as_inactive = 0;\n }\n\n void register_thread() override {\n __TBB_ASSERT(tcm_register_thread, nullptr);\n auto return_code = tcm_register_thread(my_permit_handle);\n __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr);\n }\n\n void unregister_thread() override {\n __TBB_ASSERT(tcm_unregister_thread, nullptr);\n auto return_code = tcm_unregister_thread();\n __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr);\n }\n\nprivate:\n tcm_cpu_constraints_t my_permit_constraints = TCM_PERMIT_REQUEST_CONSTRAINTS_INITIALIZER;\n tcm_permit_request_t my_permit_request = TCM_PERMIT_REQUEST_INITIALIZER;\n tcm_permit_handle_t my_permit_handle{};\n tcm_client_mutex_type my_permit_mutex;\n tcm_adaptor& my_tcm_adaptor;\n};\n\n//------------------------------------------------------------------------\n// tcm_adaptor_impl\n//------------------------------------------------------------------------\n\nstruct tcm_adaptor_impl {\n using demand_mutex_type = d1::mutex;\n demand_mutex_type my_demand_mutex;\n tcm_client_id_t client_id{};\n\n tcm_adaptor_impl(tcm_client_id_t id) : client_id(id)\n {}\n};\n\n//------------------------------------------------------------------------\n// tcm_adaptor\n//------------------------------------------------------------------------\n\ntcm_result_t renegotiation_callback(tcm_permit_handle_t, void* client_ptr, tcm_callback_flags_t) {\n __TBB_ASSERT(client_ptr, nullptr);\n static_cast(client_ptr)->actualize_permit();\n return TCM_RESULT_SUCCESS;\n}\n\nvoid tcm_adaptor::initialize() {\n tcm_functions_loaded = dynamic_link(TCMLIB_NAME, tcm_link_table, /* tcm_link_table size = */ 11);\n}\n\nbool tcm_adaptor::is_initialized() {\n return tcm_functions_loaded;\n}\n\nvoid tcm_adaptor::print_version() {\n if (is_initialized()) {\n __TBB_ASSERT(tcm_get_version_info, nullptr);\n char buffer[1024];\n tcm_get_version_info(buffer, 1024);\n std::fprintf(stderr, \"%.*s\", 1024, buffer);\n }\n}\n\ntcm_adaptor::tcm_adaptor() {\n __TBB_ASSERT(tcm_connect, nullptr);\n tcm_client_id_t client_id{};\n auto return_code = tcm_connect(renegotiation_callback, &client_id);\n if (return_code == TCM_RESULT_SUCCESS) {\n my_impl = make_cache_aligned_unique(client_id);\n }\n}\n\ntcm_adaptor::~tcm_adaptor() {\n if (my_impl) {\n __TBB_ASSERT(tcm_disconnect, nullptr);\n auto return_code = tcm_disconnect(my_impl->client_id);\n __TBB_ASSERT_EX(return_code == TCM_RESULT_SUCCESS, nullptr);\n my_impl = nullptr;\n }\n}\n\nbool tcm_adaptor::is_connected() {\n return my_impl != nullptr;\n}\n\npm_client* tcm_adaptor::create_client(arena& a) {\n return new (cache_aligned_allocate(sizeof(tcm_client))) tcm_client(*this, a);\n}\n\nvoid tcm_adaptor::register_client(pm_client* c, d1::constraints& constraints) {\n static_cast(c)->init(my_impl->client_id, constraints);\n}\n\nvoid tcm_adaptor::unregister_and_destroy_client(pm_client& c) {\n auto& client = static_cast(c);\n\n {\n tcm_adaptor_impl::demand_mutex_type::scoped_lock lock(my_impl->my_demand_mutex);\n client.~tcm_client();\n }\n cache_aligned_deallocate(&client);\n}\n\nvoid tcm_adaptor::set_active_num_workers(int) {}\n\n\nvoid tcm_adaptor::adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) {\n __TBB_ASSERT(-1 <= mandatory_delta && mandatory_delta <= 1, nullptr);\n\n auto& client = static_cast(c);\n {\n tcm_adaptor_impl::demand_mutex_type::scoped_lock lock(my_impl->my_demand_mutex);\n\n // Update client's state\n workers_delta = client.update_request(mandatory_delta, workers_delta);\n if (workers_delta == 0) return;\n\n if (client.max_workers() == 0) {\n client.deactivate_permit();\n } else {\n client.request_permit(my_impl->client_id);\n }\n }\n\n client.actualize_permit();\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/tcm_adaptor.h", "content": "/*\n Copyright (c) 2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_tcm_adaptor_H\n#define _TBB_tcm_adaptor_H\n\n#include \"scheduler_common.h\"\n\n#include \"permit_manager.h\"\n#include \"pm_client.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct tcm_adaptor_impl;\n\n//------------------------------------------------------------------------\n// Class tcm_adaptor\n//------------------------------------------------------------------------\n\nclass tcm_adaptor : public permit_manager {\npublic:\n tcm_adaptor();\n ~tcm_adaptor();\n\n pm_client* create_client(arena& a) override;\n void register_client(pm_client* client, d1::constraints& constraints) override;\n void unregister_and_destroy_client(pm_client& c) override;\n\n void set_active_num_workers(int soft_limit) override;\n\n void adjust_demand(pm_client& c, int mandatory_delta, int workers_delta) override;\n\n bool is_connected();\n\n static void initialize();\n static bool is_initialized();\n static void print_version();\nprivate:\n cache_aligned_unique_ptr my_impl;\n\n friend class tcm_client;\n}; // class tcm_adaptor\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_tcm_adaptor_H */\n" }, { "path": "src/tbb/src/tbb/thread_control_monitor.h", "content": "/*\n Copyright (c) 2021-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_thread_control_monitor_H\n#define __TBB_thread_control_monitor_H\n\n#include \"concurrent_monitor.h\"\n#include \"scheduler_common.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nstruct market_context {\n market_context() = default;\n\n market_context(std::uintptr_t first_addr, arena* a) :\n my_uniq_addr(first_addr), my_arena_addr(a)\n {}\n\n std::uintptr_t my_uniq_addr{0};\n arena* my_arena_addr{nullptr};\n};\n\n#if __TBB_RESUMABLE_TASKS\nclass resume_node : public wait_node {\n using base_type = wait_node;\npublic:\n resume_node(market_context ctx, execution_data_ext& ed_ext, task_dispatcher& target)\n : base_type(ctx), my_curr_dispatcher(ed_ext.task_disp), my_target_dispatcher(&target)\n , my_suspend_point(my_curr_dispatcher->get_suspend_point())\n {}\n\n ~resume_node() override {\n if (this->my_skipped_wakeup) {\n spin_wait_until_eq(this->my_notify_calls, 1);\n }\n\n poison_pointer(my_curr_dispatcher);\n poison_pointer(my_target_dispatcher);\n poison_pointer(my_suspend_point);\n }\n\n void init() override {\n base_type::init();\n }\n\n void wait() override {\n my_curr_dispatcher->resume(*my_target_dispatcher);\n __TBB_ASSERT(!this->my_is_in_list.load(std::memory_order_relaxed), \"Still in the queue?\");\n }\n\n void reset() override {\n base_type::reset();\n spin_wait_until_eq(this->my_notify_calls, 1);\n my_notify_calls.store(0, std::memory_order_relaxed);\n }\n\n // notify is called (perhaps, concurrently) twice from:\n // - concurrent_monitor::notify\n // - post_resume_action::register_waiter\n // The second notify is called after thread switches the stack\n // (Because we can not call resume while the stack is occupied)\n // We need calling resume only when both notifications are performed.\n void notify() override {\n if (++my_notify_calls == 2) {\n r1::resume(my_suspend_point);\n }\n }\n\nprivate:\n friend class thread_data;\n friend struct suspend_point_type::resume_task;\n task_dispatcher* my_curr_dispatcher;\n task_dispatcher* my_target_dispatcher;\n suspend_point_type* my_suspend_point;\n std::atomic my_notify_calls{0};\n};\n#endif // __TBB_RESUMABLE_TASKS\n\nclass thread_control_monitor : public concurrent_monitor_base {\n using base_type = concurrent_monitor_base;\npublic:\n using base_type::base_type;\n\n ~thread_control_monitor() {\n destroy();\n }\n\n /** per-thread descriptor for concurrent_monitor */\n using thread_context = sleep_node;\n#if __TBB_RESUMABLE_TASKS\n using resume_context = resume_node;\n#endif\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_thread_control_monitor_H\n" }, { "path": "src/tbb/src/tbb/thread_data.h", "content": "/*\n Copyright (c) 2020-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_thread_data_H\n#define __TBB_thread_data_H\n\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"oneapi/tbb/task.h\"\n\n#include \"rml_base.h\" // rml::job\n\n#include \"scheduler_common.h\"\n#include \"arena.h\"\n#include \"concurrent_monitor.h\"\n#include \"mailbox.h\"\n#include \"misc.h\" // FastRandom\n#include \"small_object_pool_impl.h\"\n#include \"intrusive_list.h\"\n\n#include \n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass task;\nclass arena_slot;\nclass task_group_context;\nclass task_dispatcher;\nclass thread_dispatcher_client;\n\nclass context_list : public intrusive_list {\npublic:\n bool orphaned{false};\n\n //! Last state propagation epoch known to this thread\n /** Together with the_context_state_propagation_epoch constitute synchronization protocol\n that keeps hot path of task group context construction destruction mostly\n lock-free.\n When local epoch equals the global one, the state of task group contexts\n registered with this thread is consistent with that of the task group trees\n they belong to. **/\n std::atomic epoch{};\n\n //! Mutex protecting access to the list of task group contexts.\n d1::mutex m_mutex{};\n\n void destroy() {\n this->~context_list();\n cache_aligned_deallocate(this);\n }\n\n void remove(d1::intrusive_list_node& val) {\n mutex::scoped_lock lock(m_mutex);\n\n intrusive_list::remove(val);\n\n if (orphaned && empty()) {\n lock.release();\n destroy();\n }\n }\n\n void push_front(d1::intrusive_list_node& val) {\n mutex::scoped_lock lock(m_mutex);\n\n intrusive_list::push_front(val);\n }\n\n void orphan() {\n mutex::scoped_lock lock(m_mutex);\n\n orphaned = true;\n if (empty()) {\n lock.release();\n destroy();\n }\n }\n};\n\n//------------------------------------------------------------------------\n// Thread Data\n//------------------------------------------------------------------------\nclass thread_data : public ::rml::job\n , public d1::intrusive_list_node\n , no_copy {\npublic:\n thread_data(unsigned short index, bool is_worker)\n : my_arena_index{ index }\n , my_is_worker{ is_worker }\n , my_is_registered { false }\n , my_task_dispatcher{ nullptr }\n , my_arena{ nullptr }\n , my_last_client{ nullptr }\n , my_arena_slot{}\n , my_random{ this }\n , my_last_observer{ nullptr }\n , my_small_object_pool{new (cache_aligned_allocate(sizeof(small_object_pool_impl))) small_object_pool_impl{}}\n , my_context_list(new (cache_aligned_allocate(sizeof(context_list))) context_list{})\n#if __TBB_RESUMABLE_TASKS\n , my_post_resume_action{ task_dispatcher::post_resume_action::none }\n , my_post_resume_arg{nullptr}\n#endif /* __TBB_RESUMABLE_TASKS */\n {\n ITT_SYNC_CREATE(&my_context_list->m_mutex, SyncType_Scheduler, SyncObj_ContextsList);\n }\n\n ~thread_data() {\n my_context_list->orphan();\n my_small_object_pool->destroy();\n poison_pointer(my_task_dispatcher);\n poison_pointer(my_arena);\n poison_pointer(my_arena_slot);\n poison_pointer(my_last_observer);\n poison_pointer(my_small_object_pool);\n poison_pointer(my_context_list);\n#if __TBB_RESUMABLE_TASKS\n poison_pointer(my_post_resume_arg);\n#endif /* __TBB_RESUMABLE_TASKS */\n }\n\n void attach_arena(arena& a, std::size_t index);\n bool is_attached_to(arena*);\n void attach_task_dispatcher(task_dispatcher&);\n void detach_task_dispatcher();\n void enter_task_dispatcher(task_dispatcher& task_disp, std::uintptr_t stealing_threshold);\n void leave_task_dispatcher();\n void propagate_task_group_state(std::atomic d1::task_group_context::* mptr_state, d1::task_group_context& src, uint32_t new_state);\n\n //! Index of the arena slot the scheduler occupies now, or occupied last time\n unsigned short my_arena_index;\n\n //! Indicates if the thread is created by RML\n const bool my_is_worker;\n\n bool my_is_registered;\n\n //! The current task dipsatcher\n task_dispatcher* my_task_dispatcher;\n\n //! The arena that I own (if external thread) or am servicing at the moment (if worker)\n arena* my_arena;\n\n thread_dispatcher_client* my_last_client;\n\n //! Pointer to the slot in the arena we own at the moment\n arena_slot* my_arena_slot;\n\n //! The mailbox (affinity mechanism) the current thread attached to\n mail_inbox my_inbox;\n\n //! The random generator\n FastRandom my_random;\n\n //! Last observer in the observers list processed on this slot\n observer_proxy* my_last_observer;\n\n //! Pool of small object for fast task allocation\n small_object_pool_impl* my_small_object_pool;\n\n context_list* my_context_list;\n#if __TBB_RESUMABLE_TASKS\n //! Suspends the current coroutine (task_dispatcher).\n void suspend(void* suspend_callback, void* user_callback);\n\n //! Resumes the target task_dispatcher.\n void resume(task_dispatcher& target);\n\n //! Set post resume action to perform after resume.\n void set_post_resume_action(task_dispatcher::post_resume_action pra, void* arg) {\n __TBB_ASSERT(my_post_resume_action == task_dispatcher::post_resume_action::none, \"The Post resume action must not be set\");\n __TBB_ASSERT(!my_post_resume_arg, \"The post resume action must not have an argument\");\n my_post_resume_action = pra;\n my_post_resume_arg = arg;\n }\n\n void clear_post_resume_action() {\n my_post_resume_action = task_dispatcher::post_resume_action::none;\n my_post_resume_arg = nullptr;\n }\n\n //! The post resume action requested after the swap contexts.\n task_dispatcher::post_resume_action my_post_resume_action;\n\n //! The post resume action argument.\n void* my_post_resume_arg;\n#endif /* __TBB_RESUMABLE_TASKS */\n\n //! The default context\n // TODO: consider using common default context because it is used only to simplify\n // cancellation check.\n d1::task_group_context my_default_context;\n};\n\ninline void thread_data::attach_arena(arena& a, std::size_t index) {\n my_arena = &a;\n my_arena_index = static_cast(index);\n my_arena_slot = a.my_slots + index;\n // Read the current slot mail_outbox and attach it to the mail_inbox (remove inbox later maybe)\n my_inbox.attach(my_arena->mailbox(index));\n}\n\ninline bool thread_data::is_attached_to(arena* a) { return my_arena == a; }\n\ninline void thread_data::attach_task_dispatcher(task_dispatcher& task_disp) {\n __TBB_ASSERT(my_task_dispatcher == nullptr, nullptr);\n __TBB_ASSERT(task_disp.m_thread_data == nullptr, nullptr);\n task_disp.m_thread_data = this;\n my_task_dispatcher = &task_disp;\n}\n\ninline void thread_data::detach_task_dispatcher() {\n __TBB_ASSERT(my_task_dispatcher != nullptr, nullptr);\n __TBB_ASSERT(my_task_dispatcher->m_thread_data == this, nullptr);\n my_task_dispatcher->m_thread_data = nullptr;\n my_task_dispatcher = nullptr;\n}\n\ninline void thread_data::enter_task_dispatcher(task_dispatcher& task_disp, std::uintptr_t stealing_threshold) {\n task_disp.set_stealing_threshold(stealing_threshold);\n attach_task_dispatcher(task_disp);\n}\n\ninline void thread_data::leave_task_dispatcher() {\n my_task_dispatcher->set_stealing_threshold(0);\n detach_task_dispatcher();\n}\n\ninline void thread_data::propagate_task_group_state(std::atomic d1::task_group_context::* mptr_state, d1::task_group_context& src, std::uint32_t new_state) {\n mutex::scoped_lock lock(my_context_list->m_mutex);\n // Acquire fence is necessary to ensure that the subsequent node->my_next load\n // returned the correct value in case it was just inserted in another thread.\n // The fence also ensures visibility of the correct ctx.my_parent value.\n for (context_list::iterator it = my_context_list->begin(); it != my_context_list->end(); ++it) {\n d1::task_group_context& ctx = __TBB_get_object_ref(d1::task_group_context, my_node, &(*it));\n if ((ctx.*mptr_state).load(std::memory_order_relaxed) != new_state)\n task_group_context_impl::propagate_task_group_state(ctx, mptr_state, src, new_state);\n }\n // Sync up local propagation epoch with the global one. Release fence prevents\n // reordering of possible store to *mptr_state after the sync point.\n my_context_list->epoch.store(the_context_state_propagation_epoch.load(std::memory_order_relaxed), std::memory_order_release);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // __TBB_thread_data_H\n\n" }, { "path": "src/tbb/src/tbb/thread_dispatcher.cpp", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"thread_dispatcher.h\"\n#include \"threading_control.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nthread_dispatcher::thread_dispatcher(threading_control& tc, unsigned hard_limit, std::size_t stack_size)\n : my_threading_control(tc)\n , my_num_workers_hard_limit(hard_limit)\n , my_stack_size(stack_size)\n{\n my_server = governor::create_rml_server( *this );\n __TBB_ASSERT( my_server, \"Failed to create RML server\" );\n}\n\nthread_dispatcher::~thread_dispatcher() {\n poison_pointer(my_server);\n}\n\nthread_dispatcher_client* thread_dispatcher::select_next_client(thread_dispatcher_client* hint) {\n unsigned next_client_priority_level = num_priority_levels;\n if (hint) {\n next_client_priority_level = hint->priority_level();\n }\n\n for (unsigned idx = 0; idx < next_client_priority_level; ++idx) {\n if (!my_client_list[idx].empty()) {\n return &*my_client_list[idx].begin();\n }\n }\n\n return hint;\n}\n\nthread_dispatcher_client* thread_dispatcher::create_client(arena& a) {\n return new (cache_aligned_allocate(sizeof(thread_dispatcher_client))) thread_dispatcher_client(a, my_clients_aba_epoch);\n}\n\n\nvoid thread_dispatcher::register_client(thread_dispatcher_client* client) {\n client_list_mutex_type::scoped_lock lock(my_list_mutex);\n insert_client(*client);\n}\n\nbool thread_dispatcher::try_unregister_client(thread_dispatcher_client* client, std::uint64_t aba_epoch, unsigned priority) {\n __TBB_ASSERT(client, nullptr);\n // we hold reference to the server, so market cannot be destroyed at any moment here\n __TBB_ASSERT(!is_poisoned(my_server), nullptr);\n my_list_mutex.lock();\n for (auto& it : my_client_list[priority]) {\n if (client == &it) {\n if (it.get_aba_epoch() == aba_epoch) {\n // Client is alive\n // Acquire my_references to sync with threads that just left the arena\n // Pay attention that references should be read before workers_requested because\n // if references is no zero some other thread might call adjust_demand and lead to\n // a race over workers_requested\n if (!client->references() && !client->has_request()) {\n // Client is abandoned. Destroy it.\n remove_client(*client);\n ++my_clients_aba_epoch;\n\n my_list_mutex.unlock();\n destroy_client(client);\n\n return true;\n }\n }\n break;\n }\n }\n my_list_mutex.unlock();\n return false;\n}\n\nvoid thread_dispatcher::destroy_client(thread_dispatcher_client* client) {\n client->~thread_dispatcher_client();\n cache_aligned_deallocate(client);\n}\n\n// Should be called under lock\nvoid thread_dispatcher::insert_client(thread_dispatcher_client& client) {\n __TBB_ASSERT(client.priority_level() < num_priority_levels, nullptr);\n my_client_list[client.priority_level()].push_front(client);\n\n __TBB_ASSERT(!my_next_client || my_next_client->priority_level() < num_priority_levels, nullptr);\n my_next_client = select_next_client(my_next_client);\n}\n\n// Should be called under lock\nvoid thread_dispatcher::remove_client(thread_dispatcher_client& client) {\n __TBB_ASSERT(client.priority_level() < num_priority_levels, nullptr);\n my_client_list[client.priority_level()].remove(client);\n\n if (my_next_client == &client) {\n my_next_client = nullptr;\n }\n my_next_client = select_next_client(my_next_client);\n}\n\nbool thread_dispatcher::is_client_alive(thread_dispatcher_client* client) {\n if (!client) {\n return false;\n }\n\n // Still cannot access internals of the client since the object itself might be destroyed.\n for (auto& priority_list : my_client_list) {\n for (auto& c : priority_list) {\n if (client == &c) {\n return true;\n }\n }\n }\n return false;\n}\n\nthread_dispatcher_client* thread_dispatcher::client_in_need(client_list_type* clients, thread_dispatcher_client* hint) {\n // TODO: make sure client with higher priority returned only if there are available slots in it.\n hint = select_next_client(hint);\n if (!hint) {\n return nullptr;\n }\n\n client_list_type::iterator it = hint;\n unsigned curr_priority_level = hint->priority_level();\n __TBB_ASSERT(it != clients[curr_priority_level].end(), nullptr);\n do {\n thread_dispatcher_client& t = *it;\n if (++it == clients[curr_priority_level].end()) {\n do {\n ++curr_priority_level %= num_priority_levels;\n } while (clients[curr_priority_level].empty());\n it = clients[curr_priority_level].begin();\n }\n if (t.try_join()) {\n return &t;\n }\n } while (it != hint);\n return nullptr;\n}\n\nthread_dispatcher_client* thread_dispatcher::client_in_need(thread_dispatcher_client* prev) {\n client_list_mutex_type::scoped_lock lock(my_list_mutex, /*is_writer=*/false);\n if (is_client_alive(prev)) {\n return client_in_need(my_client_list, prev);\n }\n return client_in_need(my_client_list, my_next_client);\n}\n\nbool thread_dispatcher::is_any_client_in_need() {\n client_list_mutex_type::scoped_lock lock(my_list_mutex, /*is_writer=*/false);\n for (auto& priority_list : my_client_list) {\n for (auto& client : priority_list) {\n if (client.is_joinable()) {\n return true;\n }\n }\n }\n return false;\n}\n\nvoid thread_dispatcher::adjust_job_count_estimate(int delta) {\n my_server->adjust_job_count_estimate(delta);\n}\n\nvoid thread_dispatcher::release(bool blocking_terminate) {\n my_join_workers = blocking_terminate;\n my_server->request_close_connection();\n}\n\nvoid thread_dispatcher::process(job& j) {\n thread_data& td = static_cast(j);\n // td.my_last_client can be dead. Don't access it until client_in_need is called\n thread_dispatcher_client* client = td.my_last_client;\n for (int i = 0; i < 2; ++i) {\n while ((client = client_in_need(client)) ) {\n td.my_last_client = client;\n client->process(td);\n }\n // Workers leave thread_dispatcher because there is no client in need. It can happen earlier than\n // adjust_job_count_estimate() decreases my_slack and RML can put this thread to sleep.\n // It might result in a busy-loop checking for my_slack<0 and calling this method instantly.\n // the yield refines this spinning.\n if ( !i ) {\n yield();\n }\n }\n}\n\n\n//! Used when RML asks for join mode during workers termination.\nbool thread_dispatcher::must_join_workers() const { return my_join_workers; }\n\n//! Returns the requested stack size of worker threads.\nstd::size_t thread_dispatcher::worker_stack_size() const { return my_stack_size; }\n\nvoid thread_dispatcher::acknowledge_close_connection() {\n my_threading_control.destroy();\n}\n\n::rml::job* thread_dispatcher::create_one_job() {\n unsigned short index = ++my_first_unused_worker_idx;\n __TBB_ASSERT(index > 0, nullptr);\n ITT_THREAD_SET_NAME(_T(\"TBB Worker Thread\"));\n // index serves as a hint decreasing conflicts between workers when they migrate between arenas\n thread_data* td = new (cache_aligned_allocate(sizeof(thread_data))) thread_data{ index, true };\n __TBB_ASSERT(index <= my_num_workers_hard_limit, nullptr);\n my_threading_control.register_thread(*td);\n return td;\n}\n\nvoid thread_dispatcher::cleanup(job& j) {\n my_threading_control.unregister_thread(static_cast(j));\n governor::auto_terminate(&j);\n}\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n" }, { "path": "src/tbb/src/tbb/thread_dispatcher.h", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_thread_dispatcher_H\n#define _TBB_thread_dispatcher_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/rw_mutex.h\"\n#include \"oneapi/tbb/task_arena.h\"\n\n#include \"arena.h\"\n#include \"governor.h\"\n#include \"thread_data.h\"\n#include \"rml_tbb.h\"\n#include \"thread_dispatcher_client.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass threading_control_impl;\n\nclass thread_dispatcher : no_copy, rml::tbb_client {\n using client_list_type = intrusive_list;\n using client_list_mutex_type = d1::rw_mutex;\npublic:\n thread_dispatcher(threading_control& tc, unsigned hard_limit, std::size_t stack_size);\n ~thread_dispatcher();\n\n thread_dispatcher_client* create_client(arena& a);\n void register_client(thread_dispatcher_client* client);\n bool try_unregister_client(thread_dispatcher_client* client, std::uint64_t aba_epoch, unsigned priority);\n bool is_any_client_in_need();\n\n void adjust_job_count_estimate(int delta);\n void release(bool blocking_terminate);\n void process(job& j) override;\n //! Used when RML asks for join mode during workers termination.\n bool must_join_workers() const;\n //! Returns the requested stack size of worker threads.\n std::size_t worker_stack_size() const;\n\nprivate:\n version_type version () const override { return 0; }\n unsigned max_job_count () const override { return my_num_workers_hard_limit; }\n std::size_t min_stack_size () const override { return worker_stack_size(); }\n void cleanup(job& j) override;\n void acknowledge_close_connection() override;\n ::rml::job* create_one_job() override;\n\n thread_dispatcher_client* select_next_client(thread_dispatcher_client* hint);\n void destroy_client(thread_dispatcher_client* client);\n void insert_client(thread_dispatcher_client& client);\n void remove_client(thread_dispatcher_client& client);\n bool is_client_alive(thread_dispatcher_client* client);\n thread_dispatcher_client* client_in_need(client_list_type* clients, thread_dispatcher_client* hint);\n thread_dispatcher_client* client_in_need(thread_dispatcher_client* prev);\n\n friend class threading_control_impl;\n static constexpr unsigned num_priority_levels = d1::num_priority_levels;\n client_list_mutex_type my_list_mutex;\n client_list_type my_client_list[num_priority_levels];\n\n thread_dispatcher_client* my_next_client{nullptr};\n\n //! Shutdown mode\n bool my_join_workers{false};\n\n threading_control& my_threading_control;\n\n //! ABA prevention marker to assign to newly created clients\n std::atomic my_clients_aba_epoch{0};\n\n //! Maximal number of workers allowed for use by the underlying resource manager\n /** It can't be changed after thread_dispatcher creation. **/\n unsigned my_num_workers_hard_limit{0};\n\n //! Stack size of worker threads\n std::size_t my_stack_size{0};\n\n //! First unused index of worker\n /** Used to assign indices to the new workers coming from RML **/\n std::atomic my_first_unused_worker_idx{0};\n\n //! Pointer to the RML server object that services this TBB instance.\n rml::tbb_server* my_server{nullptr};\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_thread_dispatcher_H\n" }, { "path": "src/tbb/src/tbb/thread_dispatcher_client.h", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_thread_dispatcher_client_H\n#define _TBB_thread_dispatcher_client_H\n\n#include \"oneapi/tbb/detail/_intrusive_list_node.h\"\n#include \"arena.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass thread_dispatcher_client : public d1::intrusive_list_node /* Need for list in thread pool */ {\npublic:\n thread_dispatcher_client(arena& a, std::uint64_t aba_epoch) : my_arena(a), my_aba_epoch(aba_epoch) {}\n\n // Interface of communication with thread pool\n bool try_join() {\n return my_arena.try_join();\n }\n\n bool is_joinable() {\n return my_arena.is_joinable();\n }\n\n void process(thread_data& td) {\n my_arena.process(td);\n }\n\n unsigned priority_level() {\n return my_arena.priority_level();\n }\n\n std::uint64_t get_aba_epoch() {\n return my_aba_epoch;\n }\n\n unsigned references() {\n return my_arena.references();\n }\n\n bool has_request() {\n return my_arena.has_request();\n }\n\nprivate:\n arena& my_arena;\n std::uint64_t my_aba_epoch;\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_thread_dispatcher_client_H\n" }, { "path": "src/tbb/src/tbb/thread_request_serializer.cpp", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"misc.h\"\n#include \"thread_request_serializer.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nthread_request_serializer::thread_request_serializer(thread_dispatcher& td, int soft_limit)\n : my_thread_dispatcher(td)\n , my_soft_limit(soft_limit)\n{}\n\nvoid thread_request_serializer::update(int delta) {\n constexpr std::uint64_t delta_mask = (pending_delta_base << 1) - 1;\n constexpr std::uint64_t counter_value = delta_mask + 1;\n\n int prev_pending_delta = my_pending_delta.fetch_add(counter_value + delta);\n\n // There is a pseudo request aggregator, so only thread that see pending_delta_base in my_pending_delta\n // Will enter to critical section and call adjust_job_count_estimate\n if (prev_pending_delta == pending_delta_base) {\n delta = int(my_pending_delta.exchange(pending_delta_base) & delta_mask) - int(pending_delta_base);\n mutex_type::scoped_lock lock(my_mutex);\n my_total_request.store(my_total_request.load(std::memory_order_relaxed) + delta, std::memory_order_relaxed);\n delta = limit_delta(delta, my_soft_limit, my_total_request.load(std::memory_order_relaxed));\n my_thread_dispatcher.adjust_job_count_estimate(delta);\n }\n}\n\nvoid thread_request_serializer::set_active_num_workers(int soft_limit) {\n mutex_type::scoped_lock lock(my_mutex);\n int delta = soft_limit - my_soft_limit;\n delta = limit_delta(delta, my_total_request.load(std::memory_order_relaxed), soft_limit);\n my_thread_dispatcher.adjust_job_count_estimate(delta);\n my_soft_limit = soft_limit;\n}\n\nint thread_request_serializer::limit_delta(int delta, int limit, int new_value) {\n // This method can be described with such pseudocode:\n // bool above_limit = prev_value >= limit && new_value >= limit;\n // bool below_limit = prev_value <= limit && new_value <= limit;\n // enum request_type { ABOVE_LIMIT, CROSS_LIMIT, BELOW_LIMIT };\n // request = above_limit ? ABOVE_LIMIT : below_limit ? BELOW_LIMIT : CROSS_LIMIT;\n\n // switch (request) {\n // case ABOVE_LIMIT:\n // delta = 0;\n // case CROSS_LIMIT:\n // delta = delta > 0 ? limit - prev_value : new_value - limit;\n // case BELOW_LIMIT:\n // // No changes to delta\n // }\n\n int prev_value = new_value - delta;\n\n // actual new_value and prev_value cannot exceed the limit\n new_value = min(limit, new_value);\n prev_value = min(limit, prev_value);\n return new_value - prev_value;\n}\n\n\nthread_request_serializer_proxy::thread_request_serializer_proxy(thread_dispatcher& td, int soft_limit) : my_serializer(td, soft_limit)\n{}\n\nvoid thread_request_serializer_proxy::register_mandatory_request(int mandatory_delta) {\n if (mandatory_delta != 0) {\n mutex_type::scoped_lock lock(my_mutex, /* is_write = */ false);\n int prev_value = my_num_mandatory_requests.fetch_add(mandatory_delta);\n\n const bool should_try_enable = mandatory_delta > 0 && prev_value == 0;\n const bool should_try_disable = mandatory_delta < 0 && prev_value == 1;\n\n if (should_try_enable) {\n enable_mandatory_concurrency(lock);\n } else if (should_try_disable) {\n disable_mandatory_concurrency(lock);\n }\n }\n}\n\nvoid thread_request_serializer_proxy::set_active_num_workers(int soft_limit) {\n mutex_type::scoped_lock lock(my_mutex, /* is_write = */ true);\n\n if (soft_limit != 0) {\n my_is_mandatory_concurrency_enabled = false;\n } else if (my_num_mandatory_requests > 0) {\n my_is_mandatory_concurrency_enabled = true;\n soft_limit = 1;\n }\n\n my_serializer.set_active_num_workers(soft_limit);\n}\n\nint thread_request_serializer_proxy::num_workers_requested() { return my_serializer.num_workers_requested(); }\n\nvoid thread_request_serializer_proxy::update(int delta) { my_serializer.update(delta); }\n\nvoid thread_request_serializer_proxy::enable_mandatory_concurrency(mutex_type::scoped_lock& lock) {\n lock.upgrade_to_writer();\n bool still_should_enable = my_num_mandatory_requests.load(std::memory_order_relaxed) > 0 &&\n !my_is_mandatory_concurrency_enabled && my_serializer.is_no_workers_avaliable();\n\n if (still_should_enable) {\n my_is_mandatory_concurrency_enabled = true;\n my_serializer.set_active_num_workers(1);\n }\n}\n\nvoid thread_request_serializer_proxy::disable_mandatory_concurrency(mutex_type::scoped_lock& lock) {\n lock.upgrade_to_writer();\n bool still_should_disable = my_num_mandatory_requests.load(std::memory_order_relaxed) <= 0 &&\n my_is_mandatory_concurrency_enabled && !my_serializer.is_no_workers_avaliable();\n\n if (still_should_disable) {\n my_is_mandatory_concurrency_enabled = false;\n my_serializer.set_active_num_workers(0);\n }\n}\n\n} // r1\n} // detail\n} // tbb\n" }, { "path": "src/tbb/src/tbb/thread_request_serializer.h", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_thread_serializer_handlers_H\n#define _TBB_thread_serializer_handlers_H\n\n#include \"oneapi/tbb/mutex.h\"\n#include \"oneapi/tbb/rw_mutex.h\"\n\n#include \"thread_dispatcher.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass thread_request_observer {\nprotected:\n virtual ~thread_request_observer() {}\npublic:\n virtual void update(int delta) = 0;\n};\n\n\nclass thread_request_serializer : public thread_request_observer {\n using mutex_type = d1::mutex;\npublic:\n thread_request_serializer(thread_dispatcher& td, int soft_limit);\n void set_active_num_workers(int soft_limit);\n int num_workers_requested() { return my_total_request.load(std::memory_order_relaxed); }\n bool is_no_workers_avaliable() { return my_soft_limit == 0; }\n\nprivate:\n friend class thread_request_serializer_proxy;\n void update(int delta) override;\n static int limit_delta(int delta, int limit, int new_value);\n\n thread_dispatcher& my_thread_dispatcher;\n int my_soft_limit{ 0 };\n std::atomic my_total_request{ 0 };\n // my_pending_delta is set to pending_delta_base to have ability to hold negative values\n // consider increase base since thead number will be bigger than 1 << 15\n static constexpr std::uint64_t pending_delta_base = 1 << 15;\n std::atomic my_pending_delta{ pending_delta_base };\n mutex_type my_mutex;\n};\n\n// Handles mandatory concurrency i.e. enables worker threads for enqueue tasks\nclass thread_request_serializer_proxy : public thread_request_observer {\n using mutex_type = d1::rw_mutex;\npublic:\n thread_request_serializer_proxy(thread_dispatcher& td, int soft_limit);\n void register_mandatory_request(int mandatory_delta);\n void set_active_num_workers(int soft_limit);\n int num_workers_requested();\n\nprivate:\n void update(int delta) override;\n void enable_mandatory_concurrency(mutex_type::scoped_lock& lock);\n void disable_mandatory_concurrency(mutex_type::scoped_lock& lock);\n\n std::atomic my_num_mandatory_requests{0};\n bool my_is_mandatory_concurrency_enabled{false};\n thread_request_serializer my_serializer;\n mutex_type my_mutex;\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_thread_serializer_handlers_H\n" }, { "path": "src/tbb/src/tbb/threading_control.cpp", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"threading_control.h\"\n#include \"permit_manager.h\"\n#include \"market.h\"\n#include \"tcm_adaptor.h\"\n#include \"thread_dispatcher.h\"\n#include \"governor.h\"\n#include \"thread_dispatcher_client.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n// ---------------------------------------- threading_control_impl --------------------------------------------------------------\n\nstd::size_t global_control_active_value_unsafe(d1::global_control::parameter);\n\nstd::pair threading_control_impl::calculate_workers_limits() {\n // Expecting that 4P is suitable for most applications.\n // Limit to 2P for large thread number.\n // TODO: ask RML for max concurrency and possibly correct hard_limit\n unsigned factor = governor::default_num_threads() <= 128 ? 4 : 2;\n\n // The requested number of threads is intentionally not considered in\n // computation of the hard limit, in order to separate responsibilities\n // and avoid complicated interactions between global_control and task_scheduler_init.\n // The threading control guarantees that at least 256 threads might be created.\n unsigned workers_app_limit = global_control_active_value_unsafe(global_control::max_allowed_parallelism);\n unsigned workers_hard_limit = max(max(factor * governor::default_num_threads(), 256u), workers_app_limit);\n unsigned workers_soft_limit = calc_workers_soft_limit(workers_hard_limit);\n \n return std::make_pair(workers_soft_limit, workers_hard_limit);\n}\n\nunsigned threading_control_impl::calc_workers_soft_limit(unsigned workers_hard_limit) {\n unsigned workers_soft_limit{};\n unsigned soft_limit = global_control_active_value_unsafe(global_control::max_allowed_parallelism);\n\n // if user set no limits (yet), use default value\n workers_soft_limit = soft_limit != 0 ? soft_limit - 1 : governor::default_num_threads() - 1;\n\n if (workers_soft_limit >= workers_hard_limit) {\n workers_soft_limit = workers_hard_limit - 1;\n }\n\n return workers_soft_limit;\n}\n\ncache_aligned_unique_ptr threading_control_impl::make_permit_manager(unsigned workers_soft_limit) {\n if (tcm_adaptor::is_initialized()) {\n auto tcm = make_cache_aligned_unique();\n if (tcm->is_connected()) {\n return tcm;\n }\n }\n return make_cache_aligned_unique(workers_soft_limit);\n}\n\ncache_aligned_unique_ptr threading_control_impl::make_thread_dispatcher(threading_control& tc,\n unsigned workers_soft_limit,\n unsigned workers_hard_limit)\n{\n stack_size_type stack_size = global_control_active_value_unsafe(global_control::thread_stack_size);\n\n cache_aligned_unique_ptr td =\n make_cache_aligned_unique(tc, workers_hard_limit, stack_size);\n // This check relies on the fact that for shared RML default_concurrency == max_concurrency\n if (!governor::UsePrivateRML && td->my_server->default_concurrency() < workers_soft_limit) {\n runtime_warning(\"RML might limit the number of workers to %u while %u is requested.\\n\",\n td->my_server->default_concurrency(), workers_soft_limit);\n }\n\n return td;\n}\n\nthreading_control_impl::threading_control_impl(threading_control* tc) {\n unsigned workers_soft_limit{}, workers_hard_limit{};\n std::tie(workers_soft_limit, workers_hard_limit) = calculate_workers_limits();\n\n my_permit_manager = make_permit_manager(workers_soft_limit);\n my_thread_dispatcher = make_thread_dispatcher(*tc, workers_soft_limit, workers_hard_limit);\n my_thread_request_serializer =\n make_cache_aligned_unique(*my_thread_dispatcher, workers_soft_limit);\n my_permit_manager->set_thread_request_observer(*my_thread_request_serializer);\n\n my_cancellation_disseminator = make_cache_aligned_unique();\n my_waiting_threads_monitor = make_cache_aligned_unique();\n}\n\nvoid threading_control_impl::release(bool blocking_terminate) {\n my_thread_dispatcher->release(blocking_terminate);\n}\n\nvoid threading_control_impl::set_active_num_workers(unsigned soft_limit) {\n __TBB_ASSERT(soft_limit <= my_thread_dispatcher->my_num_workers_hard_limit, nullptr);\n my_thread_request_serializer->set_active_num_workers(soft_limit);\n my_permit_manager->set_active_num_workers(soft_limit);\n}\n\nthreading_control_client threading_control_impl::create_client(arena& a) {\n pm_client* pm_client = my_permit_manager->create_client(a);\n thread_dispatcher_client* td_client = my_thread_dispatcher->create_client(a);\n\n return threading_control_client{pm_client, td_client};\n}\n\nthreading_control_impl::client_snapshot threading_control_impl::prepare_client_destruction(threading_control_client client) {\n auto td_client = client.get_thread_dispatcher_client();\n return {td_client->get_aba_epoch(), td_client->priority_level(), td_client, client.get_pm_client()};\n}\n\nbool threading_control_impl::try_destroy_client(threading_control_impl::client_snapshot snapshot) {\n if (my_thread_dispatcher->try_unregister_client(snapshot.my_td_client, snapshot.aba_epoch, snapshot.priority_level)) {\n my_permit_manager->unregister_and_destroy_client(*snapshot.my_pm_client);\n return true;\n }\n return false;\n}\n\nvoid threading_control_impl::publish_client(threading_control_client tc_client, d1::constraints& constraints) {\n my_permit_manager->register_client(tc_client.get_pm_client(), constraints);\n my_thread_dispatcher->register_client(tc_client.get_thread_dispatcher_client());\n}\n\nvoid threading_control_impl::register_thread(thread_data& td) {\n my_cancellation_disseminator->register_thread(td);\n}\nvoid threading_control_impl::unregister_thread(thread_data& td) {\n my_cancellation_disseminator->unregister_thread(td);\n}\n\nvoid threading_control_impl::propagate_task_group_state(std::atomic d1::task_group_context::*mptr_state,\n d1::task_group_context& src, uint32_t new_state)\n{\n my_cancellation_disseminator->propagate_task_group_state(mptr_state, src, new_state);\n}\n\nstd::size_t threading_control_impl::worker_stack_size() {\n return my_thread_dispatcher->worker_stack_size();\n}\n\nunsigned threading_control_impl::max_num_workers() {\n return my_thread_dispatcher->my_num_workers_hard_limit;\n}\n\nvoid threading_control_impl::adjust_demand(threading_control_client tc_client, int mandatory_delta, int workers_delta) {\n auto& c = *tc_client.get_pm_client();\n my_thread_request_serializer->register_mandatory_request(mandatory_delta);\n my_permit_manager->adjust_demand(c, mandatory_delta, workers_delta);\n}\n\nbool threading_control_impl::is_any_other_client_active() {\n return my_thread_request_serializer->num_workers_requested() > 0 ? my_thread_dispatcher->is_any_client_in_need() : false;\n}\n\nthread_control_monitor& threading_control_impl::get_waiting_threads_monitor() {\n return *my_waiting_threads_monitor;\n}\n\n// ---------------------------------------- threading_control -------------------------------------------------------------------\n\n// Defined in global_control.cpp\nvoid global_control_lock();\nvoid global_control_unlock();\n\nvoid threading_control::add_ref(bool is_public) {\n ++my_ref_count;\n if (is_public) {\n my_public_ref_count++;\n }\n}\n\nbool threading_control::remove_ref(bool is_public) {\n if (is_public) {\n __TBB_ASSERT(g_threading_control == this, \"Global threading control instance was destroyed prematurely?\");\n __TBB_ASSERT(my_public_ref_count.load(std::memory_order_relaxed), nullptr);\n --my_public_ref_count;\n }\n\n bool is_last_ref = --my_ref_count == 0;\n if (is_last_ref) {\n __TBB_ASSERT(!my_public_ref_count.load(std::memory_order_relaxed), nullptr);\n g_threading_control = nullptr;\n }\n\n return is_last_ref;\n}\n\nthreading_control* threading_control::get_threading_control(bool is_public) {\n threading_control* control = g_threading_control;\n if (control) {\n control->add_ref(is_public);\n }\n\n return control;\n}\n\nthreading_control* threading_control::create_threading_control() {\n // Global control should be locked before threading_control_impl\n global_control_lock();\n\n threading_control* thr_control{ nullptr };\n try_call([&] {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n\n thr_control = get_threading_control(/*public = */ true);\n if (thr_control == nullptr) {\n thr_control = new (cache_aligned_allocate(sizeof(threading_control))) threading_control(/*public_ref = */ 1, /*private_ref = */ 1);\n thr_control->my_pimpl = make_cache_aligned_unique(thr_control);\n\n __TBB_InitOnce::add_ref();\n\n if (global_control_active_value_unsafe(global_control::scheduler_handle)) {\n ++thr_control->my_public_ref_count;\n ++thr_control->my_ref_count;\n }\n\n g_threading_control = thr_control;\n }\n }).on_exception([&] {\n global_control_unlock();\n\n cache_aligned_deleter deleter{};\n deleter(thr_control);\n });\n\n global_control_unlock();\n return thr_control;\n}\n\nvoid threading_control::destroy () {\n cache_aligned_deleter deleter;\n deleter(this);\n __TBB_InitOnce::remove_ref();\n}\n\nvoid threading_control::wait_last_reference(global_mutex_type::scoped_lock& lock) {\n while (my_public_ref_count.load(std::memory_order_relaxed) == 1 && my_ref_count.load(std::memory_order_relaxed) > 1) {\n lock.release();\n // To guarantee that request_close_connection() is called by the last external thread, we need to wait till all\n // references are released. Re-read my_public_ref_count to limit waiting if new external threads are created.\n // Theoretically, new private references to the threading control can be added during waiting making it potentially\n // endless.\n // TODO: revise why the weak scheduler needs threading control's pointer and try to remove this wait.\n // Note that the threading control should know about its schedulers for cancellation/exception/priority propagation,\n // see e.g. task_group_context::cancel_group_execution()\n while (my_public_ref_count.load(std::memory_order_acquire) == 1 && my_ref_count.load(std::memory_order_acquire) > 1) {\n yield();\n }\n lock.acquire(g_threading_control_mutex);\n }\n}\n\nbool threading_control::release(bool is_public, bool blocking_terminate) {\n bool do_release = false;\n {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n if (blocking_terminate) {\n __TBB_ASSERT(is_public, \"Only an object with a public reference can request the blocking terminate\");\n wait_last_reference(lock);\n }\n do_release = remove_ref(is_public);\n }\n\n if (do_release) {\n __TBB_ASSERT(!my_public_ref_count.load(std::memory_order_relaxed), \"No public references must remain if we remove the threading control.\");\n // inform RML that blocking termination is required\n my_pimpl->release(blocking_terminate);\n return blocking_terminate;\n }\n return false;\n}\n\nthreading_control::threading_control(unsigned public_ref, unsigned ref) : my_public_ref_count(public_ref), my_ref_count(ref)\n{}\n\nthreading_control* threading_control::register_public_reference() {\n threading_control* control{nullptr};\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n control = get_threading_control(/*public = */ true);\n if (!control) {\n // We are going to create threading_control_impl, we should acquire mutexes in right order\n lock.release();\n control = create_threading_control();\n }\n\n return control;\n}\n\nbool threading_control::unregister_public_reference(bool blocking_terminate) {\n __TBB_ASSERT(g_threading_control, \"Threading control should exist until last public reference\");\n __TBB_ASSERT(g_threading_control->my_public_ref_count.load(std::memory_order_relaxed), nullptr);\n return g_threading_control->release(/*public = */ true, /*blocking_terminate = */ blocking_terminate);\n}\n\nthreading_control_client threading_control::create_client(arena& a) {\n {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n add_ref(/*public = */ false);\n }\n\n return my_pimpl->create_client(a);\n}\n\nvoid threading_control::publish_client(threading_control_client client, d1::constraints& constraints) {\n return my_pimpl->publish_client(client, constraints);\n}\n\nthreading_control::client_snapshot threading_control::prepare_client_destruction(threading_control_client client) {\n return my_pimpl->prepare_client_destruction(client);\n}\n\nbool threading_control::try_destroy_client(threading_control::client_snapshot deleter) {\n bool res = my_pimpl->try_destroy_client(deleter);\n if (res) {\n release(/*public = */ false, /*blocking_terminate = */ false);\n }\n return res;\n}\n\nvoid threading_control::set_active_num_workers(unsigned soft_limit) {\n threading_control* thr_control{nullptr};\n {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n thr_control = get_threading_control(/*public = */ false);\n }\n\n if (thr_control != nullptr) {\n thr_control->my_pimpl->set_active_num_workers(soft_limit);\n thr_control->release(/*is_public=*/false, /*blocking_terminate=*/false);\n }\n}\n\nbool threading_control::is_present() {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n return g_threading_control != nullptr;\n}\n\nbool threading_control::register_lifetime_control() {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n return get_threading_control(/*public = */ true) != nullptr;\n}\n\nbool threading_control::unregister_lifetime_control(bool blocking_terminate) {\n threading_control* thr_control{nullptr};\n {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n thr_control = g_threading_control;\n }\n\n bool released{true};\n if (thr_control) {\n released = thr_control->release(/*public = */ true, /*blocking_terminate = */ blocking_terminate);\n }\n\n return released;\n}\n\nvoid threading_control::register_thread(thread_data& td) {\n my_pimpl->register_thread(td);\n}\n\nvoid threading_control::unregister_thread(thread_data& td) {\n my_pimpl->unregister_thread(td);\n}\n\nvoid threading_control::propagate_task_group_state(std::atomic d1::task_group_context::*mptr_state,\n d1::task_group_context& src, uint32_t new_state)\n{\n my_pimpl->propagate_task_group_state(mptr_state, src, new_state);\n}\n\nstd::size_t threading_control::worker_stack_size() {\n return my_pimpl->worker_stack_size();\n}\n\nunsigned threading_control::max_num_workers() {\n global_mutex_type::scoped_lock lock(g_threading_control_mutex);\n return g_threading_control ? g_threading_control->my_pimpl->max_num_workers() : 0;\n}\n\nvoid threading_control::adjust_demand(threading_control_client client, int mandatory_delta, int workers_delta) {\n my_pimpl->adjust_demand(client, mandatory_delta, workers_delta);\n}\n\nbool threading_control::is_any_other_client_active() {\n return my_pimpl->is_any_other_client_active();\n}\n\nthread_control_monitor& threading_control::get_waiting_threads_monitor() {\n return my_pimpl->get_waiting_threads_monitor();\n}\n\n} // r1\n} // detail\n} // tbb\n" }, { "path": "src/tbb/src/tbb/threading_control.h", "content": "/*\n Copyright (c) 2022-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_threading_control_H\n#define _TBB_threading_control_H\n\n#include \"oneapi/tbb/mutex.h\"\n#include \"oneapi/tbb/global_control.h\"\n\n#include \"threading_control_client.h\"\n#include \"intrusive_list.h\"\n#include \"main.h\"\n#include \"permit_manager.h\"\n#include \"pm_client.h\"\n#include \"thread_dispatcher.h\"\n#include \"cancellation_disseminator.h\"\n#include \"thread_request_serializer.h\"\n#include \"scheduler_common.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass arena;\nclass thread_data;\n\nclass threading_control;\n\nclass threading_control_impl {\npublic:\n threading_control_impl(threading_control*);\n\npublic:\n void release(bool blocking_terminate);\n\n threading_control_client create_client(arena& a);\n void publish_client(threading_control_client client, d1::constraints& constraints);\n\n struct client_snapshot {\n std::uint64_t aba_epoch;\n unsigned priority_level;\n thread_dispatcher_client* my_td_client;\n pm_client* my_pm_client;\n };\n\n client_snapshot prepare_client_destruction(threading_control_client client);\n bool try_destroy_client(client_snapshot deleter);\n\n void register_thread(thread_data& td);\n void unregister_thread(thread_data& td);\n void propagate_task_group_state(std::atomic d1::task_group_context::*mptr_state,\n d1::task_group_context& src, uint32_t new_state);\n\n void set_active_num_workers(unsigned soft_limit);\n std::size_t worker_stack_size();\n unsigned max_num_workers();\n\n void adjust_demand(threading_control_client, int mandatory_delta, int workers_delta);\n bool is_any_other_client_active();\n\n thread_control_monitor& get_waiting_threads_monitor();\n\nprivate:\n static unsigned calc_workers_soft_limit(unsigned workers_hard_limit);\n static std::pair calculate_workers_limits();\n static cache_aligned_unique_ptr make_permit_manager(unsigned workers_soft_limit);\n static cache_aligned_unique_ptr make_thread_dispatcher(threading_control& control,\n unsigned workers_soft_limit,\n unsigned workers_hard_limit);\n\n // TODO: Consider allocation one chunk of memory and construct objects on it\n cache_aligned_unique_ptr my_permit_manager{nullptr};\n cache_aligned_unique_ptr my_thread_dispatcher{nullptr};\n cache_aligned_unique_ptr my_thread_request_serializer{nullptr};\n cache_aligned_unique_ptr my_cancellation_disseminator{nullptr};\n cache_aligned_unique_ptr my_waiting_threads_monitor{nullptr};\n};\n\n\nclass threading_control {\n using global_mutex_type = d1::mutex;\npublic:\n using client_snapshot = threading_control_impl::client_snapshot;\n\n static threading_control* register_public_reference();\n static bool unregister_public_reference(bool blocking_terminate);\n\n static bool is_present();\n static void set_active_num_workers(unsigned soft_limit);\n static bool register_lifetime_control();\n static bool unregister_lifetime_control(bool blocking_terminate);\n\n threading_control_client create_client(arena& a);\n void publish_client(threading_control_client client, d1::constraints& constraints);\n client_snapshot prepare_client_destruction(threading_control_client client);\n bool try_destroy_client(client_snapshot deleter);\n\n void register_thread(thread_data& td);\n void unregister_thread(thread_data& td);\n void propagate_task_group_state(std::atomic d1::task_group_context::*mptr_state,\n d1::task_group_context& src, uint32_t new_state);\n\n std::size_t worker_stack_size();\n static unsigned max_num_workers();\n\n void adjust_demand(threading_control_client client, int mandatory_delta, int workers_delta);\n bool is_any_other_client_active();\n\n thread_control_monitor& get_waiting_threads_monitor();\n\nprivate:\n threading_control(unsigned public_ref, unsigned ref);\n void add_ref(bool is_public);\n bool remove_ref(bool is_public);\n\n static threading_control* get_threading_control(bool is_public);\n static threading_control* create_threading_control();\n\n bool release(bool is_public, bool blocking_terminate);\n void wait_last_reference(global_mutex_type::scoped_lock& lock);\n void destroy();\n\n friend class thread_dispatcher;\n\n static threading_control* g_threading_control;\n //! Mutex guarding creation/destruction of g_threading_control, insertions/deletions in my_arenas, and cancellation propagation\n static global_mutex_type g_threading_control_mutex;\n\n cache_aligned_unique_ptr my_pimpl{nullptr};\n //! Count of external threads attached\n std::atomic my_public_ref_count{0};\n //! Reference count controlling threading_control object lifetime\n std::atomic my_ref_count{0};\n};\n\n} // r1\n} // detail\n} // tbb\n\n\n#endif // _TBB_threading_control_H\n" }, { "path": "src/tbb/src/tbb/threading_control_client.h", "content": "/*\n Copyright (c) 2022-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_threading_control_client_H\n#define _TBB_threading_control_client_H\n\n#include \"oneapi/tbb/detail/_assert.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\nclass pm_client;\nclass thread_dispatcher_client;\n\nclass threading_control_client {\npublic:\n threading_control_client() = default;\n threading_control_client(const threading_control_client&) = default;\n threading_control_client& operator=(const threading_control_client&) = default;\n\n threading_control_client(pm_client* p, thread_dispatcher_client* t) : my_pm_client(p), my_thread_dispatcher_client(t) {\n __TBB_ASSERT(my_pm_client, nullptr);\n __TBB_ASSERT(my_thread_dispatcher_client, nullptr);\n }\n\n pm_client* get_pm_client() {\n return my_pm_client;\n }\n\n thread_dispatcher_client* get_thread_dispatcher_client() {\n return my_thread_dispatcher_client;\n }\n\nprivate:\n pm_client* my_pm_client{nullptr};\n thread_dispatcher_client* my_thread_dispatcher_client{nullptr};\n};\n\n\n}\n}\n}\n\n#endif // _TBB_threading_control_client_H\n" }, { "path": "src/tbb/src/tbb/tls.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_tls_H\n#define _TBB_tls_H\n\n#include \"oneapi/tbb/detail/_config.h\"\n\n#if __TBB_USE_POSIX\n#include \n#else /* assume __TBB_USE_WINAPI */\n#include \n#endif\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\ntypedef void (*tls_dtor_t)(void*);\n\n//! Basic cross-platform wrapper class for TLS operations.\ntemplate \nclass basic_tls {\n#if __TBB_USE_POSIX\n typedef pthread_key_t tls_key_t;\npublic:\n int create( tls_dtor_t dtor = nullptr ) {\n return pthread_key_create(&my_key, dtor);\n }\n int destroy() { return pthread_key_delete(my_key); }\n void set( T value ) { pthread_setspecific(my_key, (void*)value); }\n T get() { return (T)pthread_getspecific(my_key); }\n#else /* __TBB_USE_WINAPI */\n typedef DWORD tls_key_t;\npublic:\n#if !__TBB_WIN8UI_SUPPORT\n int create() {\n tls_key_t tmp = TlsAlloc();\n if( tmp==TLS_OUT_OF_INDEXES )\n return TLS_OUT_OF_INDEXES;\n my_key = tmp;\n return 0;\n }\n int destroy() { TlsFree(my_key); my_key=0; return 0; }\n void set( T value ) { TlsSetValue(my_key, (LPVOID)value); }\n T get() { return (T)TlsGetValue(my_key); }\n#else /*!__TBB_WIN8UI_SUPPORT*/\n int create() {\n tls_key_t tmp = FlsAlloc(nullptr);\n if( tmp== (DWORD)0xFFFFFFFF )\n return (DWORD)0xFFFFFFFF;\n my_key = tmp;\n return 0;\n }\n int destroy() { FlsFree(my_key); my_key=0; return 0; }\n void set( T value ) { FlsSetValue(my_key, (LPVOID)value); }\n T get() { return (T)FlsGetValue(my_key); }\n#endif /* !__TBB_WIN8UI_SUPPORT */\n#endif /* __TBB_USE_WINAPI */\nprivate:\n tls_key_t my_key;\n};\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif /* _TBB_tls_H */\n" }, { "path": "src/tbb/src/tbb/tools_api/disable_warnings.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"ittnotify_config.h\"\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n\n#if defined _MSC_VER\n\n// #pragma warning (disable: 593) /* parameter \"XXXX\" was set but never used */\n// #pragma warning (disable: 344) /* typedef name has already been declared (with same type) */\n// #pragma warning (disable: 174) /* expression has no effect */\n// #pragma warning (disable: 4127) /* conditional expression is constant */\n// #pragma warning (disable: 4306) /* conversion from '?' to '?' of greater size */\n\n#endif\n\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if defined __INTEL_COMPILER\n\n// #pragma warning (disable: 869) /* parameter \"XXXXX\" was never referenced */\n// #pragma warning (disable: 1418) /* external function definition with no prior declaration */\n// #pragma warning (disable: 1419) /* external declaration in primary source file */\n\n#endif /* __INTEL_COMPILER */\n" }, { "path": "src/tbb/src/tbb/tools_api/ittnotify.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _ITTNOTIFY_H_\n#define _ITTNOTIFY_H_\n\n/**\n@file\n@brief Public User API functions and types\n@mainpage\n\nThe Instrumentation and Tracing Technology API (ITT API) is used to\nannotate a user's program with additional information\nthat can be used by correctness and performance tools. The user inserts\ncalls in their program. Those calls generate information that is collected\nat runtime, and used by Intel(R) Threading Tools.\n\n@section API Concepts\nThe following general concepts are used throughout the API.\n\n@subsection Unicode Support\nMany API functions take character string arguments. On Windows, there\nare two versions of each such function. The function name is suffixed\nby W if Unicode support is enabled, and by A otherwise. Any API function\nthat takes a character string argument adheres to this convention.\n\n@subsection Conditional Compilation\nMany users prefer having an option to modify ITT API code when linking it\ninside their runtimes. ITT API header file provides a mechanism to replace\nITT API function names inside your code with empty strings. To do this,\ndefine the macros INTEL_NO_ITTNOTIFY_API during compilation and remove the\nstatic library from the linker script.\n\n@subsection Domains\n[see domains]\nDomains provide a way to separate notification for different modules or\nlibraries in a program. Domains are specified by dotted character strings,\ne.g. TBB.Internal.Control.\n\nA mechanism (to be specified) is provided to enable and disable\ndomains. By default, all domains are enabled.\n@subsection Named Entities and Instances\nNamed entities (frames, regions, tasks, and markers) communicate\ninformation about the program to the analysis tools. A named entity often\nrefers to a section of program code, or to some set of logical concepts\nthat the programmer wants to group together.\n\nNamed entities relate to the programmer's static view of the program. When\nthe program actually executes, many instances of a given named entity\nmay be created.\n\nThe API annotations denote instances of named entities. The actual\nnamed entities are displayed using the analysis tools. In other words,\nthe named entities come into existence when instances are created.\n\nInstances of named entities may have instance identifiers (IDs). Some\nAPI calls use instance identifiers to create relationships between\ndifferent instances of named entities. Other API calls associate data\nwith instances of named entities.\n\nSome named entities must always have instance IDs. In particular, regions\nand frames always have IDs. Task and markers need IDs only if the ID is\nneeded in another API call (such as adding a relation or metadata).\n\nThe lifetime of instance IDs is distinct from the lifetime of\ninstances. This allows various relationships to be specified separate\nfrom the actual execution of instances. This flexibility comes at the\nexpense of extra API calls.\n\nThe same ID may not be reused for different instances, unless a previous\n[ref] __itt_id_destroy call for that ID has been issued.\n*/\n\n/** @cond exclude_from_documentation */\n#ifndef ITT_OS_WIN\n# define ITT_OS_WIN 1\n#endif /* ITT_OS_WIN */\n\n#ifndef ITT_OS_LINUX\n# define ITT_OS_LINUX 2\n#endif /* ITT_OS_LINUX */\n\n#ifndef ITT_OS_MAC\n# define ITT_OS_MAC 3\n#endif /* ITT_OS_MAC */\n\n#ifndef ITT_OS_FREEBSD\n# define ITT_OS_FREEBSD 4\n#endif /* ITT_OS_FREEBSD */\n\n#ifndef ITT_OS_OPENBSD\n# define ITT_OS_OPENBSD 5\n#endif /* ITT_OS_OPENBSD */\n\n\n#ifndef ITT_OS\n# if defined WIN32 || defined _WIN32\n# define ITT_OS ITT_OS_WIN\n# elif defined( __APPLE__ ) && defined( __MACH__ )\n# define ITT_OS ITT_OS_MAC\n# elif defined( __FreeBSD__ )\n# define ITT_OS ITT_OS_FREEBSD\n# elif defined( __OpenBSD__ )\n# define ITT_OS ITT_OS_OPENBSD\n# else\n# define ITT_OS ITT_OS_LINUX\n# endif\n#endif /* ITT_OS */\n\n#ifndef ITT_PLATFORM_WIN\n# define ITT_PLATFORM_WIN 1\n#endif /* ITT_PLATFORM_WIN */\n\n#ifndef ITT_PLATFORM_POSIX\n# define ITT_PLATFORM_POSIX 2\n#endif /* ITT_PLATFORM_POSIX */\n\n#ifndef ITT_PLATFORM_MAC\n# define ITT_PLATFORM_MAC 3\n#endif /* ITT_PLATFORM_MAC */\n\n#ifndef ITT_PLATFORM_FREEBSD\n# define ITT_PLATFORM_FREEBSD 4\n#endif /* ITT_PLATFORM_FREEBSD */\n\n#ifndef ITT_PLATFORM_OPENBSD\n# define ITT_PLATFORM_OPENBSD 5\n#endif /* ITT_PLATFORM_OPENBSD */\n\n#ifndef ITT_PLATFORM\n# if ITT_OS==ITT_OS_WIN\n# define ITT_PLATFORM ITT_PLATFORM_WIN\n# elif ITT_OS==ITT_OS_MAC\n# define ITT_PLATFORM ITT_PLATFORM_MAC\n# elif ITT_OS==ITT_OS_FREEBSD\n# define ITT_PLATFORM ITT_PLATFORM_FREEBSD\n# elif ITT_OS==ITT_OS_OPENBSD\n# define ITT_PLATFORM ITT_PLATFORM_OPENBSD\n# else\n# define ITT_PLATFORM ITT_PLATFORM_POSIX\n# endif\n#endif /* ITT_PLATFORM */\n\n#if defined(_UNICODE) && !defined(UNICODE)\n#define UNICODE\n#endif\n\n#include \n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#include \n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#include \n#if defined(UNICODE) || defined(_UNICODE)\n#include \n#endif /* UNICODE || _UNICODE */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef ITTAPI_CDECL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define ITTAPI_CDECL __cdecl\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define ITTAPI_CDECL __attribute__ ((cdecl))\n# else /* _M_IX86 || __i386__ */\n# define ITTAPI_CDECL /* actual only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* ITTAPI_CDECL */\n\n#ifndef STDCALL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define STDCALL __stdcall\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define STDCALL __attribute__ ((stdcall))\n# else /* _M_IX86 || __i386__ */\n# define STDCALL /* supported only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* STDCALL */\n\n#define ITTAPI ITTAPI_CDECL\n#define LIBITTAPI ITTAPI_CDECL\n\n/* TODO: Temporary for compatibility! */\n#define ITTAPI_CALL ITTAPI_CDECL\n#define LIBITTAPI_CALL ITTAPI_CDECL\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n/* use __forceinline (VC++ specific) */\n#if defined(__MINGW32__) && !defined(__cplusplus)\n#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__))\n#else\n#define ITT_INLINE static __forceinline\n#endif /* __MINGW32__ */\n\n#define ITT_INLINE_ATTRIBUTE /* nothing */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/*\n * Generally, functions are not inlined unless optimization is specified.\n * For functions declared inline, this attribute inlines the function even\n * if no optimization level was specified.\n */\n#ifdef __STRICT_ANSI__\n#define ITT_INLINE static\n#define ITT_INLINE_ATTRIBUTE __attribute__((unused))\n#else /* __STRICT_ANSI__ */\n#define ITT_INLINE static inline\n#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused))\n#endif /* __STRICT_ANSI__ */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/** @endcond */\n\n#ifdef INTEL_ITTNOTIFY_ENABLE_LEGACY\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# pragma message(\"WARNING!!! Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro\")\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# warning \"Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro\"\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# include \"legacy/ittnotify.h\"\n#endif /* INTEL_ITTNOTIFY_ENABLE_LEGACY */\n\n/** @cond exclude_from_documentation */\n/* Helper macro for joining tokens */\n#define ITT_JOIN_AUX(p,n) p##n\n#define ITT_JOIN(p,n) ITT_JOIN_AUX(p,n)\n\n#ifdef ITT_MAJOR\n#undef ITT_MAJOR\n#endif\n#ifdef ITT_MINOR\n#undef ITT_MINOR\n#endif\n#define ITT_MAJOR 3\n#define ITT_MINOR 0\n\n/* Standard versioning of a token with major and minor version numbers */\n#define ITT_VERSIONIZE(x) \\\n ITT_JOIN(x, \\\n ITT_JOIN(_, \\\n ITT_JOIN(ITT_MAJOR, \\\n ITT_JOIN(_, ITT_MINOR))))\n\n#ifndef INTEL_ITTNOTIFY_PREFIX\n# define INTEL_ITTNOTIFY_PREFIX __itt_\n#endif /* INTEL_ITTNOTIFY_PREFIX */\n#ifndef INTEL_ITTNOTIFY_POSTFIX\n# define INTEL_ITTNOTIFY_POSTFIX _ptr_\n#endif /* INTEL_ITTNOTIFY_POSTFIX */\n\n#define ITTNOTIFY_NAME_AUX(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n)\n#define ITTNOTIFY_NAME(n) ITT_VERSIONIZE(ITTNOTIFY_NAME_AUX(ITT_JOIN(n,INTEL_ITTNOTIFY_POSTFIX)))\n\n#define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)\n#define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)\n\n#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d)\n#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x)\n#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y)\n#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z)\n#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a)\n#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b)\n#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c)\n#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d)\n#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x)\n#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y)\n#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z)\n#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a)\n#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b)\n#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c)\n\n#ifdef ITT_STUB\n#undef ITT_STUB\n#endif\n#ifdef ITT_STUBV\n#undef ITT_STUBV\n#endif\n#define ITT_STUBV(api,type,name,args) \\\n typedef type (api* ITT_JOIN(ITTNOTIFY_NAME(name),_t)) args; \\\n extern ITT_JOIN(ITTNOTIFY_NAME(name),_t) ITTNOTIFY_NAME(name);\n#define ITT_STUB ITT_STUBV\n/** @endcond */\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif /* __cplusplus */\n\n/** @cond exclude_from_gpa_documentation */\n/**\n * @defgroup public Public API\n * @{\n * @}\n */\n\n/**\n * @defgroup control Collection Control\n * @ingroup public\n * General behavior: application continues to run, but no profiling information is being collected\n *\n * Pausing occurs not only for the current thread but for all process as well as spawned processes\n * - Intel(R) Parallel Inspector and Intel(R) Inspector XE:\n * - Does not analyze or report errors that involve memory access.\n * - Other errors are reported as usual. Pausing data collection in\n * Intel(R) Parallel Inspector and Intel(R) Inspector XE\n * only pauses tracing and analyzing memory access.\n * It does not pause tracing or analyzing threading APIs.\n * .\n * Intel(R) VTune(TM) Profiler:\n * - Does continue to record when new threads are started.\n * .\n * - Other effects:\n * - Possible reduction of runtime overhead.\n * .\n * @{\n */\n/** @brief Pause collection */\nvoid ITTAPI __itt_pause(void);\n/** @brief Resume collection */\nvoid ITTAPI __itt_resume(void);\n/** @brief Detach collection */\nvoid ITTAPI __itt_detach(void);\n\n/**\n * @enum __itt_collection_scope\n * @brief Enumerator for collection scopes\n */\ntypedef enum {\n __itt_collection_scope_host = 1 << 0,\n __itt_collection_scope_offload = 1 << 1,\n __itt_collection_scope_all = 0x7FFFFFFF\n} __itt_collection_scope;\n\n/** @brief Pause scoped collection */\nvoid ITTAPI __itt_pause_scoped(__itt_collection_scope);\n/** @brief Resume scoped collection */\nvoid ITTAPI __itt_resume_scoped(__itt_collection_scope);\n \n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, pause, (void))\nITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope))\nITT_STUBV(ITTAPI, void, resume, (void))\nITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope))\nITT_STUBV(ITTAPI, void, detach, (void))\n#define __itt_pause ITTNOTIFY_VOID(pause)\n#define __itt_pause_ptr ITTNOTIFY_NAME(pause)\n#define __itt_pause_scoped ITTNOTIFY_VOID(pause_scoped)\n#define __itt_pause_scoped_ptr ITTNOTIFY_NAME(pause_scoped)\n#define __itt_resume ITTNOTIFY_VOID(resume)\n#define __itt_resume_ptr ITTNOTIFY_NAME(resume)\n#define __itt_resume_scoped ITTNOTIFY_VOID(resume_scoped)\n#define __itt_resume_scoped_ptr ITTNOTIFY_NAME(resume_scoped)\n#define __itt_detach ITTNOTIFY_VOID(detach)\n#define __itt_detach_ptr ITTNOTIFY_NAME(detach)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_pause()\n#define __itt_pause_ptr 0\n#define __itt_pause_scoped(scope)\n#define __itt_pause_scoped_ptr 0\n#define __itt_resume()\n#define __itt_resume_ptr 0\n#define __itt_resume_scoped(scope)\n#define __itt_resume_scoped_ptr 0\n#define __itt_detach()\n#define __itt_detach_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_pause_ptr 0\n#define __itt_pause_scoped_ptr 0\n#define __itt_resume_ptr 0\n#define __itt_resume_scoped_ptr 0\n#define __itt_detach_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} control group */\n/** @endcond */\n\n/**\n * @defgroup Intel Processor Trace control\n * API from this group provides control over collection and analysis of Intel Processor Trace (Intel PT) data\n * Information about Intel Processor Trace technology can be found here (Volume 3 chapter 35):\n * https://github.com/tpn/pdfs/blob/master/Intel%2064%20and%20IA-32%20Architectures%20Software%20Developer's%20Manual%20-%20Combined%20Volumes%201-4%20-%20May%202018%20(325462-sdm-vol-1-2abcd-3abcd).pdf\n * Use this API to mark particular code regions for loading detailed performance statistics.\n * This mode makes your analysis faster and more accurate.\n * @{\n*/\ntypedef unsigned char __itt_pt_region;\n\n/**\n * @brief function saves a region name marked with Intel PT API and returns a region id.\n * Only 7 names can be registered. Attempts to register more names will be ignored and a region id with auto names will be returned.\n * For automatic naming of regions pass NULL as function parameter\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_pt_region ITTAPI __itt_pt_region_createA(const char *name);\n__itt_pt_region ITTAPI __itt_pt_region_createW(const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_pt_region_create __itt_pt_region_createW\n#else /* UNICODE */\n# define __itt_pt_region_create __itt_pt_region_createA\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_pt_region ITTAPI __itt_pt_region_create(const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name))\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_pt_region_createA ITTNOTIFY_DATA(pt_region_createA)\n#define __itt_pt_region_createA_ptr ITTNOTIFY_NAME(pt_region_createA)\n#define __itt_pt_region_createW ITTNOTIFY_DATA(pt_region_createW)\n#define __itt_pt_region_createW_ptr ITTNOTIFY_NAME(pt_region_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_pt_region_create ITTNOTIFY_DATA(pt_region_create)\n#define __itt_pt_region_create_ptr ITTNOTIFY_NAME(pt_region_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_pt_region_createA(name) (__itt_pt_region)0\n#define __itt_pt_region_createA_ptr 0\n#define __itt_pt_region_createW(name) (__itt_pt_region)0\n#define __itt_pt_region_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_pt_region_create(name) (__itt_pt_region)0\n#define __itt_pt_region_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_pt_region_createA_ptr 0\n#define __itt_pt_region_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_pt_region_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief function contains a special code pattern identified on the post-processing stage and\n * marks the beginning of a code region targeted for Intel PT analysis\n * @param[in] region - region id, 0 <= region < 8\n*/\nvoid __itt_mark_pt_region_begin(__itt_pt_region region);\n/**\n * @brief function contains a special code pattern identified on the post-processing stage and\n * marks the end of a code region targeted for Intel PT analysis\n * @param[in] region - region id, 0 <= region < 8\n*/\nvoid __itt_mark_pt_region_end(__itt_pt_region region);\n/** @} Intel PT control group*/\n\n/**\n * @defgroup threads Threads\n * @ingroup public\n * Give names to threads\n * @{\n */\n/**\n * @brief Sets thread name of calling thread\n * @param[in] name - name of thread\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_thread_set_nameA(const char *name);\nvoid ITTAPI __itt_thread_set_nameW(const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_thread_set_name __itt_thread_set_nameW\n# define __itt_thread_set_name_ptr __itt_thread_set_nameW_ptr\n#else /* UNICODE */\n# define __itt_thread_set_name __itt_thread_set_nameA\n# define __itt_thread_set_name_ptr __itt_thread_set_nameA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_thread_set_name(const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name))\nITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, thread_set_name, (const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thread_set_nameA ITTNOTIFY_VOID(thread_set_nameA)\n#define __itt_thread_set_nameA_ptr ITTNOTIFY_NAME(thread_set_nameA)\n#define __itt_thread_set_nameW ITTNOTIFY_VOID(thread_set_nameW)\n#define __itt_thread_set_nameW_ptr ITTNOTIFY_NAME(thread_set_nameW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thread_set_name ITTNOTIFY_VOID(thread_set_name)\n#define __itt_thread_set_name_ptr ITTNOTIFY_NAME(thread_set_name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thread_set_nameA(name)\n#define __itt_thread_set_nameA_ptr 0\n#define __itt_thread_set_nameW(name)\n#define __itt_thread_set_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thread_set_name(name)\n#define __itt_thread_set_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thread_set_nameA_ptr 0\n#define __itt_thread_set_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thread_set_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @cond exclude_from_gpa_documentation */\n\n/**\n * @brief Mark current thread as ignored from this point on, for the duration of its existence.\n */\nvoid ITTAPI __itt_thread_ignore(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, thread_ignore, (void))\n#define __itt_thread_ignore ITTNOTIFY_VOID(thread_ignore)\n#define __itt_thread_ignore_ptr ITTNOTIFY_NAME(thread_ignore)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_thread_ignore()\n#define __itt_thread_ignore_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_thread_ignore_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} threads group */\n\n/**\n * @defgroup suppress Error suppression\n * @ingroup public\n * General behavior: application continues to run, but errors are suppressed\n *\n * @{\n */\n\n/*****************************************************************//**\n * @name group of functions used for error suppression in correctness tools\n *********************************************************************/\n/** @{ */\n/**\n * @hideinitializer\n * @brief possible value for suppression mask\n */\n#define __itt_suppress_all_errors 0x7fffffff\n\n/**\n * @hideinitializer\n * @brief possible value for suppression mask (suppresses errors from threading analysis)\n */\n#define __itt_suppress_threading_errors 0x000000ff\n\n/**\n * @hideinitializer\n * @brief possible value for suppression mask (suppresses errors from memory analysis)\n */\n#define __itt_suppress_memory_errors 0x0000ff00\n\n/**\n * @brief Start suppressing errors identified in mask on this thread\n */\nvoid ITTAPI __itt_suppress_push(unsigned int mask);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask))\n#define __itt_suppress_push ITTNOTIFY_VOID(suppress_push)\n#define __itt_suppress_push_ptr ITTNOTIFY_NAME(suppress_push)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_suppress_push(mask)\n#define __itt_suppress_push_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_suppress_push_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Undo the effects of the matching call to __itt_suppress_push\n */\nvoid ITTAPI __itt_suppress_pop(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, suppress_pop, (void))\n#define __itt_suppress_pop ITTNOTIFY_VOID(suppress_pop)\n#define __itt_suppress_pop_ptr ITTNOTIFY_NAME(suppress_pop)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_suppress_pop()\n#define __itt_suppress_pop_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_suppress_pop_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @enum __itt_suppress_mode\n * @brief Enumerator for the suppressing modes\n */\ntypedef enum __itt_suppress_mode {\n __itt_unsuppress_range,\n __itt_suppress_range\n} __itt_suppress_mode_t;\n\n/**\n * @enum __itt_collection_state\n * @brief Enumerator for collection state.\n */\ntypedef enum {\n __itt_collection_uninitialized = 0, /* uninitialized */\n __itt_collection_init_fail = 1, /* failed to init */\n __itt_collection_collector_absent = 2, /* non work state collector is absent */\n __itt_collection_collector_exists = 3, /* work state collector exists */\n __itt_collection_init_successful = 4 /* success to init */\n} __itt_collection_state;\n\n/**\n * @brief Mark a range of memory for error suppression or unsuppression for error types included in mask\n */\nvoid ITTAPI __itt_suppress_mark_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size))\n#define __itt_suppress_mark_range ITTNOTIFY_VOID(suppress_mark_range)\n#define __itt_suppress_mark_range_ptr ITTNOTIFY_NAME(suppress_mark_range)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_suppress_mark_range(mask)\n#define __itt_suppress_mark_range_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_suppress_mark_range_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Undo the effect of a matching call to __itt_suppress_mark_range. If not matching\n * call is found, nothing is changed.\n */\nvoid ITTAPI __itt_suppress_clear_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, suppress_clear_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size))\n#define __itt_suppress_clear_range ITTNOTIFY_VOID(suppress_clear_range)\n#define __itt_suppress_clear_range_ptr ITTNOTIFY_NAME(suppress_clear_range)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_suppress_clear_range(mask)\n#define __itt_suppress_clear_range_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_suppress_clear_range_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} */\n/** @} suppress group */\n\n/**\n * @defgroup sync Synchronization\n * @ingroup public\n * Indicate user-written synchronization code\n * @{\n */\n/**\n * @hideinitializer\n * @brief possible value of attribute argument for sync object type\n */\n#define __itt_attr_barrier 1\n\n/**\n * @hideinitializer\n * @brief possible value of attribute argument for sync object type\n */\n#define __itt_attr_mutex 2\n\n/**\n@brief Name a synchronization object\n@param[in] addr Handle for the synchronization object. You should\nuse a real address to uniquely identify the synchronization object.\n@param[in] objtype null-terminated object type string. If NULL is\npassed, the name will be \"User Synchronization\".\n@param[in] objname null-terminated object name string. If NULL,\nno name will be assigned to the object.\n@param[in] attribute one of [#__itt_attr_barrier, #__itt_attr_mutex]\n */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_sync_createA(void *addr, const char *objtype, const char *objname, int attribute);\nvoid ITTAPI __itt_sync_createW(void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_sync_create __itt_sync_createW\n# define __itt_sync_create_ptr __itt_sync_createW_ptr\n#else /* UNICODE */\n# define __itt_sync_create __itt_sync_createA\n# define __itt_sync_create_ptr __itt_sync_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_sync_create (void *addr, const char *objtype, const char *objname, int attribute);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char *objtype, const char *objname, int attribute))\nITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_create, (void *addr, const char* objtype, const char* objname, int attribute))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_createA ITTNOTIFY_VOID(sync_createA)\n#define __itt_sync_createA_ptr ITTNOTIFY_NAME(sync_createA)\n#define __itt_sync_createW ITTNOTIFY_VOID(sync_createW)\n#define __itt_sync_createW_ptr ITTNOTIFY_NAME(sync_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_create ITTNOTIFY_VOID(sync_create)\n#define __itt_sync_create_ptr ITTNOTIFY_NAME(sync_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_createA(addr, objtype, objname, attribute)\n#define __itt_sync_createA_ptr 0\n#define __itt_sync_createW(addr, objtype, objname, attribute)\n#define __itt_sync_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_create(addr, objtype, objname, attribute)\n#define __itt_sync_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_createA_ptr 0\n#define __itt_sync_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n@brief Rename a synchronization object\n\nYou can use the rename call to assign or reassign a name to a given\nsynchronization object.\n@param[in] addr handle for the synchronization object.\n@param[in] name null-terminated object name string.\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_sync_renameA(void *addr, const char *name);\nvoid ITTAPI __itt_sync_renameW(void *addr, const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_sync_rename __itt_sync_renameW\n# define __itt_sync_rename_ptr __itt_sync_renameW_ptr\n#else /* UNICODE */\n# define __itt_sync_rename __itt_sync_renameA\n# define __itt_sync_rename_ptr __itt_sync_renameA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_sync_rename(void *addr, const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char *name))\nITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_rename, (void *addr, const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_renameA ITTNOTIFY_VOID(sync_renameA)\n#define __itt_sync_renameA_ptr ITTNOTIFY_NAME(sync_renameA)\n#define __itt_sync_renameW ITTNOTIFY_VOID(sync_renameW)\n#define __itt_sync_renameW_ptr ITTNOTIFY_NAME(sync_renameW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_rename ITTNOTIFY_VOID(sync_rename)\n#define __itt_sync_rename_ptr ITTNOTIFY_NAME(sync_rename)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_renameA(addr, name)\n#define __itt_sync_renameA_ptr 0\n#define __itt_sync_renameW(addr, name)\n#define __itt_sync_renameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_rename(addr, name)\n#define __itt_sync_rename_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_renameA_ptr 0\n#define __itt_sync_renameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_rename_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n @brief Destroy a synchronization object.\n @param addr Handle for the synchronization object.\n */\nvoid ITTAPI __itt_sync_destroy(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, sync_destroy, (void *addr))\n#define __itt_sync_destroy ITTNOTIFY_VOID(sync_destroy)\n#define __itt_sync_destroy_ptr ITTNOTIFY_NAME(sync_destroy)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_sync_destroy(addr)\n#define __itt_sync_destroy_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_sync_destroy_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/*****************************************************************//**\n * @name group of functions is used for performance measurement tools\n *********************************************************************/\n/** @{ */\n/**\n * @brief Enter spin loop on user-defined sync object\n */\nvoid ITTAPI __itt_sync_prepare(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, sync_prepare, (void *addr))\n#define __itt_sync_prepare ITTNOTIFY_VOID(sync_prepare)\n#define __itt_sync_prepare_ptr ITTNOTIFY_NAME(sync_prepare)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_sync_prepare(addr)\n#define __itt_sync_prepare_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_sync_prepare_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Quit spin loop without acquiring spin object\n */\nvoid ITTAPI __itt_sync_cancel(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, sync_cancel, (void *addr))\n#define __itt_sync_cancel ITTNOTIFY_VOID(sync_cancel)\n#define __itt_sync_cancel_ptr ITTNOTIFY_NAME(sync_cancel)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_sync_cancel(addr)\n#define __itt_sync_cancel_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_sync_cancel_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Successful spin loop completion (sync object acquired)\n */\nvoid ITTAPI __itt_sync_acquired(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, sync_acquired, (void *addr))\n#define __itt_sync_acquired ITTNOTIFY_VOID(sync_acquired)\n#define __itt_sync_acquired_ptr ITTNOTIFY_NAME(sync_acquired)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_sync_acquired(addr)\n#define __itt_sync_acquired_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_sync_acquired_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Start sync object releasing code. Is called before the lock release call.\n */\nvoid ITTAPI __itt_sync_releasing(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, sync_releasing, (void *addr))\n#define __itt_sync_releasing ITTNOTIFY_VOID(sync_releasing)\n#define __itt_sync_releasing_ptr ITTNOTIFY_NAME(sync_releasing)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_sync_releasing(addr)\n#define __itt_sync_releasing_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_sync_releasing_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} */\n\n/** @} sync group */\n\n/**************************************************************//**\n * @name group of functions is used for correctness checking tools\n ******************************************************************/\n/** @{ */\n/**\n * @ingroup legacy\n * @deprecated Legacy API\n * @brief Fast synchronization which does no require spinning.\n * - This special function is to be used by TBB and OpenMP libraries only when they know\n * there is no spin but they need to suppress TC warnings about shared variable modifications.\n * - It only has corresponding pointers in static library and does not have corresponding function\n * in dynamic library.\n * @see void __itt_sync_prepare(void* addr);\n */\nvoid ITTAPI __itt_fsync_prepare(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, fsync_prepare, (void *addr))\n#define __itt_fsync_prepare ITTNOTIFY_VOID(fsync_prepare)\n#define __itt_fsync_prepare_ptr ITTNOTIFY_NAME(fsync_prepare)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_fsync_prepare(addr)\n#define __itt_fsync_prepare_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_fsync_prepare_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup legacy\n * @deprecated Legacy API\n * @brief Fast synchronization which does no require spinning.\n * - This special function is to be used by TBB and OpenMP libraries only when they know\n * there is no spin but they need to suppress TC warnings about shared variable modifications.\n * - It only has corresponding pointers in static library and does not have corresponding function\n * in dynamic library.\n * @see void __itt_sync_cancel(void *addr);\n */\nvoid ITTAPI __itt_fsync_cancel(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, fsync_cancel, (void *addr))\n#define __itt_fsync_cancel ITTNOTIFY_VOID(fsync_cancel)\n#define __itt_fsync_cancel_ptr ITTNOTIFY_NAME(fsync_cancel)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_fsync_cancel(addr)\n#define __itt_fsync_cancel_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_fsync_cancel_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup legacy\n * @deprecated Legacy API\n * @brief Fast synchronization which does no require spinning.\n * - This special function is to be used by TBB and OpenMP libraries only when they know\n * there is no spin but they need to suppress TC warnings about shared variable modifications.\n * - It only has corresponding pointers in static library and does not have corresponding function\n * in dynamic library.\n * @see void __itt_sync_acquired(void *addr);\n */\nvoid ITTAPI __itt_fsync_acquired(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, fsync_acquired, (void *addr))\n#define __itt_fsync_acquired ITTNOTIFY_VOID(fsync_acquired)\n#define __itt_fsync_acquired_ptr ITTNOTIFY_NAME(fsync_acquired)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_fsync_acquired(addr)\n#define __itt_fsync_acquired_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_fsync_acquired_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup legacy\n * @deprecated Legacy API\n * @brief Fast synchronization which does no require spinning.\n * - This special function is to be used by TBB and OpenMP libraries only when they know\n * there is no spin but they need to suppress TC warnings about shared variable modifications.\n * - It only has corresponding pointers in static library and does not have corresponding function\n * in dynamic library.\n * @see void __itt_sync_releasing(void* addr);\n */\nvoid ITTAPI __itt_fsync_releasing(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, fsync_releasing, (void *addr))\n#define __itt_fsync_releasing ITTNOTIFY_VOID(fsync_releasing)\n#define __itt_fsync_releasing_ptr ITTNOTIFY_NAME(fsync_releasing)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_fsync_releasing(addr)\n#define __itt_fsync_releasing_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_fsync_releasing_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} */\n\n/**\n * @defgroup model Modeling by Intel(R) Parallel Advisor\n * @ingroup public\n * This is the subset of itt used for modeling by Intel(R) Parallel Advisor.\n * This API is called ONLY using annotate.h, by \"Annotation\" macros\n * the user places in their sources during the parallelism modeling steps.\n *\n * site_begin/end and task_begin/end take the address of handle variables,\n * which are writeable by the API. Handles must be 0 initialized prior\n * to the first call to begin, or may cause a run-time failure.\n * The handles are initialized in a multi-thread safe way by the API if\n * the handle is 0. The commonly expected idiom is one static handle to\n * identify a site or task. If a site or task of the same name has already\n * been started during this collection, the same handle MAY be returned,\n * but is not required to be - it is unspecified if data merging is done\n * based on name. These routines also take an instance variable. Like\n * the lexical instance, these must be 0 initialized. Unlike the lexical\n * instance, this is used to track a single dynamic instance.\n *\n * API used by the Intel(R) Parallel Advisor to describe potential concurrency\n * and related activities. User-added source annotations expand to calls\n * to these procedures to enable modeling of a hypothetical concurrent\n * execution serially.\n * @{\n */\n#if !defined(_ADVISOR_ANNOTATE_H_) || defined(ANNOTATE_EXPAND_NULL)\n\ntypedef void* __itt_model_site; /*!< @brief handle for lexical site */\ntypedef void* __itt_model_site_instance; /*!< @brief handle for dynamic instance */\ntypedef void* __itt_model_task; /*!< @brief handle for lexical site */\ntypedef void* __itt_model_task_instance; /*!< @brief handle for dynamic instance */\n\n/**\n * @enum __itt_model_disable\n * @brief Enumerator for the disable methods\n */\ntypedef enum {\n __itt_model_disable_observation,\n __itt_model_disable_collection\n} __itt_model_disable;\n\n#endif /* !_ADVISOR_ANNOTATE_H_ || ANNOTATE_EXPAND_NULL */\n\n/**\n * @brief ANNOTATE_SITE_BEGIN/ANNOTATE_SITE_END support.\n *\n * site_begin/end model a potential concurrency site.\n * site instances may be recursively nested with themselves.\n * site_end exits the most recently started but unended site for the current\n * thread. The handle passed to end may be used to validate structure.\n * Instances of a site encountered on different threads concurrently\n * are considered completely distinct. If the site name for two different\n * lexical sites match, it is unspecified whether they are treated as the\n * same or different for data presentation.\n */\nvoid ITTAPI __itt_model_site_begin(__itt_model_site *site, __itt_model_site_instance *instance, const char *name);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_model_site_beginW(const wchar_t *name);\n#endif\nvoid ITTAPI __itt_model_site_beginA(const char *name);\nvoid ITTAPI __itt_model_site_beginAL(const char *name, size_t siteNameLen);\nvoid ITTAPI __itt_model_site_end (__itt_model_site *site, __itt_model_site_instance *instance);\nvoid ITTAPI __itt_model_site_end_2(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_site_begin, (__itt_model_site *site, __itt_model_site_instance *instance, const char *name))\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, model_site_beginW, (const wchar_t *name))\n#endif\nITT_STUBV(ITTAPI, void, model_site_beginA, (const char *name))\nITT_STUBV(ITTAPI, void, model_site_beginAL, (const char *name, size_t siteNameLen))\nITT_STUBV(ITTAPI, void, model_site_end, (__itt_model_site *site, __itt_model_site_instance *instance))\nITT_STUBV(ITTAPI, void, model_site_end_2, (void))\n#define __itt_model_site_begin ITTNOTIFY_VOID(model_site_begin)\n#define __itt_model_site_begin_ptr ITTNOTIFY_NAME(model_site_begin)\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_site_beginW ITTNOTIFY_VOID(model_site_beginW)\n#define __itt_model_site_beginW_ptr ITTNOTIFY_NAME(model_site_beginW)\n#endif\n#define __itt_model_site_beginA ITTNOTIFY_VOID(model_site_beginA)\n#define __itt_model_site_beginA_ptr ITTNOTIFY_NAME(model_site_beginA)\n#define __itt_model_site_beginAL ITTNOTIFY_VOID(model_site_beginAL)\n#define __itt_model_site_beginAL_ptr ITTNOTIFY_NAME(model_site_beginAL)\n#define __itt_model_site_end ITTNOTIFY_VOID(model_site_end)\n#define __itt_model_site_end_ptr ITTNOTIFY_NAME(model_site_end)\n#define __itt_model_site_end_2 ITTNOTIFY_VOID(model_site_end_2)\n#define __itt_model_site_end_2_ptr ITTNOTIFY_NAME(model_site_end_2)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_site_begin(site, instance, name)\n#define __itt_model_site_begin_ptr 0\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_site_beginW(name)\n#define __itt_model_site_beginW_ptr 0\n#endif\n#define __itt_model_site_beginA(name)\n#define __itt_model_site_beginA_ptr 0\n#define __itt_model_site_beginAL(name, siteNameLen)\n#define __itt_model_site_beginAL_ptr 0\n#define __itt_model_site_end(site, instance)\n#define __itt_model_site_end_ptr 0\n#define __itt_model_site_end_2()\n#define __itt_model_site_end_2_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_site_begin_ptr 0\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_site_beginW_ptr 0\n#endif\n#define __itt_model_site_beginA_ptr 0\n#define __itt_model_site_beginAL_ptr 0\n#define __itt_model_site_end_ptr 0\n#define __itt_model_site_end_2_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_TASK_BEGIN/ANNOTATE_TASK_END support\n *\n * task_begin/end model a potential task, which is contained within the most\n * closely enclosing dynamic site. task_end exits the most recently started\n * but unended task. The handle passed to end may be used to validate\n * structure. It is unspecified if bad dynamic nesting is detected. If it\n * is, it should be encoded in the resulting data collection. The collector\n * should not fail due to construct nesting issues, nor attempt to directly\n * indicate the problem.\n */\nvoid ITTAPI __itt_model_task_begin(__itt_model_task *task, __itt_model_task_instance *instance, const char *name);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_model_task_beginW(const wchar_t *name);\nvoid ITTAPI __itt_model_iteration_taskW(const wchar_t *name);\n#endif\nvoid ITTAPI __itt_model_task_beginA(const char *name);\nvoid ITTAPI __itt_model_task_beginAL(const char *name, size_t taskNameLen);\nvoid ITTAPI __itt_model_iteration_taskA(const char *name);\nvoid ITTAPI __itt_model_iteration_taskAL(const char *name, size_t taskNameLen);\nvoid ITTAPI __itt_model_task_end (__itt_model_task *task, __itt_model_task_instance *instance);\nvoid ITTAPI __itt_model_task_end_2(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_task_begin, (__itt_model_task *task, __itt_model_task_instance *instance, const char *name))\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, model_task_beginW, (const wchar_t *name))\nITT_STUBV(ITTAPI, void, model_iteration_taskW, (const wchar_t *name))\n#endif\nITT_STUBV(ITTAPI, void, model_task_beginA, (const char *name))\nITT_STUBV(ITTAPI, void, model_task_beginAL, (const char *name, size_t taskNameLen))\nITT_STUBV(ITTAPI, void, model_iteration_taskA, (const char *name))\nITT_STUBV(ITTAPI, void, model_iteration_taskAL, (const char *name, size_t taskNameLen))\nITT_STUBV(ITTAPI, void, model_task_end, (__itt_model_task *task, __itt_model_task_instance *instance))\nITT_STUBV(ITTAPI, void, model_task_end_2, (void))\n#define __itt_model_task_begin ITTNOTIFY_VOID(model_task_begin)\n#define __itt_model_task_begin_ptr ITTNOTIFY_NAME(model_task_begin)\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_task_beginW ITTNOTIFY_VOID(model_task_beginW)\n#define __itt_model_task_beginW_ptr ITTNOTIFY_NAME(model_task_beginW)\n#define __itt_model_iteration_taskW ITTNOTIFY_VOID(model_iteration_taskW)\n#define __itt_model_iteration_taskW_ptr ITTNOTIFY_NAME(model_iteration_taskW)\n#endif\n#define __itt_model_task_beginA ITTNOTIFY_VOID(model_task_beginA)\n#define __itt_model_task_beginA_ptr ITTNOTIFY_NAME(model_task_beginA)\n#define __itt_model_task_beginAL ITTNOTIFY_VOID(model_task_beginAL)\n#define __itt_model_task_beginAL_ptr ITTNOTIFY_NAME(model_task_beginAL)\n#define __itt_model_iteration_taskA ITTNOTIFY_VOID(model_iteration_taskA)\n#define __itt_model_iteration_taskA_ptr ITTNOTIFY_NAME(model_iteration_taskA)\n#define __itt_model_iteration_taskAL ITTNOTIFY_VOID(model_iteration_taskAL)\n#define __itt_model_iteration_taskAL_ptr ITTNOTIFY_NAME(model_iteration_taskAL)\n#define __itt_model_task_end ITTNOTIFY_VOID(model_task_end)\n#define __itt_model_task_end_ptr ITTNOTIFY_NAME(model_task_end)\n#define __itt_model_task_end_2 ITTNOTIFY_VOID(model_task_end_2)\n#define __itt_model_task_end_2_ptr ITTNOTIFY_NAME(model_task_end_2)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_task_begin(task, instance, name)\n#define __itt_model_task_begin_ptr 0\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_task_beginW(name)\n#define __itt_model_task_beginW_ptr 0\n#endif\n#define __itt_model_task_beginA(name)\n#define __itt_model_task_beginA_ptr 0\n#define __itt_model_task_beginAL(name, siteNameLen)\n#define __itt_model_task_beginAL_ptr 0\n#define __itt_model_iteration_taskA(name)\n#define __itt_model_iteration_taskA_ptr 0\n#define __itt_model_iteration_taskAL(name, siteNameLen)\n#define __itt_model_iteration_taskAL_ptr 0\n#define __itt_model_task_end(task, instance)\n#define __itt_model_task_end_ptr 0\n#define __itt_model_task_end_2()\n#define __itt_model_task_end_2_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_task_begin_ptr 0\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_model_task_beginW_ptr 0\n#endif\n#define __itt_model_task_beginA_ptr 0\n#define __itt_model_task_beginAL_ptr 0\n#define __itt_model_iteration_taskA_ptr 0\n#define __itt_model_iteration_taskAL_ptr 0\n#define __itt_model_task_end_ptr 0\n#define __itt_model_task_end_2_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_LOCK_ACQUIRE/ANNOTATE_LOCK_RELEASE support\n *\n * lock_acquire/release model a potential lock for both lockset and\n * performance modeling. Each unique address is modeled as a separate\n * lock, with invalid addresses being valid lock IDs. Specifically:\n * no storage is accessed by the API at the specified address - it is only\n * used for lock identification. Lock acquires may be self-nested and are\n * unlocked by a corresponding number of releases.\n * (These closely correspond to __itt_sync_acquired/__itt_sync_releasing,\n * but may not have identical semantics.)\n */\nvoid ITTAPI __itt_model_lock_acquire(void *lock);\nvoid ITTAPI __itt_model_lock_acquire_2(void *lock);\nvoid ITTAPI __itt_model_lock_release(void *lock);\nvoid ITTAPI __itt_model_lock_release_2(void *lock);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_lock_acquire, (void *lock))\nITT_STUBV(ITTAPI, void, model_lock_acquire_2, (void *lock))\nITT_STUBV(ITTAPI, void, model_lock_release, (void *lock))\nITT_STUBV(ITTAPI, void, model_lock_release_2, (void *lock))\n#define __itt_model_lock_acquire ITTNOTIFY_VOID(model_lock_acquire)\n#define __itt_model_lock_acquire_ptr ITTNOTIFY_NAME(model_lock_acquire)\n#define __itt_model_lock_acquire_2 ITTNOTIFY_VOID(model_lock_acquire_2)\n#define __itt_model_lock_acquire_2_ptr ITTNOTIFY_NAME(model_lock_acquire_2)\n#define __itt_model_lock_release ITTNOTIFY_VOID(model_lock_release)\n#define __itt_model_lock_release_ptr ITTNOTIFY_NAME(model_lock_release)\n#define __itt_model_lock_release_2 ITTNOTIFY_VOID(model_lock_release_2)\n#define __itt_model_lock_release_2_ptr ITTNOTIFY_NAME(model_lock_release_2)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_lock_acquire(lock)\n#define __itt_model_lock_acquire_ptr 0\n#define __itt_model_lock_acquire_2(lock)\n#define __itt_model_lock_acquire_2_ptr 0\n#define __itt_model_lock_release(lock)\n#define __itt_model_lock_release_ptr 0\n#define __itt_model_lock_release_2(lock)\n#define __itt_model_lock_release_2_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_lock_acquire_ptr 0\n#define __itt_model_lock_acquire_2_ptr 0\n#define __itt_model_lock_release_ptr 0\n#define __itt_model_lock_release_2_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_RECORD_ALLOCATION/ANNOTATE_RECORD_DEALLOCATION support\n *\n * record_allocation/deallocation describe user-defined memory allocator\n * behavior, which may be required for correctness modeling to understand\n * when storage is not expected to be actually reused across threads.\n */\nvoid ITTAPI __itt_model_record_allocation (void *addr, size_t size);\nvoid ITTAPI __itt_model_record_deallocation(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_record_allocation, (void *addr, size_t size))\nITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr))\n#define __itt_model_record_allocation ITTNOTIFY_VOID(model_record_allocation)\n#define __itt_model_record_allocation_ptr ITTNOTIFY_NAME(model_record_allocation)\n#define __itt_model_record_deallocation ITTNOTIFY_VOID(model_record_deallocation)\n#define __itt_model_record_deallocation_ptr ITTNOTIFY_NAME(model_record_deallocation)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_record_allocation(addr, size)\n#define __itt_model_record_allocation_ptr 0\n#define __itt_model_record_deallocation(addr)\n#define __itt_model_record_deallocation_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_record_allocation_ptr 0\n#define __itt_model_record_deallocation_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_INDUCTION_USES support\n *\n * Note particular storage is inductive through the end of the current site\n */\nvoid ITTAPI __itt_model_induction_uses(void* addr, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_induction_uses, (void *addr, size_t size))\n#define __itt_model_induction_uses ITTNOTIFY_VOID(model_induction_uses)\n#define __itt_model_induction_uses_ptr ITTNOTIFY_NAME(model_induction_uses)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_induction_uses(addr, size)\n#define __itt_model_induction_uses_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_induction_uses_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_REDUCTION_USES support\n *\n * Note particular storage is used for reduction through the end\n * of the current site\n */\nvoid ITTAPI __itt_model_reduction_uses(void* addr, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_reduction_uses, (void *addr, size_t size))\n#define __itt_model_reduction_uses ITTNOTIFY_VOID(model_reduction_uses)\n#define __itt_model_reduction_uses_ptr ITTNOTIFY_NAME(model_reduction_uses)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_reduction_uses(addr, size)\n#define __itt_model_reduction_uses_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_reduction_uses_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_OBSERVE_USES support\n *\n * Have correctness modeling record observations about uses of storage\n * through the end of the current site\n */\nvoid ITTAPI __itt_model_observe_uses(void* addr, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_observe_uses, (void *addr, size_t size))\n#define __itt_model_observe_uses ITTNOTIFY_VOID(model_observe_uses)\n#define __itt_model_observe_uses_ptr ITTNOTIFY_NAME(model_observe_uses)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_observe_uses(addr, size)\n#define __itt_model_observe_uses_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_observe_uses_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_CLEAR_USES support\n *\n * Clear the special handling of a piece of storage related to induction,\n * reduction or observe_uses\n */\nvoid ITTAPI __itt_model_clear_uses(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_clear_uses, (void *addr))\n#define __itt_model_clear_uses ITTNOTIFY_VOID(model_clear_uses)\n#define __itt_model_clear_uses_ptr ITTNOTIFY_NAME(model_clear_uses)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_clear_uses(addr)\n#define __itt_model_clear_uses_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_clear_uses_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief ANNOTATE_DISABLE_*_PUSH/ANNOTATE_DISABLE_*_POP support\n *\n * disable_push/disable_pop push and pop disabling based on a parameter.\n * Disabling observations stops processing of memory references during\n * correctness modeling, and all annotations that occur in the disabled\n * region. This allows description of code that is expected to be handled\n * specially during conversion to parallelism or that is not recognized\n * by tools (e.g. some kinds of synchronization operations.)\n * This mechanism causes all annotations in the disabled region, other\n * than disable_push and disable_pop, to be ignored. (For example, this\n * might validly be used to disable an entire parallel site and the contained\n * tasks and locking in it for data collection purposes.)\n * The disable for collection is a more expensive operation, but reduces\n * collector overhead significantly. This applies to BOTH correctness data\n * collection and performance data collection. For example, a site\n * containing a task might only enable data collection for the first 10\n * iterations. Both performance and correctness data should reflect this,\n * and the program should run as close to full speed as possible when\n * collection is disabled.\n */\nvoid ITTAPI __itt_model_disable_push(__itt_model_disable x);\nvoid ITTAPI __itt_model_disable_pop(void);\nvoid ITTAPI __itt_model_aggregate_task(size_t x);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, model_disable_push, (__itt_model_disable x))\nITT_STUBV(ITTAPI, void, model_disable_pop, (void))\nITT_STUBV(ITTAPI, void, model_aggregate_task, (size_t x))\n#define __itt_model_disable_push ITTNOTIFY_VOID(model_disable_push)\n#define __itt_model_disable_push_ptr ITTNOTIFY_NAME(model_disable_push)\n#define __itt_model_disable_pop ITTNOTIFY_VOID(model_disable_pop)\n#define __itt_model_disable_pop_ptr ITTNOTIFY_NAME(model_disable_pop)\n#define __itt_model_aggregate_task ITTNOTIFY_VOID(model_aggregate_task)\n#define __itt_model_aggregate_task_ptr ITTNOTIFY_NAME(model_aggregate_task)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_model_disable_push(x)\n#define __itt_model_disable_push_ptr 0\n#define __itt_model_disable_pop()\n#define __itt_model_disable_pop_ptr 0\n#define __itt_model_aggregate_task(x)\n#define __itt_model_aggregate_task_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_model_disable_push_ptr 0\n#define __itt_model_disable_pop_ptr 0\n#define __itt_model_aggregate_task_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} model group */\n\n/**\n * @defgroup heap Heap\n * @ingroup public\n * Heap group\n * @{\n */\n\ntypedef void* __itt_heap_function;\n\n/**\n * @brief Create an identification for heap function\n * @return non-zero identifier or NULL\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_heap_function ITTAPI __itt_heap_function_createA(const char* name, const char* domain);\n__itt_heap_function ITTAPI __itt_heap_function_createW(const wchar_t* name, const wchar_t* domain);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_heap_function_create __itt_heap_function_createW\n# define __itt_heap_function_create_ptr __itt_heap_function_createW_ptr\n#else\n# define __itt_heap_function_create __itt_heap_function_createA\n# define __itt_heap_function_create_ptr __itt_heap_function_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_heap_function ITTAPI __itt_heap_function_create(const char* name, const char* domain);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char* name, const char* domain))\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t* name, const wchar_t* domain))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_create, (const char* name, const char* domain))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_heap_function_createA ITTNOTIFY_DATA(heap_function_createA)\n#define __itt_heap_function_createA_ptr ITTNOTIFY_NAME(heap_function_createA)\n#define __itt_heap_function_createW ITTNOTIFY_DATA(heap_function_createW)\n#define __itt_heap_function_createW_ptr ITTNOTIFY_NAME(heap_function_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_heap_function_create ITTNOTIFY_DATA(heap_function_create)\n#define __itt_heap_function_create_ptr ITTNOTIFY_NAME(heap_function_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_heap_function_createA(name, domain) (__itt_heap_function)0\n#define __itt_heap_function_createA_ptr 0\n#define __itt_heap_function_createW(name, domain) (__itt_heap_function)0\n#define __itt_heap_function_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_heap_function_create(name, domain) (__itt_heap_function)0\n#define __itt_heap_function_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_heap_function_createA_ptr 0\n#define __itt_heap_function_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_heap_function_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an allocation begin occurrence.\n */\nvoid ITTAPI __itt_heap_allocate_begin(__itt_heap_function h, size_t size, int initialized);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_allocate_begin, (__itt_heap_function h, size_t size, int initialized))\n#define __itt_heap_allocate_begin ITTNOTIFY_VOID(heap_allocate_begin)\n#define __itt_heap_allocate_begin_ptr ITTNOTIFY_NAME(heap_allocate_begin)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_allocate_begin(h, size, initialized)\n#define __itt_heap_allocate_begin_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_allocate_begin_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an allocation end occurrence.\n */\nvoid ITTAPI __itt_heap_allocate_end(__itt_heap_function h, void** addr, size_t size, int initialized);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, size_t size, int initialized))\n#define __itt_heap_allocate_end ITTNOTIFY_VOID(heap_allocate_end)\n#define __itt_heap_allocate_end_ptr ITTNOTIFY_NAME(heap_allocate_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_allocate_end(h, addr, size, initialized)\n#define __itt_heap_allocate_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_allocate_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record a free begin occurrence.\n */\nvoid ITTAPI __itt_heap_free_begin(__itt_heap_function h, void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr))\n#define __itt_heap_free_begin ITTNOTIFY_VOID(heap_free_begin)\n#define __itt_heap_free_begin_ptr ITTNOTIFY_NAME(heap_free_begin)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_free_begin(h, addr)\n#define __itt_heap_free_begin_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_free_begin_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record a free end occurrence.\n */\nvoid ITTAPI __itt_heap_free_end(__itt_heap_function h, void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr))\n#define __itt_heap_free_end ITTNOTIFY_VOID(heap_free_end)\n#define __itt_heap_free_end_ptr ITTNOTIFY_NAME(heap_free_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_free_end(h, addr)\n#define __itt_heap_free_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_free_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record a reallocation begin occurrence.\n */\nvoid ITTAPI __itt_heap_reallocate_begin(__itt_heap_function h, void* addr, size_t new_size, int initialized);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* addr, size_t new_size, int initialized))\n#define __itt_heap_reallocate_begin ITTNOTIFY_VOID(heap_reallocate_begin)\n#define __itt_heap_reallocate_begin_ptr ITTNOTIFY_NAME(heap_reallocate_begin)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_reallocate_begin(h, addr, new_size, initialized)\n#define __itt_heap_reallocate_begin_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_reallocate_begin_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record a reallocation end occurrence.\n */\nvoid ITTAPI __itt_heap_reallocate_end(__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_reallocate_end, (__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized))\n#define __itt_heap_reallocate_end ITTNOTIFY_VOID(heap_reallocate_end)\n#define __itt_heap_reallocate_end_ptr ITTNOTIFY_NAME(heap_reallocate_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_reallocate_end(h, addr, new_addr, new_size, initialized)\n#define __itt_heap_reallocate_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_reallocate_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief internal access begin */\nvoid ITTAPI __itt_heap_internal_access_begin(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void))\n#define __itt_heap_internal_access_begin ITTNOTIFY_VOID(heap_internal_access_begin)\n#define __itt_heap_internal_access_begin_ptr ITTNOTIFY_NAME(heap_internal_access_begin)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_internal_access_begin()\n#define __itt_heap_internal_access_begin_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_internal_access_begin_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief internal access end */\nvoid ITTAPI __itt_heap_internal_access_end(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_internal_access_end, (void))\n#define __itt_heap_internal_access_end ITTNOTIFY_VOID(heap_internal_access_end)\n#define __itt_heap_internal_access_end_ptr ITTNOTIFY_NAME(heap_internal_access_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_internal_access_end()\n#define __itt_heap_internal_access_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_internal_access_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief record memory growth begin */\nvoid ITTAPI __itt_heap_record_memory_growth_begin(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void))\n#define __itt_heap_record_memory_growth_begin ITTNOTIFY_VOID(heap_record_memory_growth_begin)\n#define __itt_heap_record_memory_growth_begin_ptr ITTNOTIFY_NAME(heap_record_memory_growth_begin)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_record_memory_growth_begin()\n#define __itt_heap_record_memory_growth_begin_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_record_memory_growth_begin_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief record memory growth end */\nvoid ITTAPI __itt_heap_record_memory_growth_end(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void))\n#define __itt_heap_record_memory_growth_end ITTNOTIFY_VOID(heap_record_memory_growth_end)\n#define __itt_heap_record_memory_growth_end_ptr ITTNOTIFY_NAME(heap_record_memory_growth_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_record_memory_growth_end()\n#define __itt_heap_record_memory_growth_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_record_memory_growth_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Specify the type of heap detection/reporting to modify.\n */\n/**\n * @hideinitializer\n * @brief Report on memory leaks.\n */\n#define __itt_heap_leaks 0x00000001\n\n/**\n * @hideinitializer\n * @brief Report on memory growth.\n */\n#define __itt_heap_growth 0x00000002\n\n\n/** @brief heap reset detection */\nvoid ITTAPI __itt_heap_reset_detection(unsigned int reset_mask);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask))\n#define __itt_heap_reset_detection ITTNOTIFY_VOID(heap_reset_detection)\n#define __itt_heap_reset_detection_ptr ITTNOTIFY_NAME(heap_reset_detection)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_reset_detection()\n#define __itt_heap_reset_detection_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_reset_detection_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief report */\nvoid ITTAPI __itt_heap_record(unsigned int record_mask);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, heap_record, (unsigned int record_mask))\n#define __itt_heap_record ITTNOTIFY_VOID(heap_record)\n#define __itt_heap_record_ptr ITTNOTIFY_NAME(heap_record)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_heap_record()\n#define __itt_heap_record_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_heap_record_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @} heap group */\n/** @endcond */\n/* ========================================================================== */\n\n/**\n * @defgroup domains Domains\n * @ingroup public\n * Domains group\n * @{\n */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_domain\n{\n volatile int flags; /*!< Zero if disabled, non-zero if enabled. The meaning of different non-zero values is reserved to the runtime */\n const char* nameA; /*!< Copy of original name in ASCII. */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* nameW; /*!< Copy of original name in UNICODE. */\n#else /* UNICODE || _UNICODE */\n void* nameW;\n#endif /* UNICODE || _UNICODE */\n int extra1; /*!< Reserved to the runtime */\n void* extra2; /*!< Reserved to the runtime */\n struct ___itt_domain* next;\n} __itt_domain;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @ingroup domains\n * @brief Create a domain.\n * Create domain using some domain name: the URI naming style is recommended.\n * Because the set of domains is expected to be static over the application's\n * execution time, there is no mechanism to destroy a domain.\n * Any domain can be accessed by any thread in the process, regardless of\n * which thread created the domain. This call is thread-safe.\n * @param[in] name name of domain\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_domain* ITTAPI __itt_domain_createA(const char *name);\n__itt_domain* ITTAPI __itt_domain_createW(const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_domain_create __itt_domain_createW\n# define __itt_domain_create_ptr __itt_domain_createW_ptr\n#else /* UNICODE */\n# define __itt_domain_create __itt_domain_createA\n# define __itt_domain_create_ptr __itt_domain_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_domain* ITTAPI __itt_domain_create(const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char *name))\nITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_domain_createA ITTNOTIFY_DATA(domain_createA)\n#define __itt_domain_createA_ptr ITTNOTIFY_NAME(domain_createA)\n#define __itt_domain_createW ITTNOTIFY_DATA(domain_createW)\n#define __itt_domain_createW_ptr ITTNOTIFY_NAME(domain_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_domain_create ITTNOTIFY_DATA(domain_create)\n#define __itt_domain_create_ptr ITTNOTIFY_NAME(domain_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_domain_createA(name) (__itt_domain*)0\n#define __itt_domain_createA_ptr 0\n#define __itt_domain_createW(name) (__itt_domain*)0\n#define __itt_domain_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_domain_create(name) (__itt_domain*)0\n#define __itt_domain_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_domain_createA_ptr 0\n#define __itt_domain_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_domain_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} domains group */\n\n/**\n * @defgroup ids IDs\n * @ingroup public\n * IDs group\n * @{\n */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_id\n{\n unsigned long long d1, d2, d3;\n} __itt_id;\n\n#pragma pack(pop)\n/** @endcond */\n\nstatic const __itt_id __itt_null = { 0, 0, 0 };\n\n/**\n * @ingroup ids\n * @brief A convenience function is provided to create an ID without domain control.\n * @brief This is a convenience function to initialize an __itt_id structure. This function\n * does not affect the collector runtime in any way. After you make the ID with this\n * function, you still must create it with the __itt_id_create function before using the ID\n * to identify a named entity.\n * @param[in] addr The address of object; high QWORD of the ID value.\n * @param[in] extra The extra data to unique identify object; low QWORD of the ID value.\n */\n\nITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) ITT_INLINE_ATTRIBUTE;\nITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra)\n{\n __itt_id id = __itt_null;\n id.d1 = (unsigned long long)((uintptr_t)addr);\n id.d2 = (unsigned long long)extra;\n id.d3 = (unsigned long long)0; /* Reserved. Must be zero */\n return id;\n}\n\n/**\n * @ingroup ids\n * @brief Create an instance of identifier.\n * This establishes the beginning of the lifetime of an instance of\n * the given ID in the trace. Once this lifetime starts, the ID\n * can be used to tag named entity instances in calls such as\n * __itt_task_begin, and to specify relationships among\n * identified named entity instances, using the \\ref relations APIs.\n * Instance IDs are not domain specific!\n * @param[in] domain The domain controlling the execution of this call.\n * @param[in] id The ID to create.\n */\nvoid ITTAPI __itt_id_create(const __itt_domain *domain, __itt_id id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id))\n#define __itt_id_create(d,x) ITTNOTIFY_VOID_D1(id_create,d,x)\n#define __itt_id_create_ptr ITTNOTIFY_NAME(id_create)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_id_create(domain,id)\n#define __itt_id_create_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_id_create_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup ids\n * @brief Destroy an instance of identifier.\n * This ends the lifetime of the current instance of the given ID value in the trace.\n * Any relationships that are established after this lifetime ends are invalid.\n * This call must be performed before the given ID value can be reused for a different\n * named entity instance.\n * @param[in] domain The domain controlling the execution of this call.\n * @param[in] id The ID to destroy.\n */\nvoid ITTAPI __itt_id_destroy(const __itt_domain *domain, __itt_id id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id))\n#define __itt_id_destroy(d,x) ITTNOTIFY_VOID_D1(id_destroy,d,x)\n#define __itt_id_destroy_ptr ITTNOTIFY_NAME(id_destroy)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_id_destroy(domain,id)\n#define __itt_id_destroy_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_id_destroy_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} ids group */\n\n/**\n * @defgroup handless String Handles\n * @ingroup public\n * String Handles group\n * @{\n */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_string_handle\n{\n const char* strA; /*!< Copy of original string in ASCII. */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* strW; /*!< Copy of original string in UNICODE. */\n#else /* UNICODE || _UNICODE */\n void* strW;\n#endif /* UNICODE || _UNICODE */\n int extra1; /*!< Reserved. Must be zero */\n void* extra2; /*!< Reserved. Must be zero */\n struct ___itt_string_handle* next;\n} __itt_string_handle;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @ingroup handles\n * @brief Create a string handle.\n * Create and return handle value that can be associated with a string.\n * Consecutive calls to __itt_string_handle_create with the same name\n * return the same value. Because the set of string handles is expected to remain\n * static during the application's execution time, there is no mechanism to destroy a string handle.\n * Any string handle can be accessed by any thread in the process, regardless of which thread created\n * the string handle. This call is thread-safe.\n * @param[in] name The input string\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_string_handle* ITTAPI __itt_string_handle_createA(const char *name);\n__itt_string_handle* ITTAPI __itt_string_handle_createW(const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_string_handle_create __itt_string_handle_createW\n# define __itt_string_handle_create_ptr __itt_string_handle_createW_ptr\n#else /* UNICODE */\n# define __itt_string_handle_create __itt_string_handle_createA\n# define __itt_string_handle_create_ptr __itt_string_handle_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_string_handle* ITTAPI __itt_string_handle_create(const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name))\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_string_handle_createA ITTNOTIFY_DATA(string_handle_createA)\n#define __itt_string_handle_createA_ptr ITTNOTIFY_NAME(string_handle_createA)\n#define __itt_string_handle_createW ITTNOTIFY_DATA(string_handle_createW)\n#define __itt_string_handle_createW_ptr ITTNOTIFY_NAME(string_handle_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_string_handle_create ITTNOTIFY_DATA(string_handle_create)\n#define __itt_string_handle_create_ptr ITTNOTIFY_NAME(string_handle_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_string_handle_createA(name) (__itt_string_handle*)0\n#define __itt_string_handle_createA_ptr 0\n#define __itt_string_handle_createW(name) (__itt_string_handle*)0\n#define __itt_string_handle_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_string_handle_create(name) (__itt_string_handle*)0\n#define __itt_string_handle_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_string_handle_createA_ptr 0\n#define __itt_string_handle_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_string_handle_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} handles group */\n\n/** @cond exclude_from_documentation */\ntypedef unsigned long long __itt_timestamp;\n/** @endcond */\n\n#define __itt_timestamp_none ((__itt_timestamp)-1LL)\n\n/** @cond exclude_from_gpa_documentation */\n\n/**\n * @ingroup timestamps\n * @brief Return timestamp corresponding to the current moment.\n * This returns the timestamp in the format that is the most relevant for the current\n * host or platform (RDTSC, QPC, and others). You can use the \"<\" operator to\n * compare __itt_timestamp values.\n */\n__itt_timestamp ITTAPI __itt_get_timestamp(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void))\n#define __itt_get_timestamp ITTNOTIFY_DATA(get_timestamp)\n#define __itt_get_timestamp_ptr ITTNOTIFY_NAME(get_timestamp)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_get_timestamp()\n#define __itt_get_timestamp_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_get_timestamp_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} timestamps */\n/** @endcond */\n\n/** @cond exclude_from_gpa_documentation */\n\n/**\n * @defgroup regions Regions\n * @ingroup public\n * Regions group\n * @{\n */\n/**\n * @ingroup regions\n * @brief Begin of region instance.\n * Successive calls to __itt_region_begin with the same ID are ignored\n * until a call to __itt_region_end with the same ID\n * @param[in] domain The domain for this region instance\n * @param[in] id The instance ID for this region instance. Must not be __itt_null\n * @param[in] parentid The instance ID for the parent of this region instance, or __itt_null\n * @param[in] name The name of this region\n */\nvoid ITTAPI __itt_region_begin(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name);\n\n/**\n * @ingroup regions\n * @brief End of region instance.\n * The first call to __itt_region_end with a given ID ends the\n * region. Successive calls with the same ID are ignored, as are\n * calls that do not have a matching __itt_region_begin call.\n * @param[in] domain The domain for this region instance\n * @param[in] id The instance ID for this region instance\n */\nvoid ITTAPI __itt_region_end(const __itt_domain *domain, __itt_id id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, region_end, (const __itt_domain *domain, __itt_id id))\n#define __itt_region_begin(d,x,y,z) ITTNOTIFY_VOID_D3(region_begin,d,x,y,z)\n#define __itt_region_begin_ptr ITTNOTIFY_NAME(region_begin)\n#define __itt_region_end(d,x) ITTNOTIFY_VOID_D1(region_end,d,x)\n#define __itt_region_end_ptr ITTNOTIFY_NAME(region_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_region_begin(d,x,y,z)\n#define __itt_region_begin_ptr 0\n#define __itt_region_end(d,x)\n#define __itt_region_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_region_begin_ptr 0\n#define __itt_region_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} regions group */\n\n/**\n * @defgroup frames Frames\n * @ingroup public\n * Frames are similar to regions, but are intended to be easier to use and to implement.\n * In particular:\n * - Frames always represent periods of elapsed time\n * - By default, frames have no nesting relationships\n * @{\n */\n\n/**\n * @ingroup frames\n * @brief Begin a frame instance.\n * Successive calls to __itt_frame_begin with the\n * same ID are ignored until a call to __itt_frame_end with the same ID.\n * @param[in] domain The domain for this frame instance\n * @param[in] id The instance ID for this frame instance or NULL\n */\nvoid ITTAPI __itt_frame_begin_v3(const __itt_domain *domain, __itt_id *id);\n\n/**\n * @ingroup frames\n * @brief End a frame instance.\n * The first call to __itt_frame_end with a given ID\n * ends the frame. Successive calls with the same ID are ignored, as are\n * calls that do not have a matching __itt_frame_begin call.\n * @param[in] domain The domain for this frame instance\n * @param[in] id The instance ID for this frame instance or NULL for current\n */\nvoid ITTAPI __itt_frame_end_v3(const __itt_domain *domain, __itt_id *id);\n\n/**\n * @ingroup frames\n * @brief Submits a frame instance.\n * Successive calls to __itt_frame_begin or __itt_frame_submit with the\n * same ID are ignored until a call to __itt_frame_end or __itt_frame_submit\n * with the same ID.\n * Passing special __itt_timestamp_none value as \"end\" argument means\n * take the current timestamp as the end timestamp.\n * @param[in] domain The domain for this frame instance\n * @param[in] id The instance ID for this frame instance or NULL\n * @param[in] begin Timestamp of the beginning of the frame\n * @param[in] end Timestamp of the end of the frame\n */\nvoid ITTAPI __itt_frame_submit_v3(const __itt_domain *domain, __itt_id *id,\n __itt_timestamp begin, __itt_timestamp end);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, frame_begin_v3, (const __itt_domain *domain, __itt_id *id))\nITT_STUBV(ITTAPI, void, frame_end_v3, (const __itt_domain *domain, __itt_id *id))\nITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end))\n#define __itt_frame_begin_v3(d,x) ITTNOTIFY_VOID_D1(frame_begin_v3,d,x)\n#define __itt_frame_begin_v3_ptr ITTNOTIFY_NAME(frame_begin_v3)\n#define __itt_frame_end_v3(d,x) ITTNOTIFY_VOID_D1(frame_end_v3,d,x)\n#define __itt_frame_end_v3_ptr ITTNOTIFY_NAME(frame_end_v3)\n#define __itt_frame_submit_v3(d,x,b,e) ITTNOTIFY_VOID_D3(frame_submit_v3,d,x,b,e)\n#define __itt_frame_submit_v3_ptr ITTNOTIFY_NAME(frame_submit_v3)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_frame_begin_v3(domain,id)\n#define __itt_frame_begin_v3_ptr 0\n#define __itt_frame_end_v3(domain,id)\n#define __itt_frame_end_v3_ptr 0\n#define __itt_frame_submit_v3(domain,id,begin,end)\n#define __itt_frame_submit_v3_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_frame_begin_v3_ptr 0\n#define __itt_frame_end_v3_ptr 0\n#define __itt_frame_submit_v3_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} frames group */\n/** @endcond */\n\n/**\n * @defgroup taskgroup Task Group\n * @ingroup public\n * Task Group\n * @{\n */\n/**\n * @ingroup task_groups\n * @brief Denotes a task_group instance.\n * Successive calls to __itt_task_group with the same ID are ignored.\n * @param[in] domain The domain for this task_group instance\n * @param[in] id The instance ID for this task_group instance. Must not be __itt_null.\n * @param[in] parentid The instance ID for the parent of this task_group instance, or __itt_null.\n * @param[in] name The name of this task_group\n */\nvoid ITTAPI __itt_task_group(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, task_group, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name))\n#define __itt_task_group(d,x,y,z) ITTNOTIFY_VOID_D3(task_group,d,x,y,z)\n#define __itt_task_group_ptr ITTNOTIFY_NAME(task_group)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_task_group(d,x,y,z)\n#define __itt_task_group_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_task_group_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} taskgroup group */\n\n/**\n * @defgroup tasks Tasks\n * @ingroup public\n * A task instance represents a piece of work performed by a particular\n * thread for a period of time. A call to __itt_task_begin creates a\n * task instance. This becomes the current instance for that task on that\n * thread. A following call to __itt_task_end on the same thread ends the\n * instance. There may be multiple simultaneous instances of tasks with the\n * same name on different threads. If an ID is specified, the task instance\n * receives that ID. Nested tasks are allowed.\n *\n * Note: The task is defined by the bracketing of __itt_task_begin and\n * __itt_task_end on the same thread. If some scheduling mechanism causes\n * task switching (the thread executes a different user task) or task\n * switching (the user task switches to a different thread) then this breaks\n * the notion of current instance. Additional API calls are required to\n * deal with that possibility.\n * @{\n */\n\n/**\n * @ingroup tasks\n * @brief Begin a task instance.\n * @param[in] domain The domain for this task\n * @param[in] taskid The instance ID for this task instance, or __itt_null\n * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null\n * @param[in] name The name of this task\n */\nvoid ITTAPI __itt_task_begin(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name);\n\n/**\n * @ingroup tasks\n * @brief Begin a task instance.\n * @param[in] domain The domain for this task\n * @param[in] taskid The identifier for this task instance (may be 0)\n * @param[in] parentid The parent of this task (may be 0)\n * @param[in] fn The pointer to the function you are tracing\n */\nvoid ITTAPI __itt_task_begin_fn(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, void* fn);\n\n/**\n * @ingroup tasks\n * @brief End the current task instance.\n * @param[in] domain The domain for this task\n */\nvoid ITTAPI __itt_task_end(const __itt_domain *domain);\n\n/**\n * @ingroup tasks\n * @brief Begin an overlapped task instance.\n * @param[in] domain The domain for this task.\n * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null.\n * @param[in] parentid The parent of this task, or __itt_null.\n * @param[in] name The name of this task.\n */\nvoid ITTAPI __itt_task_begin_overlapped(const __itt_domain* domain, __itt_id taskid, __itt_id parentid, __itt_string_handle* name);\n\n/**\n * @ingroup tasks\n * @brief End an overlapped task instance.\n * @param[in] domain The domain for this task\n * @param[in] taskid Explicit ID of finished task\n */\nvoid ITTAPI __itt_task_end_overlapped(const __itt_domain *domain, __itt_id taskid);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, task_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parentid, void* fn))\nITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain))\nITT_STUBV(ITTAPI, void, task_begin_overlapped, (const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id taskid))\n#define __itt_task_begin(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin,d,x,y,z)\n#define __itt_task_begin_ptr ITTNOTIFY_NAME(task_begin)\n#define __itt_task_begin_fn(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_fn,d,x,y,z)\n#define __itt_task_begin_fn_ptr ITTNOTIFY_NAME(task_begin_fn)\n#define __itt_task_end(d) ITTNOTIFY_VOID_D0(task_end,d)\n#define __itt_task_end_ptr ITTNOTIFY_NAME(task_end)\n#define __itt_task_begin_overlapped(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_overlapped,d,x,y,z)\n#define __itt_task_begin_overlapped_ptr ITTNOTIFY_NAME(task_begin_overlapped)\n#define __itt_task_end_overlapped(d,x) ITTNOTIFY_VOID_D1(task_end_overlapped,d,x)\n#define __itt_task_end_overlapped_ptr ITTNOTIFY_NAME(task_end_overlapped)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_task_begin(domain,id,parentid,name)\n#define __itt_task_begin_ptr 0\n#define __itt_task_begin_fn(domain,id,parentid,fn)\n#define __itt_task_begin_fn_ptr 0\n#define __itt_task_end(domain)\n#define __itt_task_end_ptr 0\n#define __itt_task_begin_overlapped(domain,taskid,parentid,name)\n#define __itt_task_begin_overlapped_ptr 0\n#define __itt_task_end_overlapped(domain,taskid)\n#define __itt_task_end_overlapped_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_task_begin_ptr 0\n#define __itt_task_begin_fn_ptr 0\n#define __itt_task_end_ptr 0\n#define __itt_task_begin_overlapped_ptr 0\n#define __itt_task_end_overlapped_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} tasks group */\n\n\n/**\n * @defgroup markers Markers\n * Markers represent a single discrete event in time. Markers have a scope,\n * described by an enumerated type __itt_scope. Markers are created by\n * the API call __itt_marker. A marker instance can be given an ID for use in\n * adding metadata.\n * @{\n */\n\n/**\n * @brief Describes the scope of an event object in the trace.\n */\ntypedef enum\n{\n __itt_scope_unknown = 0,\n __itt_scope_global,\n __itt_scope_track_group,\n __itt_scope_track,\n __itt_scope_task,\n __itt_scope_marker\n} __itt_scope;\n\n/** @cond exclude_from_documentation */\n#define __itt_marker_scope_unknown __itt_scope_unknown\n#define __itt_marker_scope_global __itt_scope_global\n#define __itt_marker_scope_process __itt_scope_track_group\n#define __itt_marker_scope_thread __itt_scope_track\n#define __itt_marker_scope_task __itt_scope_task\n/** @endcond */\n\n/**\n * @ingroup markers\n * @brief Create a marker instance\n * @param[in] domain The domain for this marker\n * @param[in] id The instance ID for this marker or __itt_null\n * @param[in] name The name for this marker\n * @param[in] scope The scope for this marker\n */\nvoid ITTAPI __itt_marker(const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope))\n#define __itt_marker(d,x,y,z) ITTNOTIFY_VOID_D3(marker,d,x,y,z)\n#define __itt_marker_ptr ITTNOTIFY_NAME(marker)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_marker(domain,id,name,scope)\n#define __itt_marker_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_marker_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} markers group */\n\n/**\n * @defgroup metadata Metadata\n * The metadata API is used to attach extra information to named\n * entities. Metadata can be attached to an identified named entity by ID,\n * or to the current entity (which is always a task).\n *\n * Conceptually metadata has a type (what kind of metadata), a key (the\n * name of the metadata), and a value (the actual data). The encoding of\n * the value depends on the type of the metadata.\n *\n * The type of metadata is specified by an enumerated type __itt_metdata_type.\n * @{\n */\n\n/**\n * @ingroup parameters\n * @brief describes the type of metadata\n */\ntypedef enum {\n __itt_metadata_unknown = 0,\n __itt_metadata_u64, /**< Unsigned 64-bit integer */\n __itt_metadata_s64, /**< Signed 64-bit integer */\n __itt_metadata_u32, /**< Unsigned 32-bit integer */\n __itt_metadata_s32, /**< Signed 32-bit integer */\n __itt_metadata_u16, /**< Unsigned 16-bit integer */\n __itt_metadata_s16, /**< Signed 16-bit integer */\n __itt_metadata_float, /**< Signed 32-bit floating-point */\n __itt_metadata_double /**< SIgned 64-bit floating-point */\n} __itt_metadata_type;\n\n/**\n * @ingroup parameters\n * @brief Add metadata to an instance of a named entity.\n * @param[in] domain The domain controlling the call\n * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task\n * @param[in] key The name of the metadata\n * @param[in] type The type of the metadata\n * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added.\n * @param[in] data The metadata itself\n*/\nvoid ITTAPI __itt_metadata_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data))\n#define __itt_metadata_add(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add,d,x,y,z,a,b)\n#define __itt_metadata_add_ptr ITTNOTIFY_NAME(metadata_add)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_metadata_add(d,x,y,z,a,b)\n#define __itt_metadata_add_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_metadata_add_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup parameters\n * @brief Add string metadata to an instance of a named entity.\n * @param[in] domain The domain controlling the call\n * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task\n * @param[in] key The name of the metadata\n * @param[in] data The metadata itself\n * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_metadata_str_addA(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length);\nvoid ITTAPI __itt_metadata_str_addW(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_metadata_str_add __itt_metadata_str_addW\n# define __itt_metadata_str_add_ptr __itt_metadata_str_addW_ptr\n#else /* UNICODE */\n# define __itt_metadata_str_add __itt_metadata_str_addA\n# define __itt_metadata_str_add_ptr __itt_metadata_str_addA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_metadata_str_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length);\n#endif\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length))\nITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_addA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addA,d,x,y,z,a)\n#define __itt_metadata_str_addA_ptr ITTNOTIFY_NAME(metadata_str_addA)\n#define __itt_metadata_str_addW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addW,d,x,y,z,a)\n#define __itt_metadata_str_addW_ptr ITTNOTIFY_NAME(metadata_str_addW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add,d,x,y,z,a)\n#define __itt_metadata_str_add_ptr ITTNOTIFY_NAME(metadata_str_add)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_addA(d,x,y,z,a)\n#define __itt_metadata_str_addA_ptr 0\n#define __itt_metadata_str_addW(d,x,y,z,a)\n#define __itt_metadata_str_addW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add(d,x,y,z,a)\n#define __itt_metadata_str_add_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_addA_ptr 0\n#define __itt_metadata_str_addW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup parameters\n * @brief Add metadata to an instance of a named entity.\n * @param[in] domain The domain controlling the call\n * @param[in] scope The scope of the instance to which the metadata is to be added\n\n * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task\n\n * @param[in] key The name of the metadata\n * @param[in] type The type of the metadata\n * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added.\n * @param[in] data The metadata itself\n*/\nvoid ITTAPI __itt_metadata_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data))\n#define __itt_metadata_add_with_scope(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add_with_scope,d,x,y,z,a,b)\n#define __itt_metadata_add_with_scope_ptr ITTNOTIFY_NAME(metadata_add_with_scope)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_metadata_add_with_scope(d,x,y,z,a,b)\n#define __itt_metadata_add_with_scope_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_metadata_add_with_scope_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup parameters\n * @brief Add string metadata to an instance of a named entity.\n * @param[in] domain The domain controlling the call\n * @param[in] scope The scope of the instance to which the metadata is to be added\n\n * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task\n\n * @param[in] key The name of the metadata\n * @param[in] data The metadata itself\n * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_metadata_str_add_with_scopeA(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length);\nvoid ITTAPI __itt_metadata_str_add_with_scopeW(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeW\n# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeW_ptr\n#else /* UNICODE */\n# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeA\n# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_metadata_str_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length);\n#endif\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length))\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeA,d,x,y,z,a)\n#define __itt_metadata_str_add_with_scopeA_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeA)\n#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeW,d,x,y,z,a)\n#define __itt_metadata_str_add_with_scopeW_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add_with_scope(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scope,d,x,y,z,a)\n#define __itt_metadata_str_add_with_scope_ptr ITTNOTIFY_NAME(metadata_str_add_with_scope)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a)\n#define __itt_metadata_str_add_with_scopeA_ptr 0\n#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a)\n#define __itt_metadata_str_add_with_scopeW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add_with_scope(d,x,y,z,a)\n#define __itt_metadata_str_add_with_scope_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_metadata_str_add_with_scopeA_ptr 0\n#define __itt_metadata_str_add_with_scopeW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_metadata_str_add_with_scope_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @} metadata group */\n\n/**\n * @defgroup relations Relations\n * Instances of named entities can be explicitly associated with other\n * instances using instance IDs and the relationship API calls.\n *\n * @{\n */\n\n/**\n * @ingroup relations\n * @brief The kind of relation between two instances is specified by the enumerated type __itt_relation.\n * Relations between instances can be added with an API call. The relation\n * API uses instance IDs. Relations can be added before or after the actual\n * instances are created and persist independently of the instances. This\n * is the motivation for having different lifetimes for instance IDs and\n * the actual instances.\n */\ntypedef enum\n{\n __itt_relation_is_unknown = 0,\n __itt_relation_is_dependent_on, /**< \"A is dependent on B\" means that A cannot start until B completes */\n __itt_relation_is_sibling_of, /**< \"A is sibling of B\" means that A and B were created as a group */\n __itt_relation_is_parent_of, /**< \"A is parent of B\" means that A created B */\n __itt_relation_is_continuation_of, /**< \"A is continuation of B\" means that A assumes the dependencies of B */\n __itt_relation_is_child_of, /**< \"A is child of B\" means that A was created by B (inverse of is_parent_of) */\n __itt_relation_is_continued_by, /**< \"A is continued by B\" means that B assumes the dependencies of A (inverse of is_continuation_of) */\n __itt_relation_is_predecessor_to /**< \"A is predecessor to B\" means that B cannot start until A completes (inverse of is_dependent_on) */\n} __itt_relation;\n\n/**\n * @ingroup relations\n * @brief Add a relation to the current task instance.\n * The current task instance is the head of the relation.\n * @param[in] domain The domain controlling this call\n * @param[in] relation The kind of relation\n * @param[in] tail The ID for the tail of the relation\n */\nvoid ITTAPI __itt_relation_add_to_current(const __itt_domain *domain, __itt_relation relation, __itt_id tail);\n\n/**\n * @ingroup relations\n * @brief Add a relation between two instance identifiers.\n * @param[in] domain The domain controlling this call\n * @param[in] head The ID for the head of the relation\n * @param[in] relation The kind of relation\n * @param[in] tail The ID for the tail of the relation\n */\nvoid ITTAPI __itt_relation_add(const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail))\nITT_STUBV(ITTAPI, void, relation_add, (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail))\n#define __itt_relation_add_to_current(d,x,y) ITTNOTIFY_VOID_D2(relation_add_to_current,d,x,y)\n#define __itt_relation_add_to_current_ptr ITTNOTIFY_NAME(relation_add_to_current)\n#define __itt_relation_add(d,x,y,z) ITTNOTIFY_VOID_D3(relation_add,d,x,y,z)\n#define __itt_relation_add_ptr ITTNOTIFY_NAME(relation_add)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_relation_add_to_current(d,x,y)\n#define __itt_relation_add_to_current_ptr 0\n#define __itt_relation_add(d,x,y,z)\n#define __itt_relation_add_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_relation_add_to_current_ptr 0\n#define __itt_relation_add_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} relations group */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_clock_info\n{\n unsigned long long clock_freq; /*!< Clock domain frequency */\n unsigned long long clock_base; /*!< Clock domain base timestamp */\n} __itt_clock_info;\n\n#pragma pack(pop)\n/** @endcond */\n\n/** @cond exclude_from_documentation */\ntypedef void (ITTAPI *__itt_get_clock_info_fn)(__itt_clock_info* clock_info, void* data);\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_clock_domain\n{\n __itt_clock_info info; /*!< Most recent clock domain info */\n __itt_get_clock_info_fn fn; /*!< Callback function pointer */\n void* fn_data; /*!< Input argument for the callback function */\n int extra1; /*!< Reserved. Must be zero */\n void* extra2; /*!< Reserved. Must be zero */\n struct ___itt_clock_domain* next;\n} __itt_clock_domain;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @ingroup clockdomains\n * @brief Create a clock domain.\n * Certain applications require the capability to trace their application using\n * a clock domain different than the CPU, for instance the instrumentation of events\n * that occur on a GPU.\n * Because the set of domains is expected to be static over the application's execution time,\n * there is no mechanism to destroy a domain.\n * Any domain can be accessed by any thread in the process, regardless of which thread created\n * the domain. This call is thread-safe.\n * @param[in] fn A pointer to a callback function which retrieves alternative CPU timestamps\n * @param[in] fn_data Argument for a callback function; may be NULL\n */\n__itt_clock_domain* ITTAPI __itt_clock_domain_create(__itt_get_clock_info_fn fn, void* fn_data);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data))\n#define __itt_clock_domain_create ITTNOTIFY_DATA(clock_domain_create)\n#define __itt_clock_domain_create_ptr ITTNOTIFY_NAME(clock_domain_create)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_clock_domain_create(fn,fn_data) (__itt_clock_domain*)0\n#define __itt_clock_domain_create_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_clock_domain_create_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup clockdomains\n * @brief Recalculate clock domains frequencies and clock base timestamps.\n */\nvoid ITTAPI __itt_clock_domain_reset(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, clock_domain_reset, (void))\n#define __itt_clock_domain_reset ITTNOTIFY_VOID(clock_domain_reset)\n#define __itt_clock_domain_reset_ptr ITTNOTIFY_NAME(clock_domain_reset)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_clock_domain_reset()\n#define __itt_clock_domain_reset_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_clock_domain_reset_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup clockdomain\n * @brief Create an instance of identifier. This establishes the beginning of the lifetime of\n * an instance of the given ID in the trace. Once this lifetime starts, the ID can be used to\n * tag named entity instances in calls such as __itt_task_begin, and to specify relationships among\n * identified named entity instances, using the \\ref relations APIs.\n * @param[in] domain The domain controlling the execution of this call.\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] id The ID to create.\n */\nvoid ITTAPI __itt_id_create_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id);\n\n/**\n * @ingroup clockdomain\n * @brief Destroy an instance of identifier. This ends the lifetime of the current instance of the\n * given ID value in the trace. Any relationships that are established after this lifetime ends are\n * invalid. This call must be performed before the given ID value can be reused for a different\n * named entity instance.\n * @param[in] domain The domain controlling the execution of this call.\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] id The ID to destroy.\n */\nvoid ITTAPI __itt_id_destroy_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, id_create_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id))\nITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id))\n#define __itt_id_create_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_create_ex,d,x,y,z)\n#define __itt_id_create_ex_ptr ITTNOTIFY_NAME(id_create_ex)\n#define __itt_id_destroy_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_destroy_ex,d,x,y,z)\n#define __itt_id_destroy_ex_ptr ITTNOTIFY_NAME(id_destroy_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_id_create_ex(domain,clock_domain,timestamp,id)\n#define __itt_id_create_ex_ptr 0\n#define __itt_id_destroy_ex(domain,clock_domain,timestamp,id)\n#define __itt_id_destroy_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_id_create_ex_ptr 0\n#define __itt_id_destroy_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup clockdomain\n * @brief Begin a task instance.\n * @param[in] domain The domain for this task\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] taskid The instance ID for this task instance, or __itt_null\n * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null\n * @param[in] name The name of this task\n */\nvoid ITTAPI __itt_task_begin_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name);\n\n/**\n * @ingroup clockdomain\n * @brief Begin a task instance.\n * @param[in] domain The domain for this task\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] taskid The identifier for this task instance, or __itt_null\n * @param[in] parentid The parent of this task, or __itt_null\n * @param[in] fn The pointer to the function you are tracing\n */\nvoid ITTAPI __itt_task_begin_fn_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, void* fn);\n\n/**\n * @ingroup clockdomain\n * @brief End the current task instance.\n * @param[in] domain The domain for this task\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n */\nvoid ITTAPI __itt_task_end_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, task_begin_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn))\nITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp))\n#define __itt_task_begin_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_ex,d,x,y,z,a,b)\n#define __itt_task_begin_ex_ptr ITTNOTIFY_NAME(task_begin_ex)\n#define __itt_task_begin_fn_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_fn_ex,d,x,y,z,a,b)\n#define __itt_task_begin_fn_ex_ptr ITTNOTIFY_NAME(task_begin_fn_ex)\n#define __itt_task_end_ex(d,x,y) ITTNOTIFY_VOID_D2(task_end_ex,d,x,y)\n#define __itt_task_end_ex_ptr ITTNOTIFY_NAME(task_end_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_task_begin_ex(domain,clock_domain,timestamp,id,parentid,name)\n#define __itt_task_begin_ex_ptr 0\n#define __itt_task_begin_fn_ex(domain,clock_domain,timestamp,id,parentid,fn)\n#define __itt_task_begin_fn_ex_ptr 0\n#define __itt_task_end_ex(domain,clock_domain,timestamp)\n#define __itt_task_end_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_task_begin_ex_ptr 0\n#define __itt_task_begin_fn_ex_ptr 0\n#define __itt_task_end_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @defgroup counters Counters\n * @ingroup public\n * Counters are user-defined objects with a monotonically increasing\n * value. Counter values are 64-bit unsigned integers.\n * Counters have names that can be displayed in\n * the tools.\n * @{\n */\n\n/**\n * @brief opaque structure for counter identification\n */\n/** @cond exclude_from_documentation */\n\ntypedef struct ___itt_counter* __itt_counter;\n\n/**\n * @brief Create an unsigned 64 bits integer counter with given name/domain\n *\n * After __itt_counter_create() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta),\n * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr)\n * can be used to change the value of the counter, where value_ptr is a pointer to an unsigned 64 bits integer\n *\n * The call is equal to __itt_counter_create_typed(name, domain, __itt_metadata_u64)\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain);\n__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_counter_create __itt_counter_createW\n# define __itt_counter_create_ptr __itt_counter_createW_ptr\n#else /* UNICODE */\n# define __itt_counter_create __itt_counter_createA\n# define __itt_counter_create_ptr __itt_counter_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain))\nITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA)\n#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA)\n#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW)\n#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create ITTNOTIFY_DATA(counter_create)\n#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA(name, domain)\n#define __itt_counter_createA_ptr 0\n#define __itt_counter_createW(name, domain)\n#define __itt_counter_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create(name, domain)\n#define __itt_counter_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA_ptr 0\n#define __itt_counter_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Increment the unsigned 64 bits integer counter value\n *\n * Calling this function to non-unsigned 64 bits integer counters has no effect\n */\nvoid ITTAPI __itt_counter_inc(__itt_counter id);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id))\n#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc)\n#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_inc(id)\n#define __itt_counter_inc_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_inc_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/**\n * @brief Increment the unsigned 64 bits integer counter value with x\n *\n * Calling this function to non-unsigned 64 bits integer counters has no effect\n */\nvoid ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value))\n#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta)\n#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_inc_delta(id, value)\n#define __itt_counter_inc_delta_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_inc_delta_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Decrement the unsigned 64 bits integer counter value\n *\n * Calling this function to non-unsigned 64 bits integer counters has no effect\n */\nvoid ITTAPI __itt_counter_dec(__itt_counter id);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id))\n#define __itt_counter_dec ITTNOTIFY_VOID(counter_dec)\n#define __itt_counter_dec_ptr ITTNOTIFY_NAME(counter_dec)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_dec(id)\n#define __itt_counter_dec_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_dec_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/**\n * @brief Decrement the unsigned 64 bits integer counter value with x\n *\n * Calling this function to non-unsigned 64 bits integer counters has no effect\n */\nvoid ITTAPI __itt_counter_dec_delta(__itt_counter id, unsigned long long value);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value))\n#define __itt_counter_dec_delta ITTNOTIFY_VOID(counter_dec_delta)\n#define __itt_counter_dec_delta_ptr ITTNOTIFY_NAME(counter_dec_delta)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_dec_delta(id, value)\n#define __itt_counter_dec_delta_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_dec_delta_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup counters\n * @brief Increment a counter by one.\n * The first call with a given name creates a counter by that name and sets its\n * value to zero. Successive calls increment the counter value.\n * @param[in] domain The domain controlling the call. Counter names are not domain specific.\n * The domain argument is used only to enable or disable the API calls.\n * @param[in] name The name of the counter\n */\nvoid ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name);\n\n/**\n * @ingroup counters\n * @brief Increment a counter by the value specified in delta.\n * @param[in] domain The domain controlling the call. Counter names are not domain specific.\n * The domain argument is used only to enable or disable the API calls.\n * @param[in] name The name of the counter\n * @param[in] delta The amount by which to increment the counter\n */\nvoid ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta))\n#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x)\n#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3)\n#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y)\n#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_inc_v3(domain,name)\n#define __itt_counter_inc_v3_ptr 0\n#define __itt_counter_inc_delta_v3(domain,name,delta)\n#define __itt_counter_inc_delta_v3_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_inc_v3_ptr 0\n#define __itt_counter_inc_delta_v3_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n\n/**\n * @ingroup counters\n * @brief Decrement a counter by one.\n * The first call with a given name creates a counter by that name and sets its\n * value to zero. Successive calls decrement the counter value.\n * @param[in] domain The domain controlling the call. Counter names are not domain specific.\n * The domain argument is used only to enable or disable the API calls.\n * @param[in] name The name of the counter\n */\nvoid ITTAPI __itt_counter_dec_v3(const __itt_domain *domain, __itt_string_handle *name);\n\n/**\n * @ingroup counters\n * @brief Decrement a counter by the value specified in delta.\n * @param[in] domain The domain controlling the call. Counter names are not domain specific.\n * The domain argument is used only to enable or disable the API calls.\n * @param[in] name The name of the counter\n * @param[in] delta The amount by which to decrement the counter\n */\nvoid ITTAPI __itt_counter_dec_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name))\nITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta))\n#define __itt_counter_dec_v3(d,x) ITTNOTIFY_VOID_D1(counter_dec_v3,d,x)\n#define __itt_counter_dec_v3_ptr ITTNOTIFY_NAME(counter_dec_v3)\n#define __itt_counter_dec_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_dec_delta_v3,d,x,y)\n#define __itt_counter_dec_delta_v3_ptr ITTNOTIFY_NAME(counter_dec_delta_v3)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_dec_v3(domain,name)\n#define __itt_counter_dec_v3_ptr 0\n#define __itt_counter_dec_delta_v3(domain,name,delta)\n#define __itt_counter_dec_delta_v3_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_dec_v3_ptr 0\n#define __itt_counter_dec_delta_v3_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @} counters group */\n\n\n/**\n * @brief Set the counter value\n */\nvoid ITTAPI __itt_counter_set_value(__itt_counter id, void *value_ptr);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr))\n#define __itt_counter_set_value ITTNOTIFY_VOID(counter_set_value)\n#define __itt_counter_set_value_ptr ITTNOTIFY_NAME(counter_set_value)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_set_value(id, value_ptr)\n#define __itt_counter_set_value_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_set_value_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Set the counter value\n */\nvoid ITTAPI __itt_counter_set_value_ex(__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr))\n#define __itt_counter_set_value_ex ITTNOTIFY_VOID(counter_set_value_ex)\n#define __itt_counter_set_value_ex_ptr ITTNOTIFY_NAME(counter_set_value_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr)\n#define __itt_counter_set_value_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_set_value_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Create a typed counter with given name/domain\n *\n * After __itt_counter_create_typed() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta),\n * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr)\n * can be used to change the value of the counter\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_counter ITTAPI __itt_counter_create_typedA(const char *name, const char *domain, __itt_metadata_type type);\n__itt_counter ITTAPI __itt_counter_create_typedW(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_counter_create_typed __itt_counter_create_typedW\n# define __itt_counter_create_typed_ptr __itt_counter_create_typedW_ptr\n#else /* UNICODE */\n# define __itt_counter_create_typed __itt_counter_create_typedA\n# define __itt_counter_create_typed_ptr __itt_counter_create_typedA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_counter ITTAPI __itt_counter_create_typed(const char *name, const char *domain, __itt_metadata_type type);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type))\nITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_create_typedA ITTNOTIFY_DATA(counter_create_typedA)\n#define __itt_counter_create_typedA_ptr ITTNOTIFY_NAME(counter_create_typedA)\n#define __itt_counter_create_typedW ITTNOTIFY_DATA(counter_create_typedW)\n#define __itt_counter_create_typedW_ptr ITTNOTIFY_NAME(counter_create_typedW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_typed ITTNOTIFY_DATA(counter_create_typed)\n#define __itt_counter_create_typed_ptr ITTNOTIFY_NAME(counter_create_typed)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_create_typedA(name, domain, type)\n#define __itt_counter_create_typedA_ptr 0\n#define __itt_counter_create_typedW(name, domain, type)\n#define __itt_counter_create_typedW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_typed(name, domain, type)\n#define __itt_counter_create_typed_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_create_typedA_ptr 0\n#define __itt_counter_create_typedW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_typed_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() or\n * __itt_counter_create_typed()\n */\nvoid ITTAPI __itt_counter_destroy(__itt_counter id);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id))\n#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy)\n#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_destroy(id)\n#define __itt_counter_destroy_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_destroy_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} counters group */\n\n/**\n * @ingroup markers\n * @brief Create a marker instance.\n * @param[in] domain The domain for this marker\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] id The instance ID for this marker, or __itt_null\n * @param[in] name The name for this marker\n * @param[in] scope The scope for this marker\n */\nvoid ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope))\n#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b)\n#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope)\n#define __itt_marker_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_marker_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @ingroup clockdomain\n * @brief Add a relation to the current task instance.\n * The current task instance is the head of the relation.\n * @param[in] domain The domain controlling this call\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] relation The kind of relation\n * @param[in] tail The ID for the tail of the relation\n */\nvoid ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail);\n\n/**\n * @ingroup clockdomain\n * @brief Add a relation between two instance identifiers.\n * @param[in] domain The domain controlling this call\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] head The ID for the head of the relation\n * @param[in] relation The kind of relation\n * @param[in] tail The ID for the tail of the relation\n */\nvoid ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail))\nITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail))\n#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a)\n#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex)\n#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b)\n#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail)\n#define __itt_relation_add_to_current_ex_ptr 0\n#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail)\n#define __itt_relation_add_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_relation_add_to_current_ex_ptr 0\n#define __itt_relation_add_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @cond exclude_from_documentation */\ntypedef enum ___itt_track_group_type\n{\n __itt_track_group_type_normal = 0\n} __itt_track_group_type;\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_track_group\n{\n __itt_string_handle* name; /*!< Name of the track group */\n struct ___itt_track* track; /*!< List of child tracks */\n __itt_track_group_type tgtype; /*!< Type of the track group */\n int extra1; /*!< Reserved. Must be zero */\n void* extra2; /*!< Reserved. Must be zero */\n struct ___itt_track_group* next;\n} __itt_track_group;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @brief Placeholder for custom track types. Currently, \"normal\" custom track\n * is the only available track type.\n */\ntypedef enum ___itt_track_type\n{\n __itt_track_type_normal = 0\n#ifdef INTEL_ITTNOTIFY_API_PRIVATE\n , __itt_track_type_queue\n#endif /* INTEL_ITTNOTIFY_API_PRIVATE */\n} __itt_track_type;\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_track\n{\n __itt_string_handle* name; /*!< Name of the track group */\n __itt_track_group* group; /*!< Parent group to a track */\n __itt_track_type ttype; /*!< Type of the track */\n int extra1; /*!< Reserved. Must be zero */\n void* extra2; /*!< Reserved. Must be zero */\n struct ___itt_track* next;\n} __itt_track;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @brief Create logical track group.\n */\n__itt_track_group* ITTAPI __itt_track_group_create(__itt_string_handle* name, __itt_track_group_type track_group_type);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type))\n#define __itt_track_group_create ITTNOTIFY_DATA(track_group_create)\n#define __itt_track_group_create_ptr ITTNOTIFY_NAME(track_group_create)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_track_group_create(name) (__itt_track_group*)0\n#define __itt_track_group_create_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_track_group_create_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Create logical track.\n */\n__itt_track* ITTAPI __itt_track_create(__itt_track_group* track_group, __itt_string_handle* name, __itt_track_type track_type);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_track*, track_create, (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type))\n#define __itt_track_create ITTNOTIFY_DATA(track_create)\n#define __itt_track_create_ptr ITTNOTIFY_NAME(track_create)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_track_create(track_group,name,track_type) (__itt_track*)0\n#define __itt_track_create_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_track_create_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Set the logical track.\n */\nvoid ITTAPI __itt_set_track(__itt_track* track);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, set_track, (__itt_track *track))\n#define __itt_set_track ITTNOTIFY_VOID(set_track)\n#define __itt_set_track_ptr ITTNOTIFY_NAME(set_track)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_set_track(track)\n#define __itt_set_track_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_set_track_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/* ========================================================================== */\n/** @cond exclude_from_gpa_documentation */\n/**\n * @defgroup events Events\n * @ingroup public\n * Events group\n * @{\n */\n/** @brief user event type */\ntypedef int __itt_event;\n\n/**\n * @brief Create an event notification\n * @note name or namelen being null/name and namelen not matching, user event feature not enabled\n * @return non-zero event identifier upon success and __itt_err otherwise\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_event LIBITTAPI __itt_event_createA(const char *name, int namelen);\n__itt_event LIBITTAPI __itt_event_createW(const wchar_t *name, int namelen);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_event_create __itt_event_createW\n# define __itt_event_create_ptr __itt_event_createW_ptr\n#else\n# define __itt_event_create __itt_event_createA\n# define __itt_event_create_ptr __itt_event_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_event LIBITTAPI __itt_event_create(const char *name, int namelen);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen))\nITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA ITTNOTIFY_DATA(event_createA)\n#define __itt_event_createA_ptr ITTNOTIFY_NAME(event_createA)\n#define __itt_event_createW ITTNOTIFY_DATA(event_createW)\n#define __itt_event_createW_ptr ITTNOTIFY_NAME(event_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create ITTNOTIFY_DATA(event_create)\n#define __itt_event_create_ptr ITTNOTIFY_NAME(event_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA(name, namelen) (__itt_event)0\n#define __itt_event_createA_ptr 0\n#define __itt_event_createW(name, namelen) (__itt_event)0\n#define __itt_event_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create(name, namelen) (__itt_event)0\n#define __itt_event_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA_ptr 0\n#define __itt_event_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an event occurrence.\n * @return __itt_err upon failure (invalid event id/user event feature not enabled)\n */\nint LIBITTAPI __itt_event_start(__itt_event event);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(LIBITTAPI, int, event_start, (__itt_event event))\n#define __itt_event_start ITTNOTIFY_DATA(event_start)\n#define __itt_event_start_ptr ITTNOTIFY_NAME(event_start)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_event_start(event) (int)0\n#define __itt_event_start_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_event_start_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an event end occurrence.\n * @note It is optional if events do not have durations.\n * @return __itt_err upon failure (invalid event id/user event feature not enabled)\n */\nint LIBITTAPI __itt_event_end(__itt_event event);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(LIBITTAPI, int, event_end, (__itt_event event))\n#define __itt_event_end ITTNOTIFY_DATA(event_end)\n#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_event_end(event) (int)0\n#define __itt_event_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_event_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} events group */\n\n\n/**\n * @defgroup arrays Arrays Visualizer\n * @ingroup public\n * Visualize arrays\n * @{\n */\n\n/**\n * @enum __itt_av_data_type\n * @brief Defines types of arrays data (for C/C++ intrinsic types)\n */\ntypedef enum\n{\n __itt_e_first = 0,\n __itt_e_char = 0, /* 1-byte integer */\n __itt_e_uchar, /* 1-byte unsigned integer */\n __itt_e_int16, /* 2-byte integer */\n __itt_e_uint16, /* 2-byte unsigned integer */\n __itt_e_int32, /* 4-byte integer */\n __itt_e_uint32, /* 4-byte unsigned integer */\n __itt_e_int64, /* 8-byte integer */\n __itt_e_uint64, /* 8-byte unsigned integer */\n __itt_e_float, /* 4-byte floating */\n __itt_e_double, /* 8-byte floating */\n __itt_e_last = __itt_e_double\n} __itt_av_data_type;\n\n/**\n * @brief Save an array data to a file.\n * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only).\n * @param[in] data - pointer to the array data\n * @param[in] rank - the rank of the array\n * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions.\n * The size of dimensions must be equal to the rank\n * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types)\n * @param[in] filePath - the file path; the output format is defined by the file extension\n * @param[in] columnOrder - defines how the array is stored in the linear memory.\n * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C).\n */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nint ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder);\nint ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_av_save __itt_av_saveW\n# define __itt_av_save_ptr __itt_av_saveW_ptr\n#else /* UNICODE */\n# define __itt_av_save __itt_av_saveA\n# define __itt_av_save_ptr __itt_av_saveA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nint ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder))\nITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA)\n#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA)\n#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW)\n#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_av_save ITTNOTIFY_DATA(av_save)\n#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_av_saveA(name)\n#define __itt_av_saveA_ptr 0\n#define __itt_av_saveW(name)\n#define __itt_av_saveW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_av_save(name)\n#define __itt_av_save_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_av_saveA_ptr 0\n#define __itt_av_saveW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_av_save_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\nvoid ITTAPI __itt_enable_attach(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, enable_attach, (void))\n#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach)\n#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_enable_attach()\n#define __itt_enable_attach_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_enable_attach_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @cond exclude_from_gpa_documentation */\n\n/** @} arrays group */\n\n/** @endcond */\n\n/**\n * @brief Module load notification\n * This API is used to report necessary information in case of bypassing default system loader.\n * Notification should be done immediately after this module is loaded to process memory.\n * @param[in] start_addr - module start address\n * @param[in] end_addr - module end address\n * @param[in] path - file system full path to the module\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_module_loadA(void *start_addr, void *end_addr, const char *path);\nvoid ITTAPI __itt_module_loadW(void *start_addr, void *end_addr, const wchar_t *path);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_module_load __itt_module_loadW\n# define __itt_module_load_ptr __itt_module_loadW_ptr\n#else /* UNICODE */\n# define __itt_module_load __itt_module_loadA\n# define __itt_module_load_ptr __itt_module_loadA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_module_load(void *start_addr, void *end_addr, const char *path);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, void, module_loadA, (void *start_addr, void *end_addr, const char *path))\nITT_STUB(ITTAPI, void, module_loadW, (void *start_addr, void *end_addr, const wchar_t *path))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_module_loadA ITTNOTIFY_VOID(module_loadA)\n#define __itt_module_loadA_ptr ITTNOTIFY_NAME(module_loadA)\n#define __itt_module_loadW ITTNOTIFY_VOID(module_loadW)\n#define __itt_module_loadW_ptr ITTNOTIFY_NAME(module_loadW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_module_load ITTNOTIFY_VOID(module_load)\n#define __itt_module_load_ptr ITTNOTIFY_NAME(module_load)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_module_loadA(start_addr, end_addr, path)\n#define __itt_module_loadA_ptr 0\n#define __itt_module_loadW(start_addr, end_addr, path)\n#define __itt_module_loadW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_module_load(start_addr, end_addr, path)\n#define __itt_module_load_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_module_loadA_ptr 0\n#define __itt_module_loadW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_module_load_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Report module unload\n * This API is used to report necessary information in case of bypassing default system loader.\n * Notification should be done just before the module is unloaded from process memory.\n * @param[in] addr - base address of loaded module\n */\nvoid ITTAPI __itt_module_unload(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, module_unload, (void *addr))\n#define __itt_module_unload ITTNOTIFY_VOID(module_unload)\n#define __itt_module_unload_ptr ITTNOTIFY_NAME(module_unload)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_module_unload(addr)\n#define __itt_module_unload_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_module_unload_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @cond exclude_from_documentation */\ntypedef enum\n{\n __itt_module_type_unknown = 0,\n __itt_module_type_elf,\n __itt_module_type_coff\n} __itt_module_type;\n/** @endcond */\n\n/** @cond exclude_from_documentation */\ntypedef enum\n{\n itt_section_type_unknown,\n itt_section_type_bss, /* notifies that the section contains uninitialized data. These are the relevant section types and the modules that contain them:\n * ELF module: SHT_NOBITS section type\n * COFF module: IMAGE_SCN_CNT_UNINITIALIZED_DATA section type\n */\n itt_section_type_data, /* notifies that section contains initialized data. These are the relevant section types and the modules that contain them:\n * ELF module: SHT_PROGBITS section type\n * COFF module: IMAGE_SCN_CNT_INITIALIZED_DATA section type\n */\n itt_section_type_text /* notifies that the section contains executable code. These are the relevant section types and the modules that contain them:\n * ELF module: SHT_PROGBITS section type\n * COFF module: IMAGE_SCN_CNT_CODE section type\n */\n} __itt_section_type;\n/** @endcond */\n\n/**\n * @hideinitializer\n * @brief bit-mask, detects a section attribute that indicates whether a section can be executed as code:\n * These are the relevant section attributes and the modules that contain them:\n * ELF module: PF_X section attribute\n * COFF module: IMAGE_SCN_MEM_EXECUTE attribute\n */\n#define __itt_section_exec 0x20000000\n\n/**\n * @hideinitializer\n * @brief bit-mask, detects a section attribute that indicates whether a section can be read.\n * These are the relevant section attributes and the modules that contain them:\n * ELF module: PF_R attribute\n * COFF module: IMAGE_SCN_MEM_READ attribute\n */\n#define __itt_section_read 0x40000000\n\n/**\n * @hideinitializer\n * @brief bit-mask, detects a section attribute that indicates whether a section can be written to.\n * These are the relevant section attributes and the modules that contain them:\n * ELF module: PF_W attribute\n * COFF module: IMAGE_SCN_MEM_WRITE attribute\n */\n#define __itt_section_write 0x80000000\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_section_info\n{\n const char* name; /*!< Section name in UTF8 */\n __itt_section_type type; /*!< Section content and semantics description */\n size_t flags; /*!< Section bit flags that describe attributes using bit mask\n * Zero if disabled, non-zero if enabled\n */\n void* start_addr; /*!< Section load(relocated) start address */\n size_t size; /*!< Section file offset */\n size_t file_offset; /*!< Section size */\n} __itt_section_info;\n\n#pragma pack(pop)\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_module_object\n{\n unsigned int version; /*!< API version*/\n __itt_id module_id; /*!< Unique identifier. This is unchanged for sections that belong to the same module */\n __itt_module_type module_type; /*!< Binary module format */\n const char* module_name; /*!< Unique module name or path to module in UTF8\n * Contains module name when module_bufer and module_size exist\n * Contains module path when module_bufer and module_size absent\n * module_name remains the same for the certain module_id\n */\n void* module_buffer; /*!< Module buffer content */\n size_t module_size; /*!< Module buffer size */\n /*!< If module_buffer and module_size exist, the binary module is dumped onto the system.\n * If module_buffer and module_size do not exist,\n * the binary module exists on the system already.\n * The module_name parameter contains the path to the module.\n */\n __itt_section_info* section_array; /*!< Reference to section information */\n size_t section_number;\n} __itt_module_object;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @brief Load module content and its loaded(relocated) sections.\n * This API is useful to save a module, or specify its location on the system and report information about loaded sections.\n * The target module is saved on the system if module buffer content and size are available.\n * If module buffer content and size are unavailable, the module name contains the path to the existing binary module.\n * @param[in] module_obj - provides module and section information, along with unique module identifiers (name,module ID)\n * which bind the binary module to particular sections.\n */\nvoid ITTAPI __itt_module_load_with_sections(__itt_module_object* module_obj);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj))\n#define __itt_module_load_with_sections ITTNOTIFY_VOID(module_load_with_sections)\n#define __itt_module_load_with_sections_ptr ITTNOTIFY_NAME(module_load_with_sections)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_module_load_with_sections(module_obj)\n#define __itt_module_load_with_sections_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_module_load_with_sections_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Unload a module and its loaded(relocated) sections.\n * This API notifies that the module and its sections were unloaded.\n * @param[in] module_obj - provides module and sections information, along with unique module identifiers (name,module ID)\n * which bind the binary module to particular sections.\n */\nvoid ITTAPI __itt_module_unload_with_sections(__itt_module_object* module_obj);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj))\n#define __itt_module_unload_with_sections ITTNOTIFY_VOID(module_unload_with_sections)\n#define __itt_module_unload_with_sections_ptr ITTNOTIFY_NAME(module_unload_with_sections)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_module_unload_with_sections(module_obj)\n#define __itt_module_unload_with_sections_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_module_unload_with_sections_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_histogram\n{\n const __itt_domain* domain; /*!< Domain of the histogram*/\n const char* nameA; /*!< Name of the histogram */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* nameW;\n#else /* UNICODE || _UNICODE */\n void* nameW;\n#endif /* UNICODE || _UNICODE */\n __itt_metadata_type x_type; /*!< Type of the histogram X axis */\n __itt_metadata_type y_type; /*!< Type of the histogram Y axis */\n int extra1; /*!< Reserved to the runtime */\n void* extra2; /*!< Reserved to the runtime */\n struct ___itt_histogram* next;\n} __itt_histogram;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @brief Create a typed histogram instance with given name/domain.\n * @param[in] domain The domain controlling the call.\n * @param[in] name The name of the histogram.\n * @param[in] x_type The type of the X axis in histogram (may be 0 to calculate batch statistics).\n * @param[in] y_type The type of the Y axis in histogram.\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_histogram* ITTAPI __itt_histogram_createA(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type);\n__itt_histogram* ITTAPI __itt_histogram_createW(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_histogram_create __itt_histogram_createW\n# define __itt_histogram_create_ptr __itt_histogram_createW_ptr\n#else /* UNICODE */\n# define __itt_histogram_create __itt_histogram_createA\n# define __itt_histogram_create_ptr __itt_histogram_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_histogram* ITTAPI __itt_histogram_create(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type))\nITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_histogram_createA ITTNOTIFY_DATA(histogram_createA)\n#define __itt_histogram_createA_ptr ITTNOTIFY_NAME(histogram_createA)\n#define __itt_histogram_createW ITTNOTIFY_DATA(histogram_createW)\n#define __itt_histogram_createW_ptr ITTNOTIFY_NAME(histogram_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_histogram_create ITTNOTIFY_DATA(histogram_create)\n#define __itt_histogram_create_ptr ITTNOTIFY_NAME(histogram_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_histogram_createA(domain, name, x_type, y_type) (__itt_histogram*)0\n#define __itt_histogram_createA_ptr 0\n#define __itt_histogram_createW(domain, name, x_type, y_type) (__itt_histogram*)0\n#define __itt_histogram_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_histogram_create(domain, name, x_type, y_type) (__itt_histogram*)0\n#define __itt_histogram_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_histogram_createA_ptr 0\n#define __itt_histogram_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_histogram_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Submit statistics for a histogram instance.\n * @param[in] histogram Pointer to the histogram instance to which the histogram statistic is to be dumped.\n * @param[in] length The number of elements in dumped axis data array.\n * @param[in] x_data The X axis dumped data itself (may be NULL to calculate batch statistics).\n * @param[in] y_data The Y axis dumped data itself.\n*/\nvoid ITTAPI __itt_histogram_submit(__itt_histogram* histogram, size_t length, void* x_data, void* y_data);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* histogram, size_t length, void* x_data, void* y_data))\n#define __itt_histogram_submit ITTNOTIFY_VOID(histogram_submit)\n#define __itt_histogram_submit_ptr ITTNOTIFY_NAME(histogram_submit)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_histogram_submit(histogram, length, x_data, y_data)\n#define __itt_histogram_submit_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_histogram_submit_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n\n/**\n* @brief function allows to obtain the current collection state at the moment\n* @return collection state as a enum __itt_collection_state\n*/\n__itt_collection_state __itt_get_collection_state(void);\n\n/**\n* @brief function releases resources allocated by ITT API static part\n* this API should be called from the library destructor\n* @return void\n*/\nvoid __itt_release_resources(void);\n/** @endcond */\n\n/**\n * @brief Create a typed counter with given domain pointer, string name and counter type\n*/\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_counter ITTAPI __itt_counter_createA_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type);\n__itt_counter ITTAPI __itt_counter_createW_v3(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_counter_create_v3 __itt_counter_createW_v3\n# define __itt_counter_create_v3_ptr __itt_counter_createW_v3_ptr\n#else /* UNICODE */\n# define __itt_counter_create_v3 __itt_counter_createA_v3\n# define __itt_counter_create_v3_ptr __itt_counter_createA_v3_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_counter ITTAPI __itt_counter_create_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type))\nITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA_v3 ITTNOTIFY_DATA(counter_createA_v3)\n#define __itt_counter_createA_v3_ptr ITTNOTIFY_NAME(counter_createA_v3)\n#define __itt_counter_createW_v3 ITTNOTIFY_DATA(counter_createW_v3)\n#define __itt_counter_createW_v3_ptr ITTNOTIFY_NAME(counter_createW_v3)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_v3 ITTNOTIFY_DATA(counter_create_v3)\n#define __itt_counter_create_v3_ptr ITTNOTIFY_NAME(counter_create_v3)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA_v3(domain, name, type) (__itt_counter)0\n#define __itt_counter_createA_v3_ptr 0\n#define __itt_counter_createW_v3(domain, name, type) (__itt_counter)0\n#define __itt_counter_create_typedW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_v3(domain, name, type) (__itt_counter)0\n#define __itt_counter_create_v3_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_counter_createA_v3_ptr 0\n#define __itt_counter_createW_v3_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_counter_create_v3_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Set the counter value api\n */\nvoid ITTAPI __itt_counter_set_value_v3(__itt_counter counter, void *value_ptr);\n\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr))\n#define __itt_counter_set_value_v3 ITTNOTIFY_VOID(counter_set_value_v3)\n#define __itt_counter_set_value_v3_ptr ITTNOTIFY_NAME(counter_set_value_v3)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_counter_set_value_v3(counter, value_ptr)\n#define __itt_counter_set_value_v3_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_counter_set_value_v3_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief describes the type of context metadata\n*/\ntypedef enum {\n __itt_context_unknown = 0, /*!< Undefined type */\n __itt_context_nameA, /*!< ASCII string char* type */\n __itt_context_nameW, /*!< Unicode string wchar_t* type */\n __itt_context_deviceA, /*!< ASCII string char* type */\n __itt_context_deviceW, /*!< Unicode string wchar_t* type */\n __itt_context_unitsA, /*!< ASCII string char* type */\n __itt_context_unitsW, /*!< Unicode string wchar_t* type */\n __itt_context_pci_addrA, /*!< ASCII string char* type */\n __itt_context_pci_addrW, /*!< Unicode string wchar_t* type */\n __itt_context_tid, /*!< Unsigned 64-bit integer type */\n __itt_context_max_val, /*!< Unsigned 64-bit integer type */\n __itt_context_bandwidth_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_latency_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_occupancy_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_on_thread_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_is_abs_val_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_cpu_instructions_flag, /*!< Unsigned 64-bit integer type */\n __itt_context_cpu_cycles_flag /*!< Unsigned 64-bit integer type */\n} __itt_context_type;\n\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_context_name __itt_context_nameW\n# define __itt_context_device __itt_context_deviceW\n# define __itt_context_units __itt_context_unitsW\n# define __itt_context_pci_addr __itt_context_pci_addrW\n#else /* UNICODE || _UNICODE */\n# define __itt_context_name __itt_context_nameA\n# define __itt_context_device __itt_context_deviceA\n# define __itt_context_units __itt_context_unitsA\n# define __itt_context_pci_addr __itt_context_pci_addrA\n#endif /* UNICODE || _UNICODE */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_context_metadata\n{\n __itt_context_type type; /*!< Type of the context metadata value */\n void* value; /*!< Pointer to context metadata value itself */\n} __itt_context_metadata;\n\n#pragma pack(pop)\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#pragma pack(push, 8)\n\ntypedef struct ___itt_counter_metadata\n{\n __itt_counter counter; /*!< Associated context metadata counter */\n __itt_context_type type; /*!< Type of the context metadata value */\n const char* str_valueA; /*!< String context metadata value */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* str_valueW;\n#else /* UNICODE || _UNICODE */\n void* str_valueW;\n#endif /* UNICODE || _UNICODE */\n unsigned long long value; /*!< Numeric context metadata value */\n int extra1; /*!< Reserved to the runtime */\n void* extra2; /*!< Reserved to the runtime */\n struct ___itt_counter_metadata* next;\n} __itt_counter_metadata;\n\n#pragma pack(pop)\n/** @endcond */\n\n/**\n * @brief Bind context metadata to counter instance\n * @param[in] counter Pointer to the counter instance to which the context metadata is to be associated.\n * @param[in] length The number of elements in context metadata array.\n * @param[in] metadata The context metadata itself.\n*/\nvoid ITTAPI __itt_bind_context_metadata_to_counter(__itt_counter counter, size_t length, __itt_context_metadata* metadata);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata))\n#define __itt_bind_context_metadata_to_counter ITTNOTIFY_VOID(bind_context_metadata_to_counter)\n#define __itt_bind_context_metadata_to_counter_ptr ITTNOTIFY_NAME(bind_context_metadata_to_counter)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_bind_context_metadata_to_counter(counter, length, metadata)\n#define __itt_bind_context_metadata_to_counter_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_bind_context_metadata_to_counter_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n#ifdef __cplusplus\n}\n#endif /* __cplusplus */\n\n#endif /* _ITTNOTIFY_H_ */\n\n#ifdef INTEL_ITTNOTIFY_API_PRIVATE\n\n#ifndef _ITTNOTIFY_PRIVATE_\n#define _ITTNOTIFY_PRIVATE_\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif /* __cplusplus */\n\n/**\n * @ingroup clockdomain\n * @brief Begin an overlapped task instance.\n * @param[in] domain The domain for this task\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null.\n * @param[in] parentid The parent of this task, or __itt_null.\n * @param[in] name The name of this task.\n */\nvoid ITTAPI __itt_task_begin_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name);\n\n/**\n * @ingroup clockdomain\n * @brief End an overlapped task instance.\n * @param[in] domain The domain for this task\n * @param[in] clock_domain The clock domain controlling the execution of this call.\n * @param[in] timestamp The user defined timestamp.\n * @param[in] taskid Explicit ID of finished task\n */\nvoid ITTAPI __itt_task_end_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, task_begin_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name))\nITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid))\n#define __itt_task_begin_overlapped_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_overlapped_ex,d,x,y,z,a,b)\n#define __itt_task_begin_overlapped_ex_ptr ITTNOTIFY_NAME(task_begin_overlapped_ex)\n#define __itt_task_end_overlapped_ex(d,x,y,z) ITTNOTIFY_VOID_D3(task_end_overlapped_ex,d,x,y,z)\n#define __itt_task_end_overlapped_ex_ptr ITTNOTIFY_NAME(task_end_overlapped_ex)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_task_begin_overlapped_ex(domain,clock_domain,timestamp,taskid,parentid,name)\n#define __itt_task_begin_overlapped_ex_ptr 0\n#define __itt_task_end_overlapped_ex(domain,clock_domain,timestamp,taskid)\n#define __itt_task_end_overlapped_ex_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_task_begin_overlapped_ex_ptr 0\n#define __itt_task_end_overlapped_ptr 0\n#define __itt_task_end_overlapped_ex_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @defgroup makrs_internal Marks\n * @ingroup internal\n * Marks group\n * @warning Internal API:\n * - It is not shipped to outside of Intel\n * - It is delivered to internal Intel teams using e-mail or SVN access only\n * @{\n */\n/** @brief user mark type */\ntypedef int __itt_mark_type;\n\n/**\n * @brief Creates a user mark type with the specified name using char or Unicode string.\n * @param[in] name - name of mark to create\n * @return Returns a handle to the mark type\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_mark_type ITTAPI __itt_mark_createA(const char *name);\n__itt_mark_type ITTAPI __itt_mark_createW(const wchar_t *name);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_mark_create __itt_mark_createW\n# define __itt_mark_create_ptr __itt_mark_createW_ptr\n#else /* UNICODE */\n# define __itt_mark_create __itt_mark_createA\n# define __itt_mark_create_ptr __itt_mark_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_mark_type ITTAPI __itt_mark_create(const char *name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char *name))\nITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_mark_type, mark_create, (const char *name))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_createA ITTNOTIFY_DATA(mark_createA)\n#define __itt_mark_createA_ptr ITTNOTIFY_NAME(mark_createA)\n#define __itt_mark_createW ITTNOTIFY_DATA(mark_createW)\n#define __itt_mark_createW_ptr ITTNOTIFY_NAME(mark_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_create ITTNOTIFY_DATA(mark_create)\n#define __itt_mark_create_ptr ITTNOTIFY_NAME(mark_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_createA(name) (__itt_mark_type)0\n#define __itt_mark_createA_ptr 0\n#define __itt_mark_createW(name) (__itt_mark_type)0\n#define __itt_mark_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_create(name) (__itt_mark_type)0\n#define __itt_mark_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_createA_ptr 0\n#define __itt_mark_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Creates a \"discrete\" user mark type of the specified type and an optional parameter using char or Unicode string.\n *\n * - The mark of \"discrete\" type is placed to collection results in case of success. It appears in overtime view(s) as a special tick sign.\n * - The call is \"synchronous\" - function returns after mark is actually added to results.\n * - This function is useful, for example, to mark different phases of application\n * (beginning of the next mark automatically meand end of current region).\n * - Can be used together with \"continuous\" marks (see below) at the same collection session\n * @param[in] mt - mark, created by __itt_mark_create(const char* name) function\n * @param[in] parameter - string parameter of mark\n * @return Returns zero value in case of success, non-zero value otherwise.\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nint ITTAPI __itt_markA(__itt_mark_type mt, const char *parameter);\nint ITTAPI __itt_markW(__itt_mark_type mt, const wchar_t *parameter);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_mark __itt_markW\n# define __itt_mark_ptr __itt_markW_ptr\n#else /* UNICODE */\n# define __itt_mark __itt_markA\n# define __itt_mark_ptr __itt_markA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nint ITTAPI __itt_mark(__itt_mark_type mt, const char *parameter);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, markA, (__itt_mark_type mt, const char *parameter))\nITT_STUB(ITTAPI, int, markW, (__itt_mark_type mt, const wchar_t *parameter))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark, (__itt_mark_type mt, const char *parameter))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_markA ITTNOTIFY_DATA(markA)\n#define __itt_markA_ptr ITTNOTIFY_NAME(markA)\n#define __itt_markW ITTNOTIFY_DATA(markW)\n#define __itt_markW_ptr ITTNOTIFY_NAME(markW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark ITTNOTIFY_DATA(mark)\n#define __itt_mark_ptr ITTNOTIFY_NAME(mark)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_markA(mt, parameter) (int)0\n#define __itt_markA_ptr 0\n#define __itt_markW(mt, parameter) (int)0\n#define __itt_markW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark(mt, parameter) (int)0\n#define __itt_mark_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_markA_ptr 0\n#define __itt_markW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Use this if necessary to create a \"discrete\" user event type (mark) for process\n * rather then for one thread\n * @see int __itt_mark(__itt_mark_type mt, const char* parameter);\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nint ITTAPI __itt_mark_globalA(__itt_mark_type mt, const char *parameter);\nint ITTAPI __itt_mark_globalW(__itt_mark_type mt, const wchar_t *parameter);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_mark_global __itt_mark_globalW\n# define __itt_mark_global_ptr __itt_mark_globalW_ptr\n#else /* UNICODE */\n# define __itt_mark_global __itt_mark_globalA\n# define __itt_mark_global_ptr __itt_mark_globalA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nint ITTAPI __itt_mark_global(__itt_mark_type mt, const char *parameter);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, mark_globalA, (__itt_mark_type mt, const char *parameter))\nITT_STUB(ITTAPI, int, mark_globalW, (__itt_mark_type mt, const wchar_t *parameter))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark_global, (__itt_mark_type mt, const char *parameter))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_globalA ITTNOTIFY_DATA(mark_globalA)\n#define __itt_mark_globalA_ptr ITTNOTIFY_NAME(mark_globalA)\n#define __itt_mark_globalW ITTNOTIFY_DATA(mark_globalW)\n#define __itt_mark_globalW_ptr ITTNOTIFY_NAME(mark_globalW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_global ITTNOTIFY_DATA(mark_global)\n#define __itt_mark_global_ptr ITTNOTIFY_NAME(mark_global)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_globalA(mt, parameter) (int)0\n#define __itt_mark_globalA_ptr 0\n#define __itt_mark_globalW(mt, parameter) (int)0\n#define __itt_mark_globalW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_global(mt, parameter) (int)0\n#define __itt_mark_global_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_mark_globalA_ptr 0\n#define __itt_mark_globalW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_mark_global_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Creates an \"end\" point for \"continuous\" mark with specified name.\n *\n * - Returns zero value in case of success, non-zero value otherwise.\n * Also returns non-zero value when preceding \"begin\" point for the\n * mark with the same name failed to be created or not created.\n * - The mark of \"continuous\" type is placed to collection results in\n * case of success. It appears in overtime view(s) as a special tick\n * sign (different from \"discrete\" mark) together with line from\n * corresponding \"begin\" mark to \"end\" mark.\n * @note Continuous marks can overlap and be nested inside each other.\n * Discrete mark can be nested inside marked region\n * @param[in] mt - mark, created by __itt_mark_create(const char* name) function\n * @return Returns zero value in case of success, non-zero value otherwise.\n */\nint ITTAPI __itt_mark_off(__itt_mark_type mt);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, int, mark_off, (__itt_mark_type mt))\n#define __itt_mark_off ITTNOTIFY_DATA(mark_off)\n#define __itt_mark_off_ptr ITTNOTIFY_NAME(mark_off)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_mark_off(mt) (int)0\n#define __itt_mark_off_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_mark_off_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Use this if necessary to create an \"end\" point for mark of process\n * @see int __itt_mark_off(__itt_mark_type mt);\n */\nint ITTAPI __itt_mark_global_off(__itt_mark_type mt);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt))\n#define __itt_mark_global_off ITTNOTIFY_DATA(mark_global_off)\n#define __itt_mark_global_off_ptr ITTNOTIFY_NAME(mark_global_off)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_mark_global_off(mt) (int)0\n#define __itt_mark_global_off_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_mark_global_off_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} marks group */\n\n/**\n * @defgroup counters_internal Counters\n * @ingroup internal\n * Counters group\n * @{\n */\n\n\n/**\n * @defgroup stitch Stack Stitching\n * @ingroup internal\n * Stack Stitching group\n * @{\n */\n/**\n * @brief opaque structure for counter identification\n */\ntypedef struct ___itt_caller *__itt_caller;\n\n/**\n * @brief Create the stitch point e.g. a point in call stack where other stacks should be stitched to.\n * The function returns a unique identifier which is used to match the cut points with corresponding stitch points.\n */\n__itt_caller ITTAPI __itt_stack_caller_create(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void))\n#define __itt_stack_caller_create ITTNOTIFY_DATA(stack_caller_create)\n#define __itt_stack_caller_create_ptr ITTNOTIFY_NAME(stack_caller_create)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_stack_caller_create() (__itt_caller)0\n#define __itt_stack_caller_create_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_stack_caller_create_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Destroy the information about stitch point identified by the pointer previously returned by __itt_stack_caller_create()\n */\nvoid ITTAPI __itt_stack_caller_destroy(__itt_caller id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id))\n#define __itt_stack_caller_destroy ITTNOTIFY_VOID(stack_caller_destroy)\n#define __itt_stack_caller_destroy_ptr ITTNOTIFY_NAME(stack_caller_destroy)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_stack_caller_destroy(id)\n#define __itt_stack_caller_destroy_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_stack_caller_destroy_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Sets the cut point. Stack from each event which occurs after this call will be cut\n * at the same stack level the function was called and stitched to the corresponding stitch point.\n */\nvoid ITTAPI __itt_stack_callee_enter(__itt_caller id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, stack_callee_enter, (__itt_caller id))\n#define __itt_stack_callee_enter ITTNOTIFY_VOID(stack_callee_enter)\n#define __itt_stack_callee_enter_ptr ITTNOTIFY_NAME(stack_callee_enter)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_stack_callee_enter(id)\n#define __itt_stack_callee_enter_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_stack_callee_enter_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief This function eliminates the cut point which was set by latest __itt_stack_callee_enter().\n */\nvoid ITTAPI __itt_stack_callee_leave(__itt_caller id);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, stack_callee_leave, (__itt_caller id))\n#define __itt_stack_callee_leave ITTNOTIFY_VOID(stack_callee_leave)\n#define __itt_stack_callee_leave_ptr ITTNOTIFY_NAME(stack_callee_leave)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_stack_callee_leave(id)\n#define __itt_stack_callee_leave_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_stack_callee_leave_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @} stitch group */\n\n/* ***************************************************************************************************************************** */\n\n#include \n\n/** @cond exclude_from_documentation */\ntypedef enum __itt_error_code\n{\n __itt_error_success = 0, /*!< no error */\n __itt_error_no_module = 1, /*!< module can't be loaded */\n /* %1$s -- library name; win: %2$d -- system error code; unix: %2$s -- system error message. */\n __itt_error_no_symbol = 2, /*!< symbol not found */\n /* %1$s -- library name, %2$s -- symbol name. */\n __itt_error_unknown_group = 3, /*!< unknown group specified */\n /* %1$s -- env var name, %2$s -- group name. */\n __itt_error_cant_read_env = 4, /*!< GetEnvironmentVariable() failed */\n /* %1$s -- env var name, %2$d -- system error. */\n __itt_error_env_too_long = 5, /*!< variable value too long */\n /* %1$s -- env var name, %2$d -- actual length of the var, %3$d -- max allowed length. */\n __itt_error_system = 6 /*!< pthread_mutexattr_init or pthread_mutex_init failed */\n /* %1$s -- function name, %2$d -- errno. */\n} __itt_error_code;\n\ntypedef void (__itt_error_handler_t)(__itt_error_code code, va_list);\n__itt_error_handler_t* __itt_set_error_handler(__itt_error_handler_t*);\n\nconst char* ITTAPI __itt_api_version(void);\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#define __itt_error_handler ITT_JOIN(INTEL_ITTNOTIFY_PREFIX, error_handler)\nvoid __itt_error_handler(__itt_error_code code, va_list args);\nextern const int ITTNOTIFY_NAME(err);\n#define __itt_err ITTNOTIFY_NAME(err)\nITT_STUB(ITTAPI, const char*, api_version, (void))\n#define __itt_api_version ITTNOTIFY_DATA(api_version)\n#define __itt_api_version_ptr ITTNOTIFY_NAME(api_version)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_api_version() (const char*)0\n#define __itt_api_version_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_api_version_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n#ifdef __cplusplus\n}\n#endif /* __cplusplus */\n\n#endif /* _ITTNOTIFY_PRIVATE_ */\n\n#endif /* INTEL_ITTNOTIFY_API_PRIVATE */\n" }, { "path": "src/tbb/src/tbb/tools_api/ittnotify_config.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _ITTNOTIFY_CONFIG_H_\n#define _ITTNOTIFY_CONFIG_H_\n\n/** @cond exclude_from_documentation */\n#ifndef ITT_OS_WIN\n# define ITT_OS_WIN 1\n#endif /* ITT_OS_WIN */\n\n#ifndef ITT_OS_LINUX\n# define ITT_OS_LINUX 2\n#endif /* ITT_OS_LINUX */\n\n#ifndef ITT_OS_MAC\n# define ITT_OS_MAC 3\n#endif /* ITT_OS_MAC */\n\n#ifndef ITT_OS_FREEBSD\n# define ITT_OS_FREEBSD 4\n#endif /* ITT_OS_FREEBSD */\n\n#ifndef ITT_OS_OPENBSD\n# define ITT_OS_OPENBSD 5\n#endif /* ITT_OS_OPENBSD */\n\n#ifndef ITT_OS\n# if defined WIN32 || defined _WIN32\n# define ITT_OS ITT_OS_WIN\n# elif defined( __APPLE__ ) && defined( __MACH__ )\n# define ITT_OS ITT_OS_MAC\n# elif defined( __FreeBSD__ )\n# define ITT_OS ITT_OS_FREEBSD\n# elif defined( __OpenBSD__ )\n# define ITT_OS ITT_OS_OPENBSD\n# else\n# define ITT_OS ITT_OS_LINUX\n# endif\n#endif /* ITT_OS */\n\n#ifndef ITT_PLATFORM_WIN\n# define ITT_PLATFORM_WIN 1\n#endif /* ITT_PLATFORM_WIN */\n\n#ifndef ITT_PLATFORM_POSIX\n# define ITT_PLATFORM_POSIX 2\n#endif /* ITT_PLATFORM_POSIX */\n\n#ifndef ITT_PLATFORM_MAC\n# define ITT_PLATFORM_MAC 3\n#endif /* ITT_PLATFORM_MAC */\n\n#ifndef ITT_PLATFORM_FREEBSD\n# define ITT_PLATFORM_FREEBSD 4\n#endif /* ITT_PLATFORM_FREEBSD */\n\n#ifndef ITT_PLATFORM_OPENBSD\n# define ITT_PLATFORM_OPENBSD 5\n#endif /* ITT_PLATFORM_OPENBSD */\n\n#ifndef ITT_PLATFORM\n# if ITT_OS==ITT_OS_WIN\n# define ITT_PLATFORM ITT_PLATFORM_WIN\n# elif ITT_OS==ITT_OS_MAC\n# define ITT_PLATFORM ITT_PLATFORM_MAC\n# elif ITT_OS==ITT_OS_FREEBSD\n# define ITT_PLATFORM ITT_PLATFORM_FREEBSD\n# elif ITT_OS==ITT_OS_OPENBSD\n# define ITT_PLATFORM ITT_PLATFORM_OPENBSD\n# else\n# define ITT_PLATFORM ITT_PLATFORM_POSIX\n# endif\n#endif /* ITT_PLATFORM */\n\n#if defined(_UNICODE) && !defined(UNICODE)\n#define UNICODE\n#endif\n\n#include \n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#include \n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#include \n#if defined(UNICODE) || defined(_UNICODE)\n#include \n#endif /* UNICODE || _UNICODE */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef ITTAPI_CDECL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define ITTAPI_CDECL __cdecl\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define ITTAPI_CDECL __attribute__ ((cdecl))\n# else /* _M_IX86 || __i386__ */\n# define ITTAPI_CDECL /* actual only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* ITTAPI_CDECL */\n\n#ifndef STDCALL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define STDCALL __stdcall\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define STDCALL __attribute__ ((stdcall))\n# else /* _M_IX86 || __i386__ */\n# define STDCALL /* supported only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* STDCALL */\n\n#define ITTAPI ITTAPI_CDECL\n#define LIBITTAPI ITTAPI_CDECL\n\n/* TODO: Temporary for compatibility! */\n#define ITTAPI_CALL ITTAPI_CDECL\n#define LIBITTAPI_CALL ITTAPI_CDECL\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n/* use __forceinline (VC++ specific) */\n#if defined(__MINGW32__) && !defined(__cplusplus)\n#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__))\n#else\n#define ITT_INLINE static __forceinline\n#endif /* __MINGW32__ */\n\n#define ITT_INLINE_ATTRIBUTE /* nothing */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/*\n * Generally, functions are not inlined unless optimization is specified.\n * For functions declared inline, this attribute inlines the function even\n * if no optimization level was specified.\n */\n#ifdef __STRICT_ANSI__\n#define ITT_INLINE static\n#define ITT_INLINE_ATTRIBUTE __attribute__((unused))\n#else /* __STRICT_ANSI__ */\n#define ITT_INLINE static inline\n#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused))\n#endif /* __STRICT_ANSI__ */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/** @endcond */\n\n#ifndef ITT_ARCH_IA32\n# define ITT_ARCH_IA32 1\n#endif /* ITT_ARCH_IA32 */\n\n#ifndef ITT_ARCH_IA32E\n# define ITT_ARCH_IA32E 2\n#endif /* ITT_ARCH_IA32E */\n\n#ifndef ITT_ARCH_IA64\n# define ITT_ARCH_IA64 3\n#endif /* ITT_ARCH_IA64 */\n\n#ifndef ITT_ARCH_ARM\n# define ITT_ARCH_ARM 4\n#endif /* ITT_ARCH_ARM */\n\n#ifndef ITT_ARCH_PPC64\n# define ITT_ARCH_PPC64 5\n#endif /* ITT_ARCH_PPC64 */\n\n#ifndef ITT_ARCH_ARM64\n# define ITT_ARCH_ARM64 6\n#endif /* ITT_ARCH_ARM64 */\n\n#ifndef ITT_ARCH_LOONGARCH64\n# define ITT_ARCH_LOONGARCH64 7\n#endif /* ITT_ARCH_LOONGARCH64 */\n\n#ifndef ITT_ARCH_S390X\n# define ITT_ARCH_S390X 8\n#endif /* ITT_ARCH_S390X */\n\n#ifndef ITT_ARCH_HPPA\n# define ITT_ARCH_HPPA 9\n#endif /* ITT_ARCH_HPPA */\n\n#ifndef ITT_ARCH_RISCV64\n# define ITT_ARCH_RISCV64 10\n#endif /* ITT_ARCH_RISCV64 */\n\n#ifndef ITT_ARCH\n# if defined _M_IX86 || defined __i386__\n# define ITT_ARCH ITT_ARCH_IA32\n# elif defined _M_X64 || defined _M_AMD64 || defined __x86_64__\n# define ITT_ARCH ITT_ARCH_IA32E\n# elif defined _M_IA64 || defined __ia64__\n# define ITT_ARCH ITT_ARCH_IA64\n# elif defined _M_ARM || defined __arm__\n# define ITT_ARCH ITT_ARCH_ARM\n# elif defined __aarch64__\n# define ITT_ARCH ITT_ARCH_ARM64\n# elif defined __powerpc64__\n# define ITT_ARCH ITT_ARCH_PPC64\n# elif defined __loongarch__\n# define ITT_ARCH ITT_ARCH_LOONGARCH64\n# elif defined __s390__ || defined __s390x__\n# define ITT_ARCH ITT_ARCH_S390X\n# elif defined __hppa__\n# define ITT_ARCH ITT_ARCH_HPPA\n# elif defined __riscv && __riscv_xlen == 64\n# define ITT_ARCH ITT_ARCH_RISCV64\n# endif\n\n#endif\n\n#ifdef __cplusplus\n# define ITT_EXTERN_C extern \"C\"\n# define ITT_EXTERN_C_BEGIN extern \"C\" {\n# define ITT_EXTERN_C_END }\n#else\n# define ITT_EXTERN_C /* nothing */\n# define ITT_EXTERN_C_BEGIN /* nothing */\n# define ITT_EXTERN_C_END /* nothing */\n#endif /* __cplusplus */\n\n#define ITT_TO_STR_AUX(x) #x\n#define ITT_TO_STR(x) ITT_TO_STR_AUX(x)\n\n#define __ITT_BUILD_ASSERT(expr, suffix) do { \\\n static char __itt_build_check_##suffix[(expr) ? 1 : -1]; \\\n __itt_build_check_##suffix[0] = 0; \\\n} while(0)\n#define _ITT_BUILD_ASSERT(expr, suffix) __ITT_BUILD_ASSERT((expr), suffix)\n#define ITT_BUILD_ASSERT(expr) _ITT_BUILD_ASSERT((expr), __LINE__)\n\n#define ITT_MAGIC { 0xED, 0xAB, 0xAB, 0xEC, 0x0D, 0xEE, 0xDA, 0x30 }\n\n/* Replace with snapshot date YYYYMMDD for promotion build. */\n#define API_VERSION_BUILD 20230630\n\n#ifndef API_VERSION_NUM\n#define API_VERSION_NUM 3.24.4\n#endif /* API_VERSION_NUM */\n\n#define API_VERSION \"ITT-API-Version \" ITT_TO_STR(API_VERSION_NUM) \\\n \" (\" ITT_TO_STR(API_VERSION_BUILD) \")\"\n\n/* OS communication functions */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#include \ntypedef HMODULE lib_t;\ntypedef DWORD TIDT;\ntypedef CRITICAL_SECTION mutex_t;\n#ifdef __cplusplus\n#define MUTEX_INITIALIZER {}\n#else\n#define MUTEX_INITIALIZER { 0 }\n#endif\n#define strong_alias(name, aliasname) /* empty for Windows */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#include \n#if defined(UNICODE) || defined(_UNICODE)\n#include \n#endif /* UNICODE */\n#ifndef _GNU_SOURCE\n#define _GNU_SOURCE 1 /* need for PTHREAD_MUTEX_RECURSIVE */\n#endif /* _GNU_SOURCE */\n#ifndef __USE_UNIX98\n#define __USE_UNIX98 1 /* need for PTHREAD_MUTEX_RECURSIVE, on SLES11.1 with gcc 4.3.4 wherein pthread.h missing dependency on __USE_XOPEN2K8 */\n#endif /*__USE_UNIX98*/\n#include \ntypedef void* lib_t;\ntypedef pthread_t TIDT;\ntypedef pthread_mutex_t mutex_t;\n#define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER\n#define _strong_alias(name, aliasname) \\\n extern __typeof (name) aliasname __attribute__ ((alias (#name)));\n#define strong_alias(name, aliasname) _strong_alias(name, aliasname)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_get_proc(lib, name) GetProcAddress(lib, name)\n#define __itt_mutex_init(mutex) InitializeCriticalSection(mutex)\n#define __itt_mutex_lock(mutex) EnterCriticalSection(mutex)\n#define __itt_mutex_unlock(mutex) LeaveCriticalSection(mutex)\n#define __itt_mutex_destroy(mutex) DeleteCriticalSection(mutex)\n#define __itt_load_lib(name) LoadLibraryA(name)\n#define __itt_unload_lib(handle) FreeLibrary(handle)\n#define __itt_system_error() (int)GetLastError()\n#define __itt_fstrcmp(s1, s2) lstrcmpA(s1, s2)\n#define __itt_fstrnlen(s, l) strnlen_s(s, l)\n#define __itt_fstrcpyn(s1, b, s2, l) strncpy_s(s1, b, s2, l)\n#define __itt_thread_id() GetCurrentThreadId()\n#define __itt_thread_yield() SwitchToThread()\n#ifndef ITT_SIMPLE_INIT\nITT_INLINE long\n__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;\nITT_INLINE long __itt_interlocked_increment(volatile long* ptr)\n{\n return InterlockedIncrement(ptr);\n}\nITT_INLINE long\n__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) ITT_INLINE_ATTRIBUTE;\nITT_INLINE long\n__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand)\n{\n return InterlockedCompareExchange(ptr, exchange, comperand);\n}\n#endif /* ITT_SIMPLE_INIT */\n\n#define DL_SYMBOLS (1)\n#define PTHREAD_SYMBOLS (1)\n\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\n#define __itt_get_proc(lib, name) dlsym(lib, name)\n#define __itt_mutex_init(mutex) {\\\n pthread_mutexattr_t mutex_attr; \\\n int error_code = pthread_mutexattr_init(&mutex_attr); \\\n if (error_code) \\\n __itt_report_error(__itt_error_system, \"pthread_mutexattr_init\", \\\n error_code); \\\n error_code = pthread_mutexattr_settype(&mutex_attr, \\\n PTHREAD_MUTEX_RECURSIVE); \\\n if (error_code) \\\n __itt_report_error(__itt_error_system, \"pthread_mutexattr_settype\", \\\n error_code); \\\n error_code = pthread_mutex_init(mutex, &mutex_attr); \\\n if (error_code) \\\n __itt_report_error(__itt_error_system, \"pthread_mutex_init\", \\\n error_code); \\\n error_code = pthread_mutexattr_destroy(&mutex_attr); \\\n if (error_code) \\\n __itt_report_error(__itt_error_system, \"pthread_mutexattr_destroy\", \\\n error_code); \\\n}\n#define __itt_mutex_lock(mutex) pthread_mutex_lock(mutex)\n#define __itt_mutex_unlock(mutex) pthread_mutex_unlock(mutex)\n#define __itt_mutex_destroy(mutex) pthread_mutex_destroy(mutex)\n#define __itt_load_lib(name) dlopen(name, RTLD_LAZY)\n#define __itt_unload_lib(handle) dlclose(handle)\n#define __itt_system_error() errno\n#define __itt_fstrcmp(s1, s2) strcmp(s1, s2)\n\n/* makes customer code define safe APIs for SDL_STRNLEN_S and SDL_STRNCPY_S */\n#ifdef SDL_STRNLEN_S\n#define __itt_fstrnlen(s, l) SDL_STRNLEN_S(s, l)\n#else\n#define __itt_fstrnlen(s, l) strlen(s)\n#endif /* SDL_STRNLEN_S */\n#ifdef SDL_STRNCPY_S\n#define __itt_fstrcpyn(s1, b, s2, l) SDL_STRNCPY_S(s1, b, s2, l)\n#else\n#define __itt_fstrcpyn(s1, b, s2, l) { \\\n if (b > 0) { \\\n /* 'volatile' is used to suppress the warning that a destination */ \\\n /* bound depends on the length of the source. */ \\\n volatile size_t num_to_copy = (size_t)(b - 1) < (size_t)(l) ? \\\n (size_t)(b - 1) : (size_t)(l); \\\n strncpy(s1, s2, num_to_copy); \\\n s1[num_to_copy] = 0; \\\n } \\\n}\n#endif /* SDL_STRNCPY_S */\n\n#define __itt_thread_id() pthread_self()\n#define __itt_thread_yield() sched_yield()\n#if ITT_ARCH==ITT_ARCH_IA64\n#ifdef __INTEL_COMPILER\n#define __TBB_machine_fetchadd4(addr, val) __fetchadd4_acq((void *)addr, val)\n#else /* __INTEL_COMPILER */\n#define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val)\n#endif /* __INTEL_COMPILER */\n#elif ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_IA32E /* ITT_ARCH!=ITT_ARCH_IA64 */\nITT_INLINE long\n__TBB_machine_fetchadd4(volatile void* ptr, long addend) ITT_INLINE_ATTRIBUTE;\nITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend)\n{\n long result;\n __asm__ __volatile__(\"lock\\nxadd %0,%1\"\n : \"=r\"(result),\"=m\"(*(volatile int*)ptr)\n : \"0\"(addend), \"m\"(*(volatile int*)ptr)\n : \"memory\");\n return result;\n}\n#else\n#define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val)\n#endif /* ITT_ARCH==ITT_ARCH_IA64 */\n#ifndef ITT_SIMPLE_INIT\nITT_INLINE long\n__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;\nITT_INLINE long __itt_interlocked_increment(volatile long* ptr)\n{\n return __TBB_machine_fetchadd4(ptr, 1) + 1L;\n}\nITT_INLINE long\n__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand) ITT_INLINE_ATTRIBUTE;\nITT_INLINE long\n__itt_interlocked_compare_exchange(volatile long* ptr, long exchange, long comperand)\n{\n return __sync_val_compare_and_swap(ptr, exchange, comperand);\n}\n#endif /* ITT_SIMPLE_INIT */\n\nvoid* dlopen(const char*, int) __attribute__((weak));\nvoid* dlsym(void*, const char*) __attribute__((weak));\nint dlclose(void*) __attribute__((weak));\n#define DL_SYMBOLS (dlopen && dlsym && dlclose)\n\nint pthread_mutex_init(pthread_mutex_t*, const pthread_mutexattr_t*) __attribute__((weak));\nint pthread_mutex_lock(pthread_mutex_t*) __attribute__((weak));\nint pthread_mutex_unlock(pthread_mutex_t*) __attribute__((weak));\nint pthread_mutex_destroy(pthread_mutex_t*) __attribute__((weak));\nint pthread_mutexattr_init(pthread_mutexattr_t*) __attribute__((weak));\nint pthread_mutexattr_settype(pthread_mutexattr_t*, int) __attribute__((weak));\nint pthread_mutexattr_destroy(pthread_mutexattr_t*) __attribute__((weak));\npthread_t pthread_self(void) __attribute__((weak));\n#define PTHREAD_SYMBOLS (pthread_mutex_init && pthread_mutex_lock && pthread_mutex_unlock && pthread_mutex_destroy && pthread_mutexattr_init && pthread_mutexattr_settype && pthread_mutexattr_destroy && pthread_self)\n\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/* strdup() is not included into C99 which results in a compiler warning about\n * implicitly declared symbol. To avoid the issue strdup is implemented\n * manually.\n */\n#define ITT_STRDUP_MAX_STRING_SIZE 4096\n#define __itt_fstrdup(s, new_s) do { \\\n if (s != NULL) { \\\n size_t s_len = __itt_fstrnlen(s, ITT_STRDUP_MAX_STRING_SIZE); \\\n new_s = (char *)malloc(s_len + 1); \\\n if (new_s != NULL) { \\\n __itt_fstrcpyn(new_s, s_len + 1, s, s_len); \\\n } \\\n } \\\n} while(0)\n\ntypedef enum {\n __itt_thread_normal = 0,\n __itt_thread_ignored = 1\n} __itt_thread_state;\n\n#pragma pack(push, 8)\n\ntypedef struct ___itt_thread_info\n{\n const char* nameA; /*!< Copy of original name in ASCII. */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* nameW; /*!< Copy of original name in UNICODE. */\n#else /* UNICODE || _UNICODE */\n void* nameW;\n#endif /* UNICODE || _UNICODE */\n TIDT tid;\n __itt_thread_state state; /*!< Thread state (paused or normal) */\n int extra1; /*!< Reserved to the runtime */\n void* extra2; /*!< Reserved to the runtime */\n struct ___itt_thread_info* next;\n} __itt_thread_info;\n\n#include \"ittnotify_types.h\" /* For __itt_group_id definition */\n\ntypedef struct ___itt_api_info_20101001\n{\n const char* name;\n void** func_ptr;\n void* init_func;\n __itt_group_id group;\n} __itt_api_info_20101001;\n\ntypedef struct ___itt_api_info\n{\n const char* name;\n void** func_ptr;\n void* init_func;\n void* null_func;\n __itt_group_id group;\n} __itt_api_info;\n\ntypedef struct __itt_counter_info\n{\n const char* nameA; /*!< Copy of original name in ASCII. */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* nameW; /*!< Copy of original name in UNICODE. */\n#else /* UNICODE || _UNICODE */\n void* nameW;\n#endif /* UNICODE || _UNICODE */\n const char* domainA; /*!< Copy of original name in ASCII. */\n#if defined(UNICODE) || defined(_UNICODE)\n const wchar_t* domainW; /*!< Copy of original name in UNICODE. */\n#else /* UNICODE || _UNICODE */\n void* domainW;\n#endif /* UNICODE || _UNICODE */\n int type;\n long index;\n int extra1; /*!< Reserved to the runtime */\n void* extra2; /*!< Reserved to the runtime */\n struct __itt_counter_info* next;\n} __itt_counter_info_t;\n\nstruct ___itt_domain;\nstruct ___itt_string_handle;\nstruct ___itt_histogram;\nstruct ___itt_counter_metadata;\n\n#include \"ittnotify.h\"\n\ntypedef struct ___itt_global\n{\n unsigned char magic[8];\n unsigned long version_major;\n unsigned long version_minor;\n unsigned long version_build;\n volatile long api_initialized;\n volatile long mutex_initialized;\n volatile long atomic_counter;\n mutex_t mutex;\n lib_t lib;\n void* error_handler;\n const char** dll_path_ptr;\n __itt_api_info* api_list_ptr;\n struct ___itt_global* next;\n /* Joinable structures below */\n __itt_thread_info* thread_list;\n struct ___itt_domain* domain_list;\n struct ___itt_string_handle* string_list;\n __itt_collection_state state;\n __itt_counter_info_t* counter_list;\n unsigned int ipt_collect_events;\n struct ___itt_histogram* histogram_list;\n struct ___itt_counter_metadata* counter_metadata_list;\n} __itt_global;\n\n#pragma pack(pop)\n\n#define NEW_THREAD_INFO_W(gptr,h,h_tail,t,s,n) { \\\n h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \\\n if (h != NULL) { \\\n h->tid = t; \\\n h->nameA = NULL; \\\n h->nameW = n ? _wcsdup(n) : NULL; \\\n h->state = s; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->thread_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_THREAD_INFO_A(gptr,h,h_tail,t,s,n) { \\\n h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \\\n if (h != NULL) { \\\n h->tid = t; \\\n char *n_copy = NULL; \\\n __itt_fstrdup(n, n_copy); \\\n h->nameA = n_copy; \\\n h->nameW = NULL; \\\n h->state = s; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->thread_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_DOMAIN_W(gptr,h,h_tail,name) { \\\n h = (__itt_domain*)malloc(sizeof(__itt_domain)); \\\n if (h != NULL) { \\\n h->flags = 1; /* domain is enabled by default */ \\\n h->nameA = NULL; \\\n h->nameW = name ? _wcsdup(name) : NULL; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->domain_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_DOMAIN_A(gptr,h,h_tail,name) { \\\n h = (__itt_domain*)malloc(sizeof(__itt_domain)); \\\n if (h != NULL) { \\\n h->flags = 1; /* domain is enabled by default */ \\\n char *name_copy = NULL; \\\n __itt_fstrdup(name, name_copy); \\\n h->nameA = name_copy; \\\n h->nameW = NULL; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->domain_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_STRING_HANDLE_W(gptr,h,h_tail,name) { \\\n h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \\\n if (h != NULL) { \\\n h->strA = NULL; \\\n h->strW = name ? _wcsdup(name) : NULL; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->string_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_STRING_HANDLE_A(gptr,h,h_tail,name) { \\\n h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \\\n if (h != NULL) { \\\n char *name_copy = NULL; \\\n __itt_fstrdup(name, name_copy); \\\n h->strA = name_copy; \\\n h->strW = NULL; \\\n h->extra1 = 0; /* reserved */ \\\n h->extra2 = NULL; /* reserved */ \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->string_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_COUNTER_W(gptr,h,h_tail,name,domain,type) { \\\n h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \\\n if (h != NULL) { \\\n h->nameA = NULL; \\\n h->nameW = name ? _wcsdup(name) : NULL; \\\n h->domainA = NULL; \\\n h->domainW = domain ? _wcsdup(domain) : NULL; \\\n h->type = type; \\\n h->index = 0; \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->counter_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_COUNTER_A(gptr,h,h_tail,name,domain,type) { \\\n h = (__itt_counter_info_t*)malloc(sizeof(__itt_counter_info_t)); \\\n if (h != NULL) { \\\n char *name_copy = NULL; \\\n __itt_fstrdup(name, name_copy); \\\n h->nameA = name_copy; \\\n h->nameW = NULL; \\\n char *domain_copy = NULL; \\\n __itt_fstrdup(domain, domain_copy); \\\n h->domainA = domain_copy; \\\n h->domainW = NULL; \\\n h->type = type; \\\n h->index = 0; \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->counter_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_HISTOGRAM_W(gptr,h,h_tail,domain,name,x_type,y_type) { \\\n h = (__itt_histogram*)malloc(sizeof(__itt_histogram)); \\\n if (h != NULL) { \\\n h->domain = domain; \\\n h->nameA = NULL; \\\n h->nameW = name ? _wcsdup(name) : NULL; \\\n h->x_type = x_type; \\\n h->y_type = y_type; \\\n h->extra1 = 0; \\\n h->extra2 = NULL; \\\n\th->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->histogram_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_HISTOGRAM_A(gptr,h,h_tail,domain,name,x_type,y_type) { \\\n h = (__itt_histogram*)malloc(sizeof(__itt_histogram)); \\\n if (h != NULL) { \\\n h->domain = domain; \\\n char *name_copy = NULL; \\\n __itt_fstrdup(name, name_copy); \\\n h->nameA = name_copy; \\\n h->nameW = NULL; \\\n h->x_type = x_type; \\\n h->y_type = y_type; \\\n h->extra1 = 0; \\\n h->extra2 = NULL; \\\n\th->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->histogram_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_COUNTER_METADATA_NUM(gptr,h,h_tail,counter,type,value) { \\\n h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \\\n if (h != NULL) { \\\n h->counter = counter; \\\n h->type = type; \\\n h->str_valueA = NULL; \\\n h->str_valueW = NULL; \\\n h->value = value; \\\n h->extra1 = 0; \\\n h->extra2 = NULL; \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->counter_metadata_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_COUNTER_METADATA_STR_A(gptr,h,h_tail,counter,type,str_valueA) { \\\n h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \\\n if (h != NULL) { \\\n h->counter = counter; \\\n h->type = type; \\\n char *str_value_copy = NULL; \\\n __itt_fstrdup(str_valueA, str_value_copy); \\\n h->str_valueA = str_value_copy; \\\n h->str_valueW = NULL; \\\n h->value = 0; \\\n h->extra1 = 0; \\\n h->extra2 = NULL; \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->counter_metadata_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#define NEW_COUNTER_METADATA_STR_W(gptr,h,h_tail,counter,type,str_valueW) { \\\n h = (__itt_counter_metadata*)malloc(sizeof(__itt_counter_metadata)); \\\n if (h != NULL) { \\\n h->counter = counter; \\\n h->type = type; \\\n h->str_valueA = NULL; \\\n h->str_valueW = str_valueW ? _wcsdup(str_valueW) : NULL; \\\n h->value = 0; \\\n h->extra1 = 0; \\\n h->extra2 = NULL; \\\n h->next = NULL; \\\n if (h_tail == NULL) \\\n (gptr)->counter_metadata_list = h; \\\n else \\\n h_tail->next = h; \\\n } \\\n}\n\n#endif /* _ITTNOTIFY_CONFIG_H_ */\n" }, { "path": "src/tbb/src/tbb/tools_api/ittnotify_static.c", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#define INTEL_NO_MACRO_BODY\n#define INTEL_ITTNOTIFY_API_PRIVATE\n#include \"ittnotify_config.h\"\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if !defined(PATH_MAX)\n#define PATH_MAX 512\n#endif\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\n#include \n#include \n#include \n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#include \n#include \n#include \n#include \n\n#include \"ittnotify.h\"\n#include \"legacy/ittnotify.h\"\n\n#include \"disable_warnings.h\"\n\nstatic const char api_version[] = API_VERSION \"\\0\\n@(#) $Revision$\\n\";\n\n#define _N_(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n)\n\n#ifndef HAS_CPP_ATTR\n#if defined(__cplusplus) && defined(__has_cpp_attribute)\n#define HAS_CPP_ATTR(X) __has_cpp_attribute(X)\n#else\n#define HAS_CPP_ATTR(X) 0\n#endif\n#endif\n\n#ifndef HAS_C_ATTR\n#if defined(__STDC__) && defined(__has_c_attribute)\n#define HAS_C_ATTR(X) __has_c_attribute(X)\n#else\n#define HAS_C_ATTR(X) 0\n#endif\n#endif\n\n#ifndef HAS_GNU_ATTR\n#if defined(__has_attribute)\n#define HAS_GNU_ATTR(X) __has_attribute(X)\n#else\n#define HAS_GNU_ATTR(X) 0\n#endif\n#endif\n\n#ifndef ITT_ATTRIBUTE_FALLTHROUGH\n#if (HAS_CPP_ATTR(fallthrough) || HAS_C_ATTR(fallthrough)) && (__cplusplus >= 201703L || _MSVC_LANG >= 201703L)\n#define ITT_ATTRIBUTE_FALLTHROUGH [[fallthrough]]\n#elif HAS_CPP_ATTR(gnu::fallthrough)\n#define ITT_ATTRIBUTE_FALLTHROUGH [[gnu::fallthrough]]\n#elif HAS_CPP_ATTR(clang::fallthrough)\n#define ITT_ATTRIBUTE_FALLTHROUGH [[clang::fallthrough]]\n#elif HAS_GNU_ATTR(fallthrough) && !__INTEL_COMPILER\n#define ITT_ATTRIBUTE_FALLTHROUGH __attribute__((fallthrough))\n#else\n#define ITT_ATTRIBUTE_FALLTHROUGH\n#endif\n#endif\n\n#if ITT_OS==ITT_OS_WIN\nstatic const char* ittnotify_lib_name = \"libittnotify.dll\";\n#elif ITT_OS==ITT_OS_LINUX || ITT_OS==ITT_OS_FREEBSD|| ITT_OS==ITT_OS_OPENBSD\nstatic const char* ittnotify_lib_name = \"libittnotify.so\";\n#elif ITT_OS==ITT_OS_MAC\nstatic const char* ittnotify_lib_name = \"libittnotify.dylib\";\n#else\n#error Unsupported or unknown OS.\n#endif\n\n#ifdef __ANDROID__\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#ifdef ITT_ANDROID_LOG\n #define ITT_ANDROID_LOG_TAG \"INTEL_VTUNE_USERAPI\"\n #define ITT_ANDROID_LOGI(...) ((void)__android_log_print(ANDROID_LOG_INFO, ITT_ANDROID_LOG_TAG, __VA_ARGS__))\n #define ITT_ANDROID_LOGW(...) ((void)__android_log_print(ANDROID_LOG_WARN, ITT_ANDROID_LOG_TAG, __VA_ARGS__))\n #define ITT_ANDROID_LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR,ITT_ANDROID_LOG_TAG, __VA_ARGS__))\n #define ITT_ANDROID_LOGD(...) ((void)__android_log_print(ANDROID_LOG_DEBUG,ITT_ANDROID_LOG_TAG, __VA_ARGS__))\n#else\n #define ITT_ANDROID_LOGI(...)\n #define ITT_ANDROID_LOGW(...)\n #define ITT_ANDROID_LOGE(...)\n #define ITT_ANDROID_LOGD(...)\n#endif\n\n/* default location of userapi collector on Android */\n#define ANDROID_ITTNOTIFY_DEFAULT_PATH_MASK(x) \"/data/data/com.intel.vtune/perfrun/lib\" \\\n #x \"/runtime/libittnotify.so\"\n\n#if ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_ARM\n#define ANDROID_ITTNOTIFY_DEFAULT_PATH ANDROID_ITTNOTIFY_DEFAULT_PATH_MASK(32)\n#else\n#define ANDROID_ITTNOTIFY_DEFAULT_PATH ANDROID_ITTNOTIFY_DEFAULT_PATH_MASK(64)\n#endif\n\n#endif\n\n\n#ifndef LIB_VAR_NAME\n#if ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_ARM\n#define LIB_VAR_NAME INTEL_LIBITTNOTIFY32\n#else\n#define LIB_VAR_NAME INTEL_LIBITTNOTIFY64\n#endif\n#endif /* LIB_VAR_NAME */\n\n#define ITT_MUTEX_INIT_AND_LOCK(p) { \\\n if (PTHREAD_SYMBOLS) \\\n { \\\n if (!p.mutex_initialized) \\\n { \\\n if (__itt_interlocked_compare_exchange(&p.atomic_counter, 1, 0) == 0) \\\n { \\\n __itt_mutex_init(&p.mutex); \\\n p.mutex_initialized = 1; \\\n } \\\n else \\\n while (!p.mutex_initialized) \\\n __itt_thread_yield(); \\\n } \\\n __itt_mutex_lock(&p.mutex); \\\n } \\\n}\n\n#define ITT_MUTEX_DESTROY(p) { \\\n if (PTHREAD_SYMBOLS) \\\n { \\\n if (p.mutex_initialized) \\\n { \\\n if (__itt_interlocked_compare_exchange(&p.atomic_counter, 0, 1) == 1) \\\n { \\\n __itt_mutex_destroy(&p.mutex); \\\n p.mutex_initialized = 0; \\\n } \\\n } \\\n } \\\n}\n\n#define ITT_MODULE_OBJECT_VERSION 1\n\ntypedef int (__itt_init_ittlib_t)(const char*, __itt_group_id);\n\n/* this define used to control initialization function name. */\n#ifndef __itt_init_ittlib_name\nITT_EXTERN_C int _N_(init_ittlib)(const char*, __itt_group_id);\nstatic __itt_init_ittlib_t* __itt_init_ittlib_ptr = _N_(init_ittlib);\n#define __itt_init_ittlib_name __itt_init_ittlib_ptr\n#endif /* __itt_init_ittlib_name */\n\ntypedef void (__itt_fini_ittlib_t)(void);\n\n/* this define used to control finalization function name. */\n#ifndef __itt_fini_ittlib_name\nITT_EXTERN_C void _N_(fini_ittlib)(void);\nstatic __itt_fini_ittlib_t* __itt_fini_ittlib_ptr = _N_(fini_ittlib);\n#define __itt_fini_ittlib_name __itt_fini_ittlib_ptr\n#endif /* __itt_fini_ittlib_name */\n\nextern __itt_global _N_(_ittapi_global);\n\n/* building pointers to imported funcs */\n#undef ITT_STUBV\n#undef ITT_STUB\n#define ITT_STUB(api,type,name,args,params,ptr,group,format) \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args;\\\ntypedef type api ITT_JOIN(_N_(name),_t) args; \\\nITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args \\\n{ \\\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) \\\n __itt_init_ittlib_name(NULL, __itt_group_all); \\\n if (ITTNOTIFY_NAME(name) && ITTNOTIFY_NAME(name) != ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init))) \\\n return ITTNOTIFY_NAME(name) params; \\\n else \\\n return (type)0; \\\n}\n\n#define ITT_STUBV(api,type,name,args,params,ptr,group,format) \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args;\\\ntypedef type api ITT_JOIN(_N_(name),_t) args; \\\nITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args \\\n{ \\\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL) \\\n __itt_init_ittlib_name(NULL, __itt_group_all); \\\n if (ITTNOTIFY_NAME(name) && ITTNOTIFY_NAME(name) != ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init))) \\\n ITTNOTIFY_NAME(name) params; \\\n else \\\n return; \\\n}\n\n#undef __ITT_INTERNAL_INIT\n#include \"ittnotify_static.h\"\n\n#undef ITT_STUB\n#undef ITT_STUBV\n#define ITT_STUB(api,type,name,args,params,ptr,group,format) \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args;\\\ntypedef type api ITT_JOIN(_N_(name),_t) args; \\\nITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END\n\n#define ITT_STUBV(api,type,name,args,params,ptr,group,format) \\\nstatic type api ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)) args;\\\ntypedef type api ITT_JOIN(_N_(name),_t) args; \\\nITT_EXTERN_C_BEGIN ITT_JOIN(_N_(name),_t)* ITTNOTIFY_NAME(name) = ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)); ITT_EXTERN_C_END\n\n#define __ITT_INTERNAL_INIT\n#include \"ittnotify_static.h\"\n#undef __ITT_INTERNAL_INIT\n\nITT_GROUP_LIST(group_list);\n\n#pragma pack(push, 8)\n\ntypedef struct ___itt_group_alias\n{\n const char* env_var;\n __itt_group_id groups;\n} __itt_group_alias;\n\nstatic __itt_group_alias group_alias[] = {\n { \"KMP_FOR_TPROFILE\", (__itt_group_id)(__itt_group_control | __itt_group_thread | __itt_group_sync | __itt_group_mark) },\n { \"KMP_FOR_TCHECK\", (__itt_group_id)(__itt_group_control | __itt_group_thread | __itt_group_sync | __itt_group_fsync | __itt_group_mark | __itt_group_suppress) },\n { NULL, (__itt_group_none) },\n { api_version, (__itt_group_none) } /* !!! Just to avoid unused code elimination !!! */\n};\n\n#pragma pack(pop)\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(push)\n#pragma warning(disable: 4054) /* warning C4054: 'type cast' : from function pointer 'XXX' to data pointer 'void *' */\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nstatic __itt_api_info api_list[] = {\n/* Define functions with static implementation */\n#undef ITT_STUB\n#undef ITT_STUBV\n#define ITT_STUB(api,type,name,args,params,nameindll,group,format) { ITT_TO_STR(ITT_JOIN(__itt_,nameindll)), (void**)(void*)&ITTNOTIFY_NAME(name), (void*)(size_t)&ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)), (void*)(size_t)&ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)), (__itt_group_id)(group)},\n#define ITT_STUBV ITT_STUB\n#define __ITT_INTERNAL_INIT\n#include \"ittnotify_static.h\"\n#undef __ITT_INTERNAL_INIT\n/* Define functions without static implementation */\n#undef ITT_STUB\n#undef ITT_STUBV\n#define ITT_STUB(api,type,name,args,params,nameindll,group,format) {ITT_TO_STR(ITT_JOIN(__itt_,nameindll)), (void**)(void*)&ITTNOTIFY_NAME(name), (void*)(size_t)&ITT_VERSIONIZE(ITT_JOIN(_N_(name),_init)), NULL, (__itt_group_id)(group)},\n#define ITT_STUBV ITT_STUB\n#include \"ittnotify_static.h\"\n {NULL, NULL, NULL, NULL, __itt_group_none}\n};\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(pop)\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/* static part descriptor which handles. all notification api attributes. */\n__itt_global _N_(_ittapi_global) = {\n ITT_MAGIC, /* identification info */\n ITT_MAJOR, ITT_MINOR, API_VERSION_BUILD, /* version info */\n 0, /* api_initialized */\n 0, /* mutex_initialized */\n 0, /* atomic_counter */\n MUTEX_INITIALIZER, /* mutex */\n NULL, /* dynamic library handle */\n NULL, /* error_handler */\n NULL, /* dll_path_ptr */\n (__itt_api_info*)&api_list, /* api_list_ptr */\n NULL, /* next __itt_global */\n NULL, /* thread_list */\n NULL, /* domain_list */\n NULL, /* string_list */\n __itt_collection_uninitialized, /* collection state */\n NULL, /* counter_list */\n 0, /* ipt_collect_events */\n NULL, /* histogram_list */\n NULL /* counter_metadata_list */\n};\n\ntypedef void (__itt_api_init_t)(__itt_global*, __itt_group_id);\ntypedef void (__itt_api_fini_t)(__itt_global*);\n\nstatic __itt_domain dummy_domain;\n/* ========================================================================= */\n\n#ifdef ITT_NOTIFY_EXT_REPORT\nITT_EXTERN_C void _N_(error_handler)(__itt_error_code, va_list args);\n#endif /* ITT_NOTIFY_EXT_REPORT */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(push)\n#pragma warning(disable: 4055) /* warning C4055: 'type cast' : from data pointer 'void *' to function pointer 'XXX' */\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nstatic void __itt_report_error(int code, ...)\n{\n va_list args;\n va_start(args, code);\n if (_N_(_ittapi_global).error_handler != NULL)\n {\n __itt_error_handler_t* handler = (__itt_error_handler_t*)(size_t)_N_(_ittapi_global).error_handler;\n handler((__itt_error_code)code, args);\n }\n#ifdef ITT_NOTIFY_EXT_REPORT\n _N_(error_handler)((__itt_error_code)code, args);\n#endif /* ITT_NOTIFY_EXT_REPORT */\n va_end(args);\n}\n\nstatic int __itt_is_collector_available(void);\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(pop)\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_domain* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(domain_createW),_init))(const wchar_t* name)\n{\n __itt_domain *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(domain_createW) && ITTNOTIFY_NAME(domain_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(domain_createW),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(domain_createW)(name);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return &dummy_domain;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameW != NULL && !wcscmp(h->nameW, name)) break;\n }\n if (h == NULL)\n {\n NEW_DOMAIN_W(&_N_(_ittapi_global), h, h_tail, name);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return h;\n}\n\nstatic __itt_domain* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(domain_createA),_init))(const char* name)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_domain* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(domain_create),_init))(const char* name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_domain *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(domain_createA) && ITTNOTIFY_NAME(domain_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(domain_createA),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(domain_createA)(name);\n }\n#else\n if (ITTNOTIFY_NAME(domain_create) && ITTNOTIFY_NAME(domain_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(domain_create),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(domain_create)(name);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return &dummy_domain;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).domain_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameA != NULL && !__itt_fstrcmp(h->nameA, name)) break;\n }\n if (h == NULL)\n {\n NEW_DOMAIN_A(&_N_(_ittapi_global), h, h_tail, name);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return h;\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(module_load_with_sections),_init))(__itt_module_object* module_obj)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(module_load_with_sections) && ITTNOTIFY_NAME(module_load_with_sections) != ITT_VERSIONIZE(ITT_JOIN(_N_(module_load_with_sections),_init)))\n {\n if(module_obj != NULL)\n {\n module_obj->version = ITT_MODULE_OBJECT_VERSION;\n ITTNOTIFY_NAME(module_load_with_sections)(module_obj);\n }\n }\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(module_unload_with_sections),_init))(__itt_module_object* module_obj)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(module_unload_with_sections) && ITTNOTIFY_NAME(module_unload_with_sections) != ITT_VERSIONIZE(ITT_JOIN(_N_(module_unload_with_sections),_init)))\n {\n if(module_obj != NULL)\n {\n module_obj->version = ITT_MODULE_OBJECT_VERSION;\n ITTNOTIFY_NAME(module_unload_with_sections)(module_obj);\n }\n }\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_createW),_init))(const wchar_t* name)\n{\n __itt_string_handle *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(string_handle_createW) && ITTNOTIFY_NAME(string_handle_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_createW),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(string_handle_createW)(name);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->strW != NULL && !wcscmp(h->strW, name)) break;\n }\n if (h == NULL)\n {\n NEW_STRING_HANDLE_W(&_N_(_ittapi_global), h, h_tail, name);\n }\n }\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return h;\n}\n\nstatic __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_createA),_init))(const char* name)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_create),_init))(const char* name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_string_handle *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(string_handle_createA) && ITTNOTIFY_NAME(string_handle_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_createA),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(string_handle_createA)(name);\n }\n#else\n if (ITTNOTIFY_NAME(string_handle_create) && ITTNOTIFY_NAME(string_handle_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_create),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(string_handle_create)(name);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).string_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->strA != NULL && !__itt_fstrcmp(h->strA, name)) break;\n }\n if (h == NULL)\n {\n NEW_STRING_HANDLE_A(&_N_(_ittapi_global), h, h_tail, name);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return h;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW),_init))(const wchar_t *name, const wchar_t *domain)\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n __itt_metadata_type type = __itt_metadata_u64;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(counter_createW) && ITTNOTIFY_NAME(counter_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_createW)(name, domain);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain == NULL) ||\n (h->domainW != NULL && domain != NULL && !wcscmp(h->domainW, domain)))) break;\n\n }\n if (h == NULL)\n {\n NEW_COUNTER_W(&_N_(_ittapi_global), h, h_tail, name, domain, type);\n }\n }\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA),_init))(const char *name, const char *domain)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create),_init))(const char *name, const char *domain)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n __itt_metadata_type type = __itt_metadata_u64;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(counter_createA) && ITTNOTIFY_NAME(counter_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_createA)(name, domain);\n }\n#else\n if (ITTNOTIFY_NAME(counter_create) && ITTNOTIFY_NAME(counter_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_create)(name, domain);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain == NULL) ||\n (h->domainA != NULL && domain != NULL && !__itt_fstrcmp(h->domainA, domain)))) break;\n }\n if (h == NULL)\n {\n NEW_COUNTER_A(&_N_(_ittapi_global), h, h_tail, name, domain, type);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedW),_init))(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type)\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(counter_create_typedW) && ITTNOTIFY_NAME(counter_create_typedW) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedW),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_create_typedW)(name, domain, type);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain == NULL) ||\n (h->domainW != NULL && domain != NULL && !wcscmp(h->domainW, domain)))) break;\n\n }\n if (h == NULL)\n {\n NEW_COUNTER_W(&_N_(_ittapi_global), h, h_tail, name, domain, type);\n }\n }\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedA),_init))(const char *name, const char *domain, __itt_metadata_type type)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typed),_init))(const char *name, const char *domain, __itt_metadata_type type)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n\n if (name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(counter_create_typedA) && ITTNOTIFY_NAME(counter_create_typedA) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typedA),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_create_typedA)(name, domain, type);\n }\n#else\n if (ITTNOTIFY_NAME(counter_create_typed) && ITTNOTIFY_NAME(counter_create_typed) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_typed),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_create_typed)(name, domain, type);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain == NULL) ||\n (h->domainA != NULL && domain != NULL && !__itt_fstrcmp(h->domainA, domain)))) break;\n }\n if (h == NULL)\n {\n NEW_COUNTER_A(&_N_(_ittapi_global), h, h_tail, name, domain, type);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createW),_init))(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type)\n{\n __itt_histogram *h_tail = NULL, *h = NULL;\n\n if (domain == NULL || name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(histogram_createW) && ITTNOTIFY_NAME(histogram_createW) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createW),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(histogram_createW)(domain, name, x_type, y_type);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).histogram_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->domain == NULL) continue;\n else if (h->domain == domain && h->nameW != NULL && !wcscmp(h->nameW, name)) break;\n }\n if (h == NULL)\n {\n NEW_HISTOGRAM_W(&_N_(_ittapi_global), h, h_tail, domain, name, x_type, y_type);\n }\n }\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_histogram*)h;\n}\n\nstatic __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createA),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_histogram* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_create),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_histogram *h_tail = NULL, *h = NULL;\n\n if (domain == NULL || name == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(histogram_createA) && ITTNOTIFY_NAME(histogram_createA) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_createA),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(histogram_createA)(domain, name, x_type, y_type);\n }\n#else\n if (ITTNOTIFY_NAME(histogram_create) && ITTNOTIFY_NAME(histogram_create) != ITT_VERSIONIZE(ITT_JOIN(_N_(histogram_create),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(histogram_create)(domain, name, x_type, y_type);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).histogram_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->domain == NULL) continue;\n else if (h->domain == domain && h->nameA != NULL && !__itt_fstrcmp(h->nameA, name)) break;\n }\n if (h == NULL)\n {\n NEW_HISTOGRAM_A(&_N_(_ittapi_global), h, h_tail, domain, name, x_type, y_type);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_histogram*)h;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW_v3),_init))(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type)\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n\n if (name == NULL || domain == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(counter_createW_v3) && ITTNOTIFY_NAME(counter_createW_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createW_v3),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_createW_v3)(domain, name, type);\n }\n else\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameW != NULL && h->type == (int)type && !wcscmp(h->nameW, name) && ((h->domainW == NULL && domain->nameW == NULL) ||\n (h->domainW != NULL && domain->nameW != NULL && !wcscmp(h->domainW, domain->nameW)))) break;\n\n }\n if (h == NULL)\n {\n NEW_COUNTER_W(&_N_(_ittapi_global),h,h_tail,name,domain->nameW,type);\n }\n }\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA_v3),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type type)\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nstatic __itt_counter ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_v3),_init))(const __itt_domain* domain, const char* name, __itt_metadata_type type)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n __itt_counter_info_t *h_tail = NULL, *h = NULL;\n\n if (name == NULL || domain == NULL)\n {\n return NULL;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(counter_createA_v3) && ITTNOTIFY_NAME(counter_createA_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_createA_v3),_init)))\n {\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_createA_v3)(domain, name, type);\n }\n#else\n if (ITTNOTIFY_NAME(counter_create_v3) && ITTNOTIFY_NAME(counter_create_v3) != ITT_VERSIONIZE(ITT_JOIN(_N_(counter_create_v3),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return ITTNOTIFY_NAME(counter_create_v3)(domain, name, type);\n }\n#endif\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return NULL;\n }\n }\n if (__itt_is_collector_available())\n {\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->nameA != NULL && h->type == (int)type && !__itt_fstrcmp(h->nameA, name) && ((h->domainA == NULL && domain->nameA == NULL) ||\n (h->domainA != NULL && domain->nameA != NULL && !__itt_fstrcmp(h->domainA, domain->nameA)))) break;\n }\n if (h == NULL)\n {\n NEW_COUNTER_A(&_N_(_ittapi_global),h,h_tail,name,domain->nameA,type);\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return (__itt_counter)h;\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(bind_context_metadata_to_counter),_init))(__itt_counter counter, size_t length, __itt_context_metadata* metadata)\n{\n __itt_counter_metadata *h_tail = NULL, *h = NULL;\n\n if (counter == NULL || length == 0 || metadata == NULL)\n {\n return;\n }\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (ITTNOTIFY_NAME(bind_context_metadata_to_counter) && ITTNOTIFY_NAME(bind_context_metadata_to_counter) != ITT_VERSIONIZE(ITT_JOIN(_N_(bind_context_metadata_to_counter),_init)))\n {\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n ITTNOTIFY_NAME(bind_context_metadata_to_counter)(counter, length, metadata);\n }\n else\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#else\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif\n return;\n }\n }\n if (__itt_is_collector_available())\n {\n size_t item;\n char* str_valueA = NULL;\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n wchar_t* str_valueW = NULL;\n#endif\n unsigned long long value = 0;\n __itt_context_type type = __itt_context_unknown;\n\n for (item = 0; item < length; item++)\n {\n type = metadata[item].type;\n for (h_tail = NULL, h = _N_(_ittapi_global).counter_metadata_list; h != NULL; h_tail = h, h = h->next)\n {\n if (h->counter != NULL && h->counter == counter && h->type == type) break;\n }\n if (h == NULL && counter != NULL && type != __itt_context_unknown)\n {\n if (type == __itt_context_nameA || type == __itt_context_deviceA || type == __itt_context_unitsA || type == __itt_context_pci_addrA)\n {\n str_valueA = (char*)(metadata[item].value);\n NEW_COUNTER_METADATA_STR_A(&_N_(_ittapi_global),h,h_tail,counter,type,str_valueA);\n }\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n else if (type == __itt_context_nameW || type == __itt_context_deviceW || type == __itt_context_unitsW || type == __itt_context_pci_addrW)\n {\n str_valueW = (wchar_t*)(metadata[item].value);\n NEW_COUNTER_METADATA_STR_W(&_N_(_ittapi_global),h,h_tail,counter,type,str_valueW);\n }\n#endif\n else if (type >= __itt_context_tid && type <= __itt_context_cpu_cycles_flag)\n {\n value = *(unsigned long long*)(metadata[item].value);\n NEW_COUNTER_METADATA_NUM(&_N_(_ittapi_global),h,h_tail,counter,type,value);\n }\n }\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n}\n/* -------------------------------------------------------------------------- */\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(pause),_init))(void)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(pause) && ITTNOTIFY_NAME(pause) != ITT_VERSIONIZE(ITT_JOIN(_N_(pause),_init)))\n {\n ITTNOTIFY_NAME(pause)();\n }\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(resume),_init))(void)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(resume) && ITTNOTIFY_NAME(resume) != ITT_VERSIONIZE(ITT_JOIN(_N_(resume),_init)))\n {\n ITTNOTIFY_NAME(resume)();\n }\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(pause_scoped),_init))(__itt_collection_scope scope)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(pause_scoped) && ITTNOTIFY_NAME(pause_scoped) != ITT_VERSIONIZE(ITT_JOIN(_N_(pause_scoped),_init)))\n {\n ITTNOTIFY_NAME(pause_scoped)(scope);\n }\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(resume_scoped),_init))(__itt_collection_scope scope)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(resume_scoped) && ITTNOTIFY_NAME(resume_scoped) != ITT_VERSIONIZE(ITT_JOIN(_N_(resume_scoped),_init)))\n {\n ITTNOTIFY_NAME(resume_scoped)(scope);\n }\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameW),_init))(const wchar_t* name)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(thread_set_nameW) && ITTNOTIFY_NAME(thread_set_nameW) != ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameW),_init)))\n {\n ITTNOTIFY_NAME(thread_set_nameW)(name);\n }\n}\n\nstatic int ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thr_name_setW),_init))(const wchar_t* name, int namelen)\n{\n (void)namelen;\n ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameW),_init))(name);\n return 0;\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameA),_init))(const char* name)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_name),_init))(const char* name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n if (ITTNOTIFY_NAME(thread_set_nameA) && ITTNOTIFY_NAME(thread_set_nameA) != ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameA),_init)))\n {\n ITTNOTIFY_NAME(thread_set_nameA)(name);\n }\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n if (ITTNOTIFY_NAME(thread_set_name) && ITTNOTIFY_NAME(thread_set_name) != ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_name),_init)))\n {\n ITTNOTIFY_NAME(thread_set_name)(name);\n }\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nstatic int ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thr_name_setA),_init))(const char* name, int namelen)\n{\n (void)namelen;\n ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_nameA),_init))(name);\n return 0;\n}\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nstatic int ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thr_name_set),_init))(const char* name, int namelen)\n{\n (void)namelen;\n ITT_VERSIONIZE(ITT_JOIN(_N_(thread_set_name),_init))(name);\n return 0;\n}\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thread_ignore),_init))(void)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n if (ITTNOTIFY_NAME(thread_ignore) && ITTNOTIFY_NAME(thread_ignore) != ITT_VERSIONIZE(ITT_JOIN(_N_(thread_ignore),_init)))\n {\n ITTNOTIFY_NAME(thread_ignore)();\n }\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(thr_ignore),_init))(void)\n{\n ITT_VERSIONIZE(ITT_JOIN(_N_(thread_ignore),_init))();\n}\n\nstatic void ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(enable_attach),_init))(void)\n{\n#ifdef __ANDROID__\n /*\n * if LIB_VAR_NAME env variable were set before then stay previous value\n * else set default path\n */\n setenv(ITT_TO_STR(LIB_VAR_NAME), ANDROID_ITTNOTIFY_DEFAULT_PATH, 0);\n#endif\n}\n\n/* -------------------------------------------------------------------------- */\n\nstatic const char* __itt_fsplit(const char* s, const char* sep, const char** out, int* len)\n{\n int i;\n int j;\n\n if (!s || !sep || !out || !len)\n return NULL;\n\n for (i = 0; s[i]; i++)\n {\n int b = 0;\n for (j = 0; sep[j]; j++)\n if (s[i] == sep[j])\n {\n b = 1;\n break;\n }\n if (!b)\n break;\n }\n\n if (!s[i])\n return NULL;\n\n *len = 0;\n *out = &s[i];\n\n for (; s[i]; i++, (*len)++)\n {\n int b = 0;\n for (j = 0; sep[j]; j++)\n if (s[i] == sep[j])\n {\n b = 1;\n break;\n }\n if (b)\n break;\n }\n\n for (; s[i]; i++)\n {\n int b = 0;\n for (j = 0; sep[j]; j++)\n if (s[i] == sep[j])\n {\n b = 1;\n break;\n }\n if (!b)\n break;\n }\n\n return &s[i];\n}\n\n/* This function return value of env variable that placed into static buffer.\n * !!! The same static buffer is used for subsequent calls. !!!\n * This was done to avoid dynamic allocation for few calls.\n * Actually we need this function only four times.\n */\nstatic const char* __itt_get_env_var(const char* name)\n{\n#define MAX_ENV_VALUE_SIZE 4086\n static char env_buff[MAX_ENV_VALUE_SIZE];\n static char* env_value = (char*)env_buff;\n\n if (name != NULL)\n {\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n size_t max_len = MAX_ENV_VALUE_SIZE - (size_t)(env_value - env_buff);\n DWORD rc = GetEnvironmentVariableA(name, env_value, (DWORD)max_len);\n if (rc >= max_len)\n __itt_report_error(__itt_error_env_too_long, name, (size_t)rc - 1, (size_t)(max_len - 1));\n else if (rc > 0)\n {\n const char* ret = (const char*)env_value;\n env_value += rc + 1;\n return ret;\n }\n else\n {\n /* If environment variable is empty, GetEnvironmentVariables()\n * returns zero (number of characters (not including terminating null),\n * and GetLastError() returns ERROR_SUCCESS. */\n DWORD err = GetLastError();\n if (err == ERROR_SUCCESS)\n return env_value;\n\n if (err != ERROR_ENVVAR_NOT_FOUND)\n __itt_report_error(__itt_error_cant_read_env, name, (int)err);\n }\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\n char* env = getenv(name);\n if (env != NULL)\n {\n size_t len = __itt_fstrnlen(env, MAX_ENV_VALUE_SIZE);\n size_t max_len = MAX_ENV_VALUE_SIZE - (size_t)(env_value - env_buff);\n if (len < max_len)\n {\n const char* ret = (const char*)env_value;\n __itt_fstrcpyn(env_value, max_len, env, len + 1);\n env_value += len + 1;\n return ret;\n } else\n __itt_report_error(__itt_error_env_too_long, name, (size_t)len, (size_t)(max_len - 1));\n }\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n }\n return NULL;\n}\n\nstatic const char* __itt_get_lib_name(void)\n{\n const char* lib_name = __itt_get_env_var(ITT_TO_STR(LIB_VAR_NAME));\n\n#ifdef __ANDROID__\n if (lib_name == NULL)\n {\n\n#if ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_ARM\n const char* const marker_filename = \"com.intel.itt.collector_lib_32\";\n#else\n const char* const marker_filename = \"com.intel.itt.collector_lib_64\";\n#endif\n\n char system_wide_marker_filename[PATH_MAX] = {0};\n int itt_marker_file_fd = -1;\n ssize_t res = 0;\n\n res = snprintf(system_wide_marker_filename, PATH_MAX - 1, \"%s%s\", \"/data/local/tmp/\", marker_filename);\n if (res < 0)\n {\n ITT_ANDROID_LOGE(\"Unable to concatenate marker file string.\");\n return lib_name;\n }\n itt_marker_file_fd = open(system_wide_marker_filename, O_RDONLY);\n\n if (itt_marker_file_fd == -1)\n {\n const pid_t my_pid = getpid();\n char cmdline_path[PATH_MAX] = {0};\n char package_name[PATH_MAX] = {0};\n char app_sandbox_file[PATH_MAX] = {0};\n int cmdline_fd = 0;\n\n ITT_ANDROID_LOGI(\"Unable to open system-wide marker file.\");\n res = snprintf(cmdline_path, PATH_MAX - 1, \"/proc/%d/cmdline\", my_pid);\n if (res < 0)\n {\n ITT_ANDROID_LOGE(\"Unable to get cmdline path string.\");\n return lib_name;\n }\n\n ITT_ANDROID_LOGI(\"CMD file: %s\\n\", cmdline_path);\n cmdline_fd = open(cmdline_path, O_RDONLY);\n if (cmdline_fd == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to open %s file!\", cmdline_path);\n return lib_name;\n }\n res = read(cmdline_fd, package_name, PATH_MAX - 1);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to read %s file!\", cmdline_path);\n res = close(cmdline_fd);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to close %s file!\", cmdline_path);\n }\n return lib_name;\n }\n res = close(cmdline_fd);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to close %s file!\", cmdline_path);\n return lib_name;\n }\n ITT_ANDROID_LOGI(\"Package name: %s\\n\", package_name);\n res = snprintf(app_sandbox_file, PATH_MAX - 1, \"/data/data/%s/%s\", package_name, marker_filename);\n if (res < 0)\n {\n ITT_ANDROID_LOGE(\"Unable to concatenate marker file string.\");\n return lib_name;\n }\n\n ITT_ANDROID_LOGI(\"Lib marker file name: %s\\n\", app_sandbox_file);\n itt_marker_file_fd = open(app_sandbox_file, O_RDONLY);\n if (itt_marker_file_fd == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to open app marker file!\");\n return lib_name;\n }\n }\n\n {\n char itt_lib_name[PATH_MAX] = {0};\n\n res = read(itt_marker_file_fd, itt_lib_name, PATH_MAX - 1);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to read %s file!\", itt_marker_file_fd);\n res = close(itt_marker_file_fd);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to close %s file!\", itt_marker_file_fd);\n }\n return lib_name;\n }\n ITT_ANDROID_LOGI(\"ITT Lib path: %s\", itt_lib_name);\n res = close(itt_marker_file_fd);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to close %s file!\", itt_marker_file_fd);\n return lib_name;\n }\n ITT_ANDROID_LOGI(\"Set env %s to %s\", ITT_TO_STR(LIB_VAR_NAME), itt_lib_name);\n res = setenv(ITT_TO_STR(LIB_VAR_NAME), itt_lib_name, 0);\n if (res == -1)\n {\n ITT_ANDROID_LOGE(\"Unable to set env var!\");\n return lib_name;\n }\n lib_name = __itt_get_env_var(ITT_TO_STR(LIB_VAR_NAME));\n ITT_ANDROID_LOGI(\"ITT Lib path from env: %s\", lib_name);\n }\n }\n#endif\n\n return lib_name;\n}\n\n/* Avoid clashes with std::min */\n#define __itt_min(a,b) ((a) < (b) ? (a) : (b))\n\nstatic __itt_group_id __itt_get_groups(void)\n{\n int i;\n __itt_group_id res = __itt_group_none;\n const char* var_name = \"INTEL_ITTNOTIFY_GROUPS\";\n const char* group_str = __itt_get_env_var(var_name);\n\n if (group_str != NULL)\n {\n int len;\n char gr[255];\n const char* chunk;\n while ((group_str = __itt_fsplit(group_str, \",; \", &chunk, &len)) != NULL)\n {\n int min_len = __itt_min(len, (int)(sizeof(gr) - 1));\n __itt_fstrcpyn(gr, sizeof(gr) - 1, chunk, min_len);\n gr[min_len] = 0;\n\n for (i = 0; group_list[i].name != NULL; i++)\n {\n if (!__itt_fstrcmp(gr, group_list[i].name))\n {\n res = (__itt_group_id)(res | group_list[i].id);\n break;\n }\n }\n }\n /* TODO: !!! Workaround for bug with warning for unknown group !!!\n * Should be fixed in new initialization scheme.\n * Now the following groups should be set always. */\n for (i = 0; group_list[i].id != __itt_group_none; i++)\n if (group_list[i].id != __itt_group_all &&\n group_list[i].id > __itt_group_splitter_min &&\n group_list[i].id < __itt_group_splitter_max)\n res = (__itt_group_id)(res | group_list[i].id);\n return res;\n }\n else\n {\n for (i = 0; group_alias[i].env_var != NULL; i++)\n if (__itt_get_env_var(group_alias[i].env_var) != NULL)\n return group_alias[i].groups;\n }\n\n return res;\n}\n\n#undef __itt_min\n\nstatic int __itt_lib_version(lib_t lib)\n{\n if (lib == NULL)\n return 0;\n if (__itt_get_proc(lib, \"__itt_api_init\"))\n return 2;\n if (__itt_get_proc(lib, \"__itt_api_version\"))\n return 1;\n return 0;\n}\n\n/* It's not used right now! Comment it out to avoid warnings.\nstatic void __itt_reinit_all_pointers(void)\n{\n register int i;\n // Fill all pointers with initial stubs\n for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++)\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].init_func;\n}\n*/\n\nstatic void __itt_nullify_all_pointers(void)\n{\n int i;\n /* Nulify all pointers except domain_create, string_handle_create and counter_create */\n for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++)\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].null_func;\n}\n\nstatic int __itt_is_collector_available(void)\n{\n int is_available;\n\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).state == __itt_collection_uninitialized)\n {\n _N_(_ittapi_global).state = (NULL == __itt_get_lib_name()) ? __itt_collection_collector_absent : __itt_collection_collector_exists;\n }\n is_available = (_N_(_ittapi_global).state == __itt_collection_collector_exists ||\n _N_(_ittapi_global).state == __itt_collection_init_successful);\n __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n return is_available;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(push)\n#pragma warning(disable: 4054) /* warning C4054: 'type cast' : from function pointer 'XXX' to data pointer 'void *' */\n#pragma warning(disable: 4055) /* warning C4055: 'type cast' : from data pointer 'void *' to function pointer 'XXX' */\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nITT_EXTERN_C void _N_(fini_ittlib)(void)\n{\n __itt_api_fini_t* __itt_api_fini_ptr = NULL;\n static volatile TIDT current_thread = 0;\n\n if (_N_(_ittapi_global).api_initialized)\n {\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n if (_N_(_ittapi_global).api_initialized)\n {\n if (current_thread == 0)\n {\n if (PTHREAD_SYMBOLS) current_thread = __itt_thread_id();\n if (_N_(_ittapi_global).lib != NULL)\n {\n __itt_api_fini_ptr = (__itt_api_fini_t*)(size_t)__itt_get_proc(_N_(_ittapi_global).lib, \"__itt_api_fini\");\n }\n if (__itt_api_fini_ptr)\n {\n __itt_api_fini_ptr(&_N_(_ittapi_global));\n }\n\n __itt_nullify_all_pointers();\n\n /* TODO: !!! not safe !!! don't support unload so far.\n * if (_N_(_ittapi_global).lib != NULL)\n * __itt_unload_lib(_N_(_ittapi_global).lib);\n * _N_(_ittapi_global).lib = NULL;\n */\n _N_(_ittapi_global).api_initialized = 0;\n current_thread = 0;\n }\n }\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n }\n}\n\n/* !!! this function should be called under mutex lock !!! */\nstatic void __itt_free_allocated_resources(void)\n{\n __itt_string_handle* current_string = _N_(_ittapi_global).string_list;\n while (current_string != NULL)\n {\n __itt_string_handle* tmp = current_string->next;\n free((char*)current_string->strA);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n free((wchar_t*)current_string->strW);\n#endif\n free(current_string);\n current_string = tmp;\n }\n _N_(_ittapi_global).string_list = NULL;\n\n __itt_domain* current_domain = _N_(_ittapi_global).domain_list;\n while (current_domain != NULL)\n {\n __itt_domain* tmp = current_domain->next;\n free((char*)current_domain->nameA);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n free((wchar_t*)current_domain->nameW);\n#endif\n free(current_domain);\n current_domain = tmp;\n }\n _N_(_ittapi_global).domain_list = NULL;\n\n __itt_counter_info_t* current_couter = _N_(_ittapi_global).counter_list;\n while (current_couter != NULL)\n {\n __itt_counter_info_t* tmp = current_couter->next;\n free((char*)current_couter->nameA);\n free((char*)current_couter->domainA);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n free((wchar_t*)current_couter->nameW);\n free((wchar_t*)current_couter->domainW);\n#endif\n free(current_couter);\n current_couter = tmp;\n }\n _N_(_ittapi_global).counter_list = NULL;\n\n __itt_histogram* current_histogram = _N_(_ittapi_global).histogram_list;\n while (current_histogram != NULL)\n {\n __itt_histogram* tmp = current_histogram->next;\n free((char*)current_histogram->nameA);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n free((wchar_t*)current_histogram->nameW);\n#endif\n free(current_histogram);\n current_histogram = tmp;\n }\n _N_(_ittapi_global).histogram_list = NULL;\n\n \n __itt_counter_metadata* current_counter_metadata = _N_(_ittapi_global).counter_metadata_list;\n while (current_counter_metadata != NULL)\n {\n __itt_counter_metadata* tmp = current_counter_metadata->next;\n free((char*)current_counter_metadata->str_valueA);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n free((wchar_t*)current_counter_metadata->str_valueW);\n#endif\n free(current_counter_metadata);\n current_counter_metadata = tmp;\n }\n _N_(_ittapi_global).counter_metadata_list = NULL;\n}\n\nITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_groups)\n{\n int i;\n __itt_group_id groups;\n#ifdef ITT_COMPLETE_GROUP\n __itt_group_id zero_group = __itt_group_none;\n#endif /* ITT_COMPLETE_GROUP */\n static volatile TIDT current_thread = 0;\n\n if (!_N_(_ittapi_global).api_initialized)\n {\n#ifndef ITT_SIMPLE_INIT\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n#endif /* ITT_SIMPLE_INIT */\n\n if (!_N_(_ittapi_global).api_initialized)\n {\n if (current_thread == 0)\n {\n if (PTHREAD_SYMBOLS) current_thread = __itt_thread_id();\n if (lib_name == NULL)\n {\n lib_name = __itt_get_lib_name();\n }\n groups = __itt_get_groups();\n if (DL_SYMBOLS && (groups != __itt_group_none || lib_name != NULL))\n {\n _N_(_ittapi_global).lib = __itt_load_lib((lib_name == NULL) ? ittnotify_lib_name : lib_name);\n\n if (_N_(_ittapi_global).lib != NULL)\n {\n _N_(_ittapi_global).state = __itt_collection_init_successful;\n __itt_api_init_t* __itt_api_init_ptr;\n int lib_version = __itt_lib_version(_N_(_ittapi_global).lib);\n\n switch (lib_version)\n {\n case 0:\n groups = __itt_group_legacy;\n ITT_ATTRIBUTE_FALLTHROUGH;\n case 1:\n /* Fill all pointers from dynamic library */\n for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++)\n {\n if (_N_(_ittapi_global).api_list_ptr[i].group & groups & init_groups)\n {\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = (void*)__itt_get_proc(_N_(_ittapi_global).lib, _N_(_ittapi_global).api_list_ptr[i].name);\n if (*_N_(_ittapi_global).api_list_ptr[i].func_ptr == NULL)\n {\n /* Restore pointers for function with static implementation */\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].null_func;\n __itt_report_error(__itt_error_no_symbol, lib_name, _N_(_ittapi_global).api_list_ptr[i].name);\n#ifdef ITT_COMPLETE_GROUP\n zero_group = (__itt_group_id)(zero_group | _N_(_ittapi_global).api_list_ptr[i].group);\n#endif /* ITT_COMPLETE_GROUP */\n }\n }\n else\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].null_func;\n }\n\n if (groups == __itt_group_legacy)\n {\n /* Compatibility with legacy tools */\n ITTNOTIFY_NAME(thread_ignore) = ITTNOTIFY_NAME(thr_ignore);\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n ITTNOTIFY_NAME(sync_createA) = ITTNOTIFY_NAME(sync_set_nameA);\n ITTNOTIFY_NAME(sync_createW) = ITTNOTIFY_NAME(sync_set_nameW);\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\n ITTNOTIFY_NAME(sync_create) = ITTNOTIFY_NAME(sync_set_name);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n ITTNOTIFY_NAME(sync_prepare) = ITTNOTIFY_NAME(notify_sync_prepare);\n ITTNOTIFY_NAME(sync_cancel) = ITTNOTIFY_NAME(notify_sync_cancel);\n ITTNOTIFY_NAME(sync_acquired) = ITTNOTIFY_NAME(notify_sync_acquired);\n ITTNOTIFY_NAME(sync_releasing) = ITTNOTIFY_NAME(notify_sync_releasing);\n }\n\n#ifdef ITT_COMPLETE_GROUP\n for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++)\n if (_N_(_ittapi_global).api_list_ptr[i].group & zero_group)\n *_N_(_ittapi_global).api_list_ptr[i].func_ptr = _N_(_ittapi_global).api_list_ptr[i].null_func;\n#endif /* ITT_COMPLETE_GROUP */\n break;\n case 2:\n __itt_api_init_ptr = (__itt_api_init_t*)(size_t)__itt_get_proc(_N_(_ittapi_global).lib, \"__itt_api_init\");\n if (__itt_api_init_ptr)\n __itt_api_init_ptr(&_N_(_ittapi_global), init_groups);\n break;\n }\n }\n else\n {\n _N_(_ittapi_global).state = __itt_collection_init_fail;\n __itt_free_allocated_resources();\n __itt_nullify_all_pointers();\n\n __itt_report_error(__itt_error_no_module, lib_name,\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n __itt_system_error()\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n dlerror()\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n );\n }\n }\n else\n {\n _N_(_ittapi_global).state = __itt_collection_collector_absent;\n __itt_nullify_all_pointers();\n }\n _N_(_ittapi_global).api_initialized = 1;\n current_thread = 0;\n /* !!! Just to avoid unused code elimination !!! */\n if (__itt_fini_ittlib_ptr == _N_(fini_ittlib)) current_thread = 0;\n }\n }\n\n#ifndef ITT_SIMPLE_INIT\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n#endif /* ITT_SIMPLE_INIT */\n }\n\n /* Evaluating if any function ptr is non empty and it's in init_groups */\n for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++)\n {\n if (*_N_(_ittapi_global).api_list_ptr[i].func_ptr != _N_(_ittapi_global).api_list_ptr[i].null_func &&\n _N_(_ittapi_global).api_list_ptr[i].group & init_groups)\n {\n return 1;\n }\n }\n return 0;\n}\n\nITT_EXTERN_C __itt_error_handler_t* _N_(set_error_handler)(__itt_error_handler_t* handler)\n{\n __itt_error_handler_t* prev = (__itt_error_handler_t*)(size_t)_N_(_ittapi_global).error_handler;\n _N_(_ittapi_global).error_handler = (void*)(size_t)handler;\n return prev;\n}\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#if _MSC_VER\n#pragma warning(pop)\n#endif\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** __itt_mark_pt_region functions marks region of interest\n * region parameter defines different regions.\n * 0 <= region < 8 */\n\n#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__)\nvoid __itt_pt_mark(__itt_pt_region region);\nvoid __itt_pt_mark_event(__itt_pt_region region);\n#endif\n\nITT_EXTERN_C void _N_(mark_pt_region_begin)(__itt_pt_region region)\n{\n#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__)\n if (_N_(_ittapi_global).ipt_collect_events == 1)\n {\n __itt_pt_mark_event(2*region);\n }\n else\n {\n __itt_pt_mark(2*region);\n }\n#else\n (void)region;\n#endif\n}\n\nITT_EXTERN_C void _N_(mark_pt_region_end)(__itt_pt_region region)\n{\n#if defined(ITT_API_IPT_SUPPORT) && (ITT_PLATFORM==ITT_PLATFORM_WIN || ITT_PLATFORM==ITT_PLATFORM_POSIX) && !defined(__ANDROID__)\n if (_N_(_ittapi_global).ipt_collect_events == 1)\n {\n __itt_pt_mark_event(2*region + 1);\n }\n else\n {\n __itt_pt_mark(2*region + 1);\n }\n#else\n (void)region;\n#endif\n}\n\nITT_EXTERN_C __itt_collection_state (_N_(get_collection_state))(void)\n{\n if (!_N_(_ittapi_global).api_initialized && _N_(_ittapi_global).thread_list == NULL)\n {\n __itt_init_ittlib_name(NULL, __itt_group_all);\n }\n return _N_(_ittapi_global).state;\n}\n\n/* !!! should be called from the library destructor !!!\n * this function destroys the mutex and frees resources\n * allocated by ITT API static part\n */\nITT_EXTERN_C void (_N_(release_resources))(void)\n{\n ITT_MUTEX_INIT_AND_LOCK(_N_(_ittapi_global));\n __itt_free_allocated_resources();\n if (PTHREAD_SYMBOLS) __itt_mutex_unlock(&_N_(_ittapi_global).mutex);\n ITT_MUTEX_DESTROY(_N_(_ittapi_global));\n}\n" }, { "path": "src/tbb/src/tbb/tools_api/ittnotify_static.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"ittnotify_config.h\"\n\n#ifndef ITT_FORMAT_DEFINED\n# ifndef ITT_FORMAT\n# define ITT_FORMAT\n# endif /* ITT_FORMAT */\n# ifndef ITT_NO_PARAMS\n# define ITT_NO_PARAMS\n# endif /* ITT_NO_PARAMS */\n#endif /* ITT_FORMAT_DEFINED */\n\n/*\n * parameters for macro expected:\n * ITT_STUB(api, type, func_name, arguments, params, func_name_in_dll, group, printf_fmt)\n */\n#ifdef __ITT_INTERNAL_INIT\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char *name), (ITT_FORMAT name), domain_createA, __itt_group_structure, \"\\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name), (ITT_FORMAT name), domain_createW, __itt_group_structure, \"\\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name), (ITT_FORMAT name), domain_create, __itt_group_structure, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj), (ITT_FORMAT module_obj), module_load_with_sections, __itt_group_module, \"%p\")\nITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj), (ITT_FORMAT module_obj), module_unload_with_sections, __itt_group_module, \"%p\")\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name), (ITT_FORMAT name), string_handle_createA, __itt_group_structure, \"\\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name), (ITT_FORMAT name), string_handle_createW, __itt_group_structure, \"\\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name), (ITT_FORMAT name), string_handle_create, __itt_group_structure, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_createA, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), counter_createW, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\"\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain), (ITT_FORMAT name, domain), counter_create, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedA, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\", %d\")\nITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typedW, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\", %d\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type), (ITT_FORMAT name, domain, type), counter_create_typed, __itt_group_counter, \"\\\"%s\\\", \\\"%s\\\", %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n\nITT_STUBV(ITTAPI, void, pause, (void), (ITT_NO_PARAMS), pause, __itt_group_control | __itt_group_legacy, \"no args\")\nITT_STUBV(ITTAPI, void, resume, (void), (ITT_NO_PARAMS), resume, __itt_group_control | __itt_group_legacy, \"no args\")\nITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope scope), (ITT_FORMAT scope), pause_scoped, __itt_group_control, \"%d\")\nITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope scope), (ITT_FORMAT scope), resume_scoped, __itt_group_control, \"%d\")\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name), (ITT_FORMAT name), thread_set_nameA, __itt_group_thread, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name), (ITT_FORMAT name), thread_set_nameW, __itt_group_thread, \"\\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, thread_set_name, (const char *name), (ITT_FORMAT name), thread_set_name, __itt_group_thread, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, thread_ignore, (void), (ITT_NO_PARAMS), thread_ignore, __itt_group_thread, \"no args\")\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, int, thr_name_setA, (const char *name, int namelen), (ITT_FORMAT name, namelen), thr_name_setA, __itt_group_thread | __itt_group_legacy, \"\\\"%s\\\", %d\")\nITT_STUB(LIBITTAPI, int, thr_name_setW, (const wchar_t *name, int namelen), (ITT_FORMAT name, namelen), thr_name_setW, __itt_group_thread | __itt_group_legacy, \"\\\"%S\\\", %d\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, int, thr_name_set, (const char *name, int namelen), (ITT_FORMAT name, namelen), thr_name_set, __itt_group_thread | __itt_group_legacy, \"\\\"%s\\\", %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(LIBITTAPI, void, thr_ignore, (void), (ITT_NO_PARAMS), thr_ignore, __itt_group_thread | __itt_group_legacy, \"no args\")\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_createA, __itt_group_structure, \"%p, \\\"%s\\\", %d, %d\")\nITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_createW, __itt_group_structure, \"%p, \\\"%s\\\", %d, %d\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type), (ITT_FORMAT domain, name, x_type, y_type), histogram_create, __itt_group_structure, \"%p, \\\"%s\\\", %d, %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n #if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_createA_v3, __itt_group_counter, \"%p, \\\"%s\\\", %d\")\nITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_createW_v3, __itt_group_counter, \"%p, \\\"%s\\\", %d\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char *name, __itt_metadata_type type), (ITT_FORMAT domain, name, type), counter_create_v3, __itt_group_counter, \"%p, \\\"%s\\\", %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata), (ITT_FORMAT counter, length, metadata), bind_context_metadata_to_counter, __itt_group_structure, \"%p, %lu, %p\")\n \n#endif /* __ITT_INTERNAL_BODY */\n\nITT_STUBV(ITTAPI, void, enable_attach, (void), (ITT_NO_PARAMS), enable_attach, __itt_group_all, \"no args\")\n\n#else /* __ITT_INTERNAL_INIT */\n\nITT_STUBV(ITTAPI, void, detach, (void), (ITT_NO_PARAMS), detach, __itt_group_control | __itt_group_legacy, \"no args\")\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char *objtype, const char *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_createA, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%s\\\", \\\"%s\\\", %x\")\nITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_createW, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%S\\\", \\\"%S\\\", %x\")\nITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char *name), (ITT_FORMAT addr, name), sync_renameA, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name), (ITT_FORMAT addr, name), sync_renameW, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_create, (void *addr, const char *objtype, const char *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_create, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%s\\\", \\\"%s\\\", %x\")\nITT_STUBV(ITTAPI, void, sync_rename, (void *addr, const char *name), (ITT_FORMAT addr, name), sync_rename, __itt_group_sync | __itt_group_fsync, \"%p, \\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_destroy, (void *addr), (ITT_FORMAT addr), sync_destroy, __itt_group_sync | __itt_group_fsync, \"%p\")\n\nITT_STUBV(ITTAPI, void, sync_prepare, (void* addr), (ITT_FORMAT addr), sync_prepare, __itt_group_sync, \"%p\")\nITT_STUBV(ITTAPI, void, sync_cancel, (void *addr), (ITT_FORMAT addr), sync_cancel, __itt_group_sync, \"%p\")\nITT_STUBV(ITTAPI, void, sync_acquired, (void *addr), (ITT_FORMAT addr), sync_acquired, __itt_group_sync, \"%p\")\nITT_STUBV(ITTAPI, void, sync_releasing, (void* addr), (ITT_FORMAT addr), sync_releasing, __itt_group_sync, \"%p\")\n\nITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask), (ITT_FORMAT mask), suppress_push, __itt_group_suppress, \"%p\")\nITT_STUBV(ITTAPI, void, suppress_pop, (void), (ITT_NO_PARAMS), suppress_pop, __itt_group_suppress, \"no args\")\nITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size),(ITT_FORMAT mode, mask, address, size), suppress_mark_range, __itt_group_suppress, \"%d, %p, %p, %d\")\nITT_STUBV(ITTAPI, void, suppress_clear_range,(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size),(ITT_FORMAT mode, mask, address, size), suppress_clear_range,__itt_group_suppress, \"%d, %p, %p, %d\")\n\nITT_STUBV(ITTAPI, void, fsync_prepare, (void* addr), (ITT_FORMAT addr), sync_prepare, __itt_group_fsync, \"%p\")\nITT_STUBV(ITTAPI, void, fsync_cancel, (void *addr), (ITT_FORMAT addr), sync_cancel, __itt_group_fsync, \"%p\")\nITT_STUBV(ITTAPI, void, fsync_acquired, (void *addr), (ITT_FORMAT addr), sync_acquired, __itt_group_fsync, \"%p\")\nITT_STUBV(ITTAPI, void, fsync_releasing, (void* addr), (ITT_FORMAT addr), sync_releasing, __itt_group_fsync, \"%p\")\n\nITT_STUBV(ITTAPI, void, model_site_begin, (__itt_model_site *site, __itt_model_site_instance *instance, const char *name), (ITT_FORMAT site, instance, name), model_site_begin, __itt_group_model, \"%p, %p, \\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_site_end, (__itt_model_site *site, __itt_model_site_instance *instance), (ITT_FORMAT site, instance), model_site_end, __itt_group_model, \"%p, %p\")\nITT_STUBV(ITTAPI, void, model_task_begin, (__itt_model_task *task, __itt_model_task_instance *instance, const char *name), (ITT_FORMAT task, instance, name), model_task_begin, __itt_group_model, \"%p, %p, \\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_task_end, (__itt_model_task *task, __itt_model_task_instance *instance), (ITT_FORMAT task, instance), model_task_end, __itt_group_model, \"%p, %p\")\nITT_STUBV(ITTAPI, void, model_lock_acquire, (void *lock), (ITT_FORMAT lock), model_lock_acquire, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_lock_release, (void *lock), (ITT_FORMAT lock), model_lock_release, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_record_allocation, (void *addr, size_t size), (ITT_FORMAT addr, size), model_record_allocation, __itt_group_model, \"%p, %d\")\nITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr), (ITT_FORMAT addr), model_record_deallocation, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_induction_uses, (void* addr, size_t size), (ITT_FORMAT addr, size), model_induction_uses, __itt_group_model, \"%p, %d\")\nITT_STUBV(ITTAPI, void, model_reduction_uses, (void* addr, size_t size), (ITT_FORMAT addr, size), model_reduction_uses, __itt_group_model, \"%p, %d\")\nITT_STUBV(ITTAPI, void, model_observe_uses, (void* addr, size_t size), (ITT_FORMAT addr, size), model_observe_uses, __itt_group_model, \"%p, %d\")\nITT_STUBV(ITTAPI, void, model_clear_uses, (void* addr), (ITT_FORMAT addr), model_clear_uses, __itt_group_model, \"%p\")\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, model_site_beginW, (const wchar_t *name), (ITT_FORMAT name), model_site_beginW, __itt_group_model, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_task_beginW, (const wchar_t *name), (ITT_FORMAT name), model_task_beginW, __itt_group_model, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_iteration_taskW, (const wchar_t *name), (ITT_FORMAT name), model_iteration_taskW, __itt_group_model, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, model_site_beginA, (const char *name), (ITT_FORMAT name), model_site_beginA, __itt_group_model, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_site_beginAL, (const char *name, size_t len), (ITT_FORMAT name, len), model_site_beginAL, __itt_group_model, \"\\\"%s\\\", %d\")\nITT_STUBV(ITTAPI, void, model_task_beginA, (const char *name), (ITT_FORMAT name), model_task_beginA, __itt_group_model, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_task_beginAL, (const char *name, size_t len), (ITT_FORMAT name, len), model_task_beginAL, __itt_group_model, \"\\\"%s\\\", %d\")\nITT_STUBV(ITTAPI, void, model_iteration_taskA, (const char *name), (ITT_FORMAT name), model_iteration_taskA, __itt_group_model, \"\\\"%s\\\"\")\nITT_STUBV(ITTAPI, void, model_iteration_taskAL, (const char *name, size_t len), (ITT_FORMAT name, len), model_iteration_taskAL, __itt_group_model, \"\\\"%s\\\", %d\")\nITT_STUBV(ITTAPI, void, model_site_end_2, (void), (ITT_NO_PARAMS), model_site_end_2, __itt_group_model, \"no args\")\nITT_STUBV(ITTAPI, void, model_task_end_2, (void), (ITT_NO_PARAMS), model_task_end_2, __itt_group_model, \"no args\")\nITT_STUBV(ITTAPI, void, model_lock_acquire_2, (void *lock), (ITT_FORMAT lock), model_lock_acquire_2, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_lock_release_2, (void *lock), (ITT_FORMAT lock), model_lock_release_2, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_aggregate_task, (size_t count), (ITT_FORMAT count), model_aggregate_task, __itt_group_model, \"%d\")\nITT_STUBV(ITTAPI, void, model_disable_push, (__itt_model_disable x), (ITT_FORMAT x), model_disable_push, __itt_group_model, \"%p\")\nITT_STUBV(ITTAPI, void, model_disable_pop, (void), (ITT_NO_PARAMS), model_disable_pop, __itt_group_model, \"no args\")\n#endif /* __ITT_INTERNAL_BODY */\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char *name, const char *domain), (ITT_FORMAT name, domain), heap_function_createA, __itt_group_heap, \"\\\"%s\\\", \\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t *name, const wchar_t *domain), (ITT_FORMAT name, domain), heap_function_createW, __itt_group_heap, \"\\\"%s\\\", \\\"%s\\\"\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_heap_function, heap_function_create, (const char *name, const char *domain), (ITT_FORMAT name, domain), heap_function_create, __itt_group_heap, \"\\\"%s\\\", \\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* __ITT_INTERNAL_BODY */\nITT_STUBV(ITTAPI, void, heap_allocate_begin, (__itt_heap_function h, size_t size, int initialized), (ITT_FORMAT h, size, initialized), heap_allocate_begin, __itt_group_heap, \"%p, %lu, %d\")\nITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, size_t size, int initialized), (ITT_FORMAT h, addr, size, initialized), heap_allocate_end, __itt_group_heap, \"%p, %p, %lu, %d\")\nITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr), (ITT_FORMAT h, addr), heap_free_begin, __itt_group_heap, \"%p, %p\")\nITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr), (ITT_FORMAT h, addr), heap_free_end, __itt_group_heap, \"%p, %p\")\nITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* addr, size_t new_size, int initialized), (ITT_FORMAT h, addr, new_size, initialized), heap_reallocate_begin, __itt_group_heap, \"%p, %p, %lu, %d\")\nITT_STUBV(ITTAPI, void, heap_reallocate_end, (__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized), (ITT_FORMAT h, addr, new_addr, new_size, initialized), heap_reallocate_end, __itt_group_heap, \"%p, %p, %p, %lu, %d\")\nITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void), (ITT_NO_PARAMS), heap_internal_access_begin, __itt_group_heap, \"no args\")\nITT_STUBV(ITTAPI, void, heap_internal_access_end, (void), (ITT_NO_PARAMS), heap_internal_access_end, __itt_group_heap, \"no args\")\nITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void), (ITT_NO_PARAMS), heap_record_memory_growth_begin, __itt_group_heap, \"no args\")\nITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void), (ITT_NO_PARAMS), heap_record_memory_growth_end, __itt_group_heap, \"no args\")\nITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask), (ITT_FORMAT reset_mask), heap_reset_detection, __itt_group_heap, \"%u\")\nITT_STUBV(ITTAPI, void, heap_record, (unsigned int record_mask), (ITT_FORMAT record_mask), heap_record, __itt_group_heap, \"%u\")\n\nITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), id_create, __itt_group_structure, \"%p, %lu\")\nITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), id_destroy, __itt_group_structure, \"%p, %lu\")\n\nITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void), (ITT_NO_PARAMS), get_timestamp, __itt_group_structure, \"no args\")\n\nITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), region_begin, __itt_group_structure, \"%p, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, region_end, (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), region_end, __itt_group_structure, \"%p, %lu\")\n\n#ifndef __ITT_INTERNAL_BODY\nITT_STUBV(ITTAPI, void, frame_begin_v3, (const __itt_domain *domain, __itt_id *id), (ITT_FORMAT domain, id), frame_begin_v3, __itt_group_structure, \"%p, %p\")\nITT_STUBV(ITTAPI, void, frame_end_v3, (const __itt_domain *domain, __itt_id *id), (ITT_FORMAT domain, id), frame_end_v3, __itt_group_structure, \"%p, %p\")\nITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end), (ITT_FORMAT domain, id, begin, end), frame_submit_v3, __itt_group_structure, \"%p, %p, %lu, %lu\")\n#endif /* __ITT_INTERNAL_BODY */\n\nITT_STUBV(ITTAPI, void, task_group, (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), task_group, __itt_group_structure, \"%p, %lu, %lu, %p\")\n\nITT_STUBV(ITTAPI, void, task_begin, (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), task_begin, __itt_group_structure, \"%p, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parent, void* fn), (ITT_FORMAT domain, id, parent, fn), task_begin_fn, __itt_group_structure, \"%p, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain), (ITT_FORMAT domain), task_end, __itt_group_structure, \"%p\")\n\nITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name), (ITT_FORMAT domain, name), counter_inc_v3, __itt_group_structure, \"%p, %p\")\nITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_inc_delta_v3, __itt_group_structure, \"%p, %p, %lu\")\nITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name), (ITT_FORMAT domain, name), counter_dec_v3, __itt_group_structure, \"%p, %p\")\nITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long value), (ITT_FORMAT domain, name, value), counter_dec_delta_v3, __itt_group_structure, \"%p, %p, %lu\")\n\nITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope), (ITT_FORMAT domain, id, name, scope), marker, __itt_group_structure, \"%p, %lu, %p, %d\")\n\nITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data), (ITT_FORMAT domain, id, key, type, count, data), metadata_add, __itt_group_structure, \"%p, %lu, %p, %d, %lu, %p\")\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char* data, size_t length), (ITT_FORMAT domain, id, key, data, length), metadata_str_addA, __itt_group_structure, \"%p, %lu, %p, %p, %lu\")\nITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t* data, size_t length), (ITT_FORMAT domain, id, key, data, length), metadata_str_addW, __itt_group_structure, \"%p, %lu, %p, %p, %lu\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char* data, size_t length), (ITT_FORMAT domain, id, key, data, length), metadata_str_add, __itt_group_structure, \"%p, %lu, %p, %p, %lu\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, relation, tail), relation_add_to_current, __itt_group_structure, \"%p, %lu, %p\")\nITT_STUBV(ITTAPI, void, relation_add, (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, head, relation, tail), relation_add, __itt_group_structure, \"%p, %p, %lu, %p\")\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen), (ITT_FORMAT name, namelen), event_createA, __itt_group_mark | __itt_group_legacy, \"\\\"%s\\\", %d\")\nITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen), (ITT_FORMAT name, namelen), event_createW, __itt_group_mark | __itt_group_legacy, \"\\\"%S\\\", %d\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen), (ITT_FORMAT name, namelen), event_create, __itt_group_mark | __itt_group_legacy, \"\\\"%s\\\", %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, int, event_start, (__itt_event event), (ITT_FORMAT event), event_start, __itt_group_mark | __itt_group_legacy, \"%d\")\nITT_STUB(LIBITTAPI, int, event_end, (__itt_event event), (ITT_FORMAT event), event_end, __itt_group_mark | __itt_group_legacy, \"%d\")\n#endif /* __ITT_INTERNAL_BODY */\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, sync_set_nameA, (void *addr, const char *objtype, const char *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_nameA, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p, \\\"%s\\\", \\\"%s\\\", %x\")\nITT_STUBV(ITTAPI, void, sync_set_nameW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_nameW, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p, \\\"%S\\\", \\\"%S\\\", %x\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_set_name, (void *addr, const char *objtype, const char *objname, int attribute), (ITT_FORMAT addr, objtype, objname, attribute), sync_set_name, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"p, \\\"%s\\\", \\\"%s\\\", %x\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, int, notify_sync_nameA, (void *p, const char *objtype, int typelen, const char *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_nameA, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p, \\\"%s\\\", %d, \\\"%s\\\", %d, %x\")\nITT_STUB(LIBITTAPI, int, notify_sync_nameW, (void *p, const wchar_t *objtype, int typelen, const wchar_t *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_nameW, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p, \\\"%S\\\", %d, \\\"%S\\\", %d, %x\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, int, notify_sync_name, (void *p, const char *objtype, int typelen, const char *objname, int namelen, int attribute), (ITT_FORMAT p, objtype, typelen, objname, namelen, attribute), notify_sync_name, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p, \\\"%s\\\", %d, \\\"%s\\\", %d, %x\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\nITT_STUBV(LIBITTAPI, void, notify_sync_prepare, (void *p), (ITT_FORMAT p), notify_sync_prepare, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p\")\nITT_STUBV(LIBITTAPI, void, notify_sync_cancel, (void *p), (ITT_FORMAT p), notify_sync_cancel, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p\")\nITT_STUBV(LIBITTAPI, void, notify_sync_acquired, (void *p), (ITT_FORMAT p), notify_sync_acquired, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p\")\nITT_STUBV(LIBITTAPI, void, notify_sync_releasing, (void *p), (ITT_FORMAT p), notify_sync_releasing, __itt_group_sync | __itt_group_fsync | __itt_group_legacy, \"%p\")\n#endif /* __ITT_INTERNAL_BODY */\n\nITT_STUBV(LIBITTAPI, void, memory_read, (void *addr, size_t size), (ITT_FORMAT addr, size), memory_read, __itt_group_legacy, \"%p, %lu\")\nITT_STUBV(LIBITTAPI, void, memory_write, (void *addr, size_t size), (ITT_FORMAT addr, size), memory_write, __itt_group_legacy, \"%p, %lu\")\nITT_STUBV(LIBITTAPI, void, memory_update, (void *addr, size_t size), (ITT_FORMAT addr, size), memory_update, __itt_group_legacy, \"%p, %lu\")\n\nITT_STUB(LIBITTAPI, __itt_state_t, state_get, (void), (ITT_NO_PARAMS), state_get, __itt_group_legacy, \"no args\")\nITT_STUB(LIBITTAPI, __itt_state_t, state_set, (__itt_state_t s), (ITT_FORMAT s), state_set, __itt_group_legacy, \"%d\")\nITT_STUB(LIBITTAPI, __itt_obj_state_t, obj_mode_set, (__itt_obj_prop_t p, __itt_obj_state_t s), (ITT_FORMAT p, s), obj_mode_set, __itt_group_legacy, \"%d, %d\")\nITT_STUB(LIBITTAPI, __itt_thr_state_t, thr_mode_set, (__itt_thr_prop_t p, __itt_thr_state_t s), (ITT_FORMAT p, s), thr_mode_set, __itt_group_legacy, \"%d, %d\")\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_frame, frame_createA, (const char *domain), (ITT_FORMAT domain), frame_createA, __itt_group_frame, \"\\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_frame, frame_createW, (const wchar_t *domain), (ITT_FORMAT domain), frame_createW, __itt_group_frame, \"\\\"%s\\\"\")\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_frame, frame_create, (const char *domain), (ITT_FORMAT domain), frame_create, __itt_group_frame, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name), (ITT_FORMAT name), pt_region_createA, __itt_group_structure, \"\\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name), (ITT_FORMAT name), pt_region_createW, __itt_group_structure, \"\\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name), (ITT_FORMAT name), pt_region_create, __itt_group_structure, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* __ITT_INTERNAL_BODY */\nITT_STUBV(ITTAPI, void, frame_begin, (__itt_frame frame), (ITT_FORMAT frame), frame_begin, __itt_group_frame, \"%p\")\nITT_STUBV(ITTAPI, void, frame_end, (__itt_frame frame), (ITT_FORMAT frame), frame_end, __itt_group_frame, \"%p\")\n\nITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id), (ITT_FORMAT id), counter_destroy, __itt_group_counter, \"%p\")\nITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id), (ITT_FORMAT id), counter_inc, __itt_group_counter, \"%p\")\nITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value), (ITT_FORMAT id, value), counter_inc_delta, __itt_group_counter, \"%p, %lu\")\nITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id), (ITT_FORMAT id), counter_dec, __itt_group_counter, \"%p\")\nITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value), (ITT_FORMAT id, value), counter_dec_delta, __itt_group_counter, \"%p, %lu\")\nITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr), (ITT_FORMAT id, value_ptr), counter_set_value, __itt_group_counter, \"%p, %p\")\nITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr), (ITT_FORMAT id, clock_domain, timestamp, value_ptr), counter_set_value_ex, __itt_group_counter, \"%p, %p, %llu, %p\")\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char *name), (ITT_FORMAT name), mark_createA, __itt_group_mark, \"\\\"%s\\\"\")\nITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name), (ITT_FORMAT name), mark_createW, __itt_group_mark, \"\\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_mark_type, mark_create, (const char *name), (ITT_FORMAT name), mark_create, __itt_group_mark, \"\\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* __ITT_INTERNAL_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, markA, (__itt_mark_type mt, const char *parameter), (ITT_FORMAT mt, parameter), markA, __itt_group_mark, \"%d, \\\"%s\\\"\")\nITT_STUB(ITTAPI, int, markW, (__itt_mark_type mt, const wchar_t *parameter), (ITT_FORMAT mt, parameter), markW, __itt_group_mark, \"%d, \\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark, (__itt_mark_type mt, const char *parameter), (ITT_FORMAT mt, parameter), mark, __itt_group_mark, \"%d, \\\"%s\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark_off, (__itt_mark_type mt), (ITT_FORMAT mt), mark_off, __itt_group_mark, \"%d\")\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, mark_globalA, (__itt_mark_type mt, const char *parameter), (ITT_FORMAT mt, parameter), mark_globalA, __itt_group_mark, \"%d, \\\"%s\\\"\")\nITT_STUB(ITTAPI, int, mark_globalW, (__itt_mark_type mt, const wchar_t *parameter), (ITT_FORMAT mt, parameter), mark_globalW, __itt_group_mark, \"%d, \\\"%S\\\"\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark_global, (__itt_mark_type mt, const char *parameter), (ITT_FORMAT mt, parameter), mark_global, __itt_group_mark, \"%d, \\\"%S\\\"\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt), (ITT_FORMAT mt), mark_global_off, __itt_group_mark, \"%d\")\n\n#ifndef __ITT_INTERNAL_BODY\nITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void), (ITT_NO_PARAMS), stack_caller_create, __itt_group_stitch, \"no args\")\n#endif /* __ITT_INTERNAL_BODY */\nITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id), (ITT_FORMAT id), stack_caller_destroy, __itt_group_stitch, \"%p\")\nITT_STUBV(ITTAPI, void, stack_callee_enter, (__itt_caller id), (ITT_FORMAT id), stack_callee_enter, __itt_group_stitch, \"%p\")\nITT_STUBV(ITTAPI, void, stack_callee_leave, (__itt_caller id), (ITT_FORMAT id), stack_callee_leave, __itt_group_stitch, \"%p\")\n\nITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data), (ITT_FORMAT fn, fn_data), clock_domain_create, __itt_group_structure, \"%p, %p\")\nITT_STUBV(ITTAPI, void, clock_domain_reset, (void), (ITT_NO_PARAMS), clock_domain_reset, __itt_group_structure, \"no args\")\nITT_STUBV(ITTAPI, void, id_create_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id), (ITT_FORMAT domain, clock_domain, timestamp, id), id_create_ex, __itt_group_structure, \"%p, %p, %lu, %lu\")\nITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id), (ITT_FORMAT domain, clock_domain, timestamp, id), id_destroy_ex, __itt_group_structure, \"%p, %p, %lu, %lu\")\nITT_STUBV(ITTAPI, void, task_begin_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name), (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, name), task_begin_ex, __itt_group_structure, \"%p, %p, %lu, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn), (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, fn), task_begin_fn_ex, __itt_group_structure, \"%p, %p, %lu, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp), (ITT_FORMAT domain, clock_domain, timestamp), task_end_ex, __itt_group_structure, \"%p, %p, %lu\")\nITT_STUBV(ITTAPI, void, task_begin_overlapped, (const __itt_domain *domain, __itt_id id, __itt_id parent, __itt_string_handle *name), (ITT_FORMAT domain, id, parent, name), task_begin_overlapped, __itt_group_structure, \"%p, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_begin_overlapped_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name), (ITT_FORMAT domain, clock_domain, timestamp, id, parentid, name), task_begin_overlapped_ex, __itt_group_structure, \"%p, %p, %lu, %lu, %lu, %p\")\nITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id id), (ITT_FORMAT domain, id), task_end_overlapped, __itt_group_structure, \"%p, %lu\")\nITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id), (ITT_FORMAT domain, clock_domain, timestamp, id), task_end_overlapped_ex, __itt_group_structure, \"%p, %p, %lu, %lu\")\nITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope), (ITT_FORMAT domain, clock_domain, timestamp, id, name, scope), marker_ex, __itt_group_structure, \"%p, %p, %lu, %lu, %p, %d\")\nITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data), (ITT_FORMAT domain, scope, key, type, count, data), metadata_add_with_scope, __itt_group_structure, \"%p, %d, %p, %d, %lu, %p\")\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length), (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scopeA, __itt_group_structure, \"%p, %d, %p, %p, %lu\")\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length), (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scopeW, __itt_group_structure, \"%p, %d, %p, %p, %lu\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length), (ITT_FORMAT domain, scope, key, data, length), metadata_str_add_with_scope, __itt_group_structure, \"%p, %d, %p, %p, %lu\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, clock_domain, timestamp, relation, tail), relation_add_to_current_ex, __itt_group_structure, \"%p, %p, %lu, %d, %lu\")\nITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail), (ITT_FORMAT domain, clock_domain, timestamp, head, relation, tail), relation_add_ex, __itt_group_structure, \"%p, %p, %lu, %lu, %d, %lu\")\nITT_STUB(ITTAPI, __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type), (ITT_FORMAT name, track_group_type), track_group_create, __itt_group_structure, \"%p, %d\")\nITT_STUB(ITTAPI, __itt_track*, track_create, (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type), (ITT_FORMAT track_group, name, track_type), track_create, __itt_group_structure, \"%p, %p, %d\")\nITT_STUBV(ITTAPI, void, set_track, (__itt_track *track), (ITT_FORMAT track), set_track, __itt_group_structure, \"%p\")\n\n#ifndef __ITT_INTERNAL_BODY\nITT_STUB(ITTAPI, const char*, api_version, (void), (ITT_NO_PARAMS), api_version, __itt_group_all & ~__itt_group_legacy, \"no args\")\n#endif /* __ITT_INTERNAL_BODY */\n\n#ifndef __ITT_INTERNAL_BODY\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_saveA, __itt_group_arrays, \"%p, %d, %p, %d, \\\"%s\\\", %d\")\nITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_saveW, __itt_group_arrays, \"%p, %d, %p, %d, \\\"%S\\\", %d\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder), (ITT_FORMAT data, rank, dimensions, type, filePath, columnOrder), av_save, __itt_group_arrays, \"%p, %d, %p, %d, \\\"%s\\\", %d\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* __ITT_INTERNAL_BODY */\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, module_loadA, (void *start_addr, void* end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_loadA, __itt_group_module, \"%p, %p, %p\")\nITT_STUBV(ITTAPI, void, module_loadW, (void *start_addr, void* end_addr, const wchar_t *path), (ITT_FORMAT start_addr, end_addr, path), module_loadW, __itt_group_module, \"%p, %p, %p\")\n#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path), (ITT_FORMAT start_addr, end_addr, path), module_load, __itt_group_module, \"%p, %p, %p\")\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, module_unload, (void *start_addr), (ITT_FORMAT start_addr), module_unload, __itt_group_module, \"%p\")\n\nITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* histogram, size_t length, void* x_data, void* y_data), (ITT_FORMAT histogram, length, x_data, y_data), histogram_submit, __itt_group_structure, \"%p, %lu, %p, %p\")\n\nITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr), (ITT_FORMAT counter, value_ptr), counter_set_value_v3, __itt_group_counter, \"%p, %p\")\n \n#endif /* __ITT_INTERNAL_INIT */\n" }, { "path": "src/tbb/src/tbb/tools_api/ittnotify_types.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _ITTNOTIFY_TYPES_H_\n#define _ITTNOTIFY_TYPES_H_\n\ntypedef enum ___itt_group_id\n{\n __itt_group_none \t\t= 0,\n __itt_group_legacy \t\t= 1<<0,\n __itt_group_control \t\t= 1<<1,\n __itt_group_thread \t\t= 1<<2,\n __itt_group_mark \t\t= 1<<3,\n __itt_group_sync \t\t= 1<<4,\n __itt_group_fsync \t\t= 1<<5,\n __itt_group_jit \t\t= 1<<6,\n __itt_group_model \t\t= 1<<7,\n __itt_group_splitter_min \t= 1<<7,\n __itt_group_counter \t\t= 1<<8,\n __itt_group_frame \t\t= 1<<9,\n __itt_group_stitch \t\t= 1<<10,\n __itt_group_heap \t\t= 1<<11,\n __itt_group_splitter_max \t= 1<<12,\n __itt_group_structure \t\t= 1<<12,\n __itt_group_suppress \t\t= 1<<13,\n __itt_group_arrays \t\t= 1<<14,\n __itt_group_module \t\t= 1<<15,\n __itt_group_all \t\t= -1\n} __itt_group_id;\n\n#pragma pack(push, 8)\n\ntypedef struct ___itt_group_list\n{\n __itt_group_id id;\n const char* name;\n} __itt_group_list;\n\n#pragma pack(pop)\n\n#define ITT_GROUP_LIST(varname) \\\n static __itt_group_list varname[] = { \\\n { __itt_group_all, \"all\" }, \\\n { __itt_group_control, \"control\" }, \\\n { __itt_group_thread, \"thread\" }, \\\n { __itt_group_mark, \"mark\" }, \\\n { __itt_group_sync, \"sync\" }, \\\n { __itt_group_fsync, \"fsync\" }, \\\n { __itt_group_jit, \"jit\" }, \\\n { __itt_group_model, \"model\" }, \\\n { __itt_group_counter, \"counter\" }, \\\n { __itt_group_frame, \"frame\" }, \\\n { __itt_group_stitch, \"stitch\" }, \\\n { __itt_group_heap, \"heap\" }, \\\n { __itt_group_structure, \"structure\" }, \\\n { __itt_group_suppress, \"suppress\" }, \\\n { __itt_group_arrays, \"arrays\" }, \\\n\t\t{ __itt_group_module, \"module\" }, \\\n { __itt_group_none, NULL } \\\n }\n\n#endif /* _ITTNOTIFY_TYPES_H_ */\n" }, { "path": "src/tbb/src/tbb/tools_api/legacy/ittnotify.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _LEGACY_ITTNOTIFY_H_\n#define _LEGACY_ITTNOTIFY_H_\n\n/**\n * @file\n * @brief Legacy User API functions and types\n */\n\n/** @cond exclude_from_documentation */\n#ifndef ITT_OS_WIN\n# define ITT_OS_WIN 1\n#endif /* ITT_OS_WIN */\n\n#ifndef ITT_OS_LINUX\n# define ITT_OS_LINUX 2\n#endif /* ITT_OS_LINUX */\n\n#ifndef ITT_OS_MAC\n# define ITT_OS_MAC 3\n#endif /* ITT_OS_MAC */\n\n#ifndef ITT_OS_FREEBSD\n# define ITT_OS_FREEBSD 4\n#endif /* ITT_OS_FREEBSD */\n\n#ifndef ITT_OS_OPENBSD\n# define ITT_OS_OPENBSD 5\n#endif /* ITT_OS_OPENBSD */\n\n#ifndef ITT_OS\n# if defined WIN32 || defined _WIN32\n# define ITT_OS ITT_OS_WIN\n# elif defined( __APPLE__ ) && defined( __MACH__ )\n# define ITT_OS ITT_OS_MAC\n# elif defined( __FreeBSD__ )\n# define ITT_OS ITT_OS_FREEBSD\n# elif defined( __OpenBSD__ )\n# define ITT_OS ITT_OS_OPENBSD\n# else\n# define ITT_OS ITT_OS_LINUX\n# endif\n#endif /* ITT_OS */\n\n#ifndef ITT_PLATFORM_WIN\n# define ITT_PLATFORM_WIN 1\n#endif /* ITT_PLATFORM_WIN */\n\n#ifndef ITT_PLATFORM_POSIX\n# define ITT_PLATFORM_POSIX 2\n#endif /* ITT_PLATFORM_POSIX */\n\n#ifndef ITT_PLATFORM_MAC\n# define ITT_PLATFORM_MAC 3\n#endif /* ITT_PLATFORM_MAC */\n\n#ifndef ITT_PLATFORM_FREEBSD\n# define ITT_PLATFORM_FREEBSD 4\n#endif /* ITT_PLATFORM_FREEBSD */\n\n#ifndef ITT_PLATFORM_OPENBSD\n# define ITT_PLATFORM_OPENBSD 5\n#endif /* ITT_PLATFORM_OPENBSD */\n\n#ifndef ITT_PLATFORM\n# if ITT_OS==ITT_OS_WIN\n# define ITT_PLATFORM ITT_PLATFORM_WIN\n# elif ITT_OS==ITT_OS_MAC\n# define ITT_PLATFORM ITT_PLATFORM_MAC\n# elif ITT_OS==ITT_OS_FREEBSD\n# define ITT_PLATFORM ITT_PLATFORM_FREEBSD\n# elif ITT_OS==ITT_OS_OPENBSD\n# define ITT_PLATFORM ITT_PLATFORM_OPENBSD\n# else\n# define ITT_PLATFORM ITT_PLATFORM_POSIX\n# endif\n#endif /* ITT_PLATFORM */\n\n#if defined(_UNICODE) && !defined(UNICODE)\n#define UNICODE\n#endif\n\n#include \n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#include \n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#include \n#if defined(UNICODE) || defined(_UNICODE)\n#include \n#endif /* UNICODE || _UNICODE */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n#ifndef ITTAPI_CDECL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define ITTAPI_CDECL __cdecl\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define ITTAPI_CDECL __attribute__ ((cdecl))\n# else /* _M_IX86 || __i386__ */\n# define ITTAPI_CDECL /* actual only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* ITTAPI_CDECL */\n\n#ifndef STDCALL\n# if ITT_PLATFORM==ITT_PLATFORM_WIN\n# define STDCALL __stdcall\n# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n# if defined _M_IX86 || defined __i386__\n# define STDCALL __attribute__ ((stdcall))\n# else /* _M_IX86 || __i386__ */\n# define STDCALL /* supported only on x86 platform */\n# endif /* _M_IX86 || __i386__ */\n# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* STDCALL */\n\n#define ITTAPI ITTAPI_CDECL\n#define LIBITTAPI ITTAPI_CDECL\n\n/* TODO: Temporary for compatibility! */\n#define ITTAPI_CALL ITTAPI_CDECL\n#define LIBITTAPI_CALL ITTAPI_CDECL\n\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n/* use __forceinline (VC++ specific) */\n#if defined(__MINGW32__) && !defined(__cplusplus)\n#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__))\n#else\n#define ITT_INLINE static __forceinline\n#endif /* __MINGW32__ */\n\n#define ITT_INLINE_ATTRIBUTE /* nothing */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/*\n * Generally, functions are not inlined unless optimization is specified.\n * For functions declared inline, this attribute inlines the function even\n * if no optimization level was specified.\n */\n#ifdef __STRICT_ANSI__\n#define ITT_INLINE static\n#define ITT_INLINE_ATTRIBUTE __attribute__((unused))\n#else /* __STRICT_ANSI__ */\n#define ITT_INLINE static inline\n#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused))\n#endif /* __STRICT_ANSI__ */\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n/** @endcond */\n\n/** @cond exclude_from_documentation */\n/* Helper macro for joining tokens */\n#define ITT_JOIN_AUX(p,n) p##n\n#define ITT_JOIN(p,n) ITT_JOIN_AUX(p,n)\n\n#ifdef ITT_MAJOR\n#undef ITT_MAJOR\n#endif\n#ifdef ITT_MINOR\n#undef ITT_MINOR\n#endif\n#define ITT_MAJOR 3\n#define ITT_MINOR 0\n\n/* Standard versioning of a token with major and minor version numbers */\n#define ITT_VERSIONIZE(x) \\\n ITT_JOIN(x, \\\n ITT_JOIN(_, \\\n ITT_JOIN(ITT_MAJOR, \\\n ITT_JOIN(_, ITT_MINOR))))\n\n#ifndef INTEL_ITTNOTIFY_PREFIX\n# define INTEL_ITTNOTIFY_PREFIX __itt_\n#endif /* INTEL_ITTNOTIFY_PREFIX */\n#ifndef INTEL_ITTNOTIFY_POSTFIX\n# define INTEL_ITTNOTIFY_POSTFIX _ptr_\n#endif /* INTEL_ITTNOTIFY_POSTFIX */\n\n#define ITTNOTIFY_NAME_AUX(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n)\n#define ITTNOTIFY_NAME(n) ITT_VERSIONIZE(ITTNOTIFY_NAME_AUX(ITT_JOIN(n,INTEL_ITTNOTIFY_POSTFIX)))\n\n#define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)\n#define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)\n\n#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d)\n#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x)\n#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y)\n#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z)\n#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a)\n#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b)\n#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c)\n#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d)\n#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x)\n#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y)\n#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z)\n#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a)\n#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b)\n#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c)\n\n#ifdef ITT_STUB\n#undef ITT_STUB\n#endif\n#ifdef ITT_STUBV\n#undef ITT_STUBV\n#endif\n#define ITT_STUBV(api,type,name,args) \\\n typedef type (api* ITT_JOIN(ITTNOTIFY_NAME(name),_t)) args; \\\n extern ITT_JOIN(ITTNOTIFY_NAME(name),_t) ITTNOTIFY_NAME(name);\n#define ITT_STUB ITT_STUBV\n/** @endcond */\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif /* __cplusplus */\n\n/**\n * @defgroup legacy Legacy API\n * @{\n * @}\n */\n\n/**\n * @defgroup legacy_control Collection Control\n * @ingroup legacy\n * General behavior: application continues to run, but no profiling information is being collected\n *\n * Pausing occurs not only for the current thread but for all process as well as spawned processes\n * - Intel(R) Parallel Inspector and Intel(R) Inspector XE:\n * - Does not analyze or report errors that involve memory access.\n * - Other errors are reported as usual. Pausing data collection in\n * Intel(R) Parallel Inspector and Intel(R) Inspector XE\n * only pauses tracing and analyzing memory access.\n * It does not pause tracing or analyzing threading APIs.\n * .\n * - Intel(R) VTune(TM) Profiler:\n * - Does continue to record when new threads are started.\n * .\n * - Other effects:\n * - Possible reduction of runtime overhead.\n * .\n * @{\n */\n#ifndef _ITTNOTIFY_H_\n/** @brief Pause collection */\nvoid ITTAPI __itt_pause(void);\n/** @brief Resume collection */\nvoid ITTAPI __itt_resume(void);\n/** @brief Detach collection */\nvoid ITTAPI __itt_detach(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, pause, (void))\nITT_STUBV(ITTAPI, void, resume, (void))\nITT_STUBV(ITTAPI, void, detach, (void))\n#define __itt_pause ITTNOTIFY_VOID(pause)\n#define __itt_pause_ptr ITTNOTIFY_NAME(pause)\n#define __itt_resume ITTNOTIFY_VOID(resume)\n#define __itt_resume_ptr ITTNOTIFY_NAME(resume)\n#define __itt_detach ITTNOTIFY_VOID(detach)\n#define __itt_detach_ptr ITTNOTIFY_NAME(detach)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_pause()\n#define __itt_pause_ptr 0\n#define __itt_resume()\n#define __itt_resume_ptr 0\n#define __itt_detach()\n#define __itt_detach_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_pause_ptr 0\n#define __itt_resume_ptr 0\n#define __itt_detach_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n#endif /* _ITTNOTIFY_H_ */\n/** @} legacy_control group */\n\n/**\n * @defgroup legacy_threads Threads\n * @ingroup legacy\n * Threads group\n * @warning Legacy API\n * @{\n */\n/**\n * @deprecated Legacy API\n * @brief Set name to be associated with thread in analysis GUI.\n * @return __itt_err upon failure (name or namelen being null,name and namelen mismatched)\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nint LIBITTAPI __itt_thr_name_setA(const char *name, int namelen);\nint LIBITTAPI __itt_thr_name_setW(const wchar_t *name, int namelen);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_thr_name_set __itt_thr_name_setW\n# define __itt_thr_name_set_ptr __itt_thr_name_setW_ptr\n#else\n# define __itt_thr_name_set __itt_thr_name_setA\n# define __itt_thr_name_set_ptr __itt_thr_name_setA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nint LIBITTAPI __itt_thr_name_set(const char *name, int namelen);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, int, thr_name_setA, (const char *name, int namelen))\nITT_STUB(LIBITTAPI, int, thr_name_setW, (const wchar_t *name, int namelen))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, int, thr_name_set, (const char *name, int namelen))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thr_name_setA ITTNOTIFY_DATA(thr_name_setA)\n#define __itt_thr_name_setA_ptr ITTNOTIFY_NAME(thr_name_setA)\n#define __itt_thr_name_setW ITTNOTIFY_DATA(thr_name_setW)\n#define __itt_thr_name_setW_ptr ITTNOTIFY_NAME(thr_name_setW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thr_name_set ITTNOTIFY_DATA(thr_name_set)\n#define __itt_thr_name_set_ptr ITTNOTIFY_NAME(thr_name_set)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thr_name_setA(name, namelen)\n#define __itt_thr_name_setA_ptr 0\n#define __itt_thr_name_setW(name, namelen)\n#define __itt_thr_name_setW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thr_name_set(name, namelen)\n#define __itt_thr_name_set_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_thr_name_setA_ptr 0\n#define __itt_thr_name_setW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_thr_name_set_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Mark current thread as ignored from this point on, for the duration of its existence.\n */\nvoid LIBITTAPI __itt_thr_ignore(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, thr_ignore, (void))\n#define __itt_thr_ignore ITTNOTIFY_VOID(thr_ignore)\n#define __itt_thr_ignore_ptr ITTNOTIFY_NAME(thr_ignore)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_thr_ignore()\n#define __itt_thr_ignore_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_thr_ignore_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} legacy_threads group */\n\n/**\n * @defgroup legacy_sync Synchronization\n * @ingroup legacy\n * Synchronization group\n * @warning Legacy API\n * @{\n */\n/**\n * @hideinitializer\n * @brief possible value of attribute argument for sync object type\n */\n#define __itt_attr_barrier 1\n\n/**\n * @hideinitializer\n * @brief possible value of attribute argument for sync object type\n */\n#define __itt_attr_mutex 2\n\n/**\n * @deprecated Legacy API\n * @brief Assign a name to a sync object using char or Unicode string\n * @param[in] addr - pointer to the sync object. You should use a real pointer to your object\n * to make sure that the values don't clash with other object addresses\n * @param[in] objtype - null-terminated object type string. If NULL is passed, the object will\n * be assumed to be of generic \"User Synchronization\" type\n * @param[in] objname - null-terminated object name string. If NULL, no name will be assigned\n * to the object -- you can use the __itt_sync_rename call later to assign\n * the name\n * @param[in] attribute - one of [#__itt_attr_barrier, #__itt_attr_mutex] values which defines the\n * exact semantics of how prepare/acquired/releasing calls work.\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nvoid ITTAPI __itt_sync_set_nameA(void *addr, const char *objtype, const char *objname, int attribute);\nvoid ITTAPI __itt_sync_set_nameW(void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_sync_set_name __itt_sync_set_nameW\n# define __itt_sync_set_name_ptr __itt_sync_set_nameW_ptr\n#else /* UNICODE */\n# define __itt_sync_set_name __itt_sync_set_nameA\n# define __itt_sync_set_name_ptr __itt_sync_set_nameA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nvoid ITTAPI __itt_sync_set_name(void *addr, const char* objtype, const char* objname, int attribute);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUBV(ITTAPI, void, sync_set_nameA, (void *addr, const char *objtype, const char *objname, int attribute))\nITT_STUBV(ITTAPI, void, sync_set_nameW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUBV(ITTAPI, void, sync_set_name, (void *addr, const char *objtype, const char *objname, int attribute))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_set_nameA ITTNOTIFY_VOID(sync_set_nameA)\n#define __itt_sync_set_nameA_ptr ITTNOTIFY_NAME(sync_set_nameA)\n#define __itt_sync_set_nameW ITTNOTIFY_VOID(sync_set_nameW)\n#define __itt_sync_set_nameW_ptr ITTNOTIFY_NAME(sync_set_nameW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_set_name ITTNOTIFY_VOID(sync_set_name)\n#define __itt_sync_set_name_ptr ITTNOTIFY_NAME(sync_set_name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_set_nameA(addr, objtype, objname, attribute)\n#define __itt_sync_set_nameA_ptr 0\n#define __itt_sync_set_nameW(addr, objtype, objname, attribute)\n#define __itt_sync_set_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_set_name(addr, objtype, objname, attribute)\n#define __itt_sync_set_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_sync_set_nameA_ptr 0\n#define __itt_sync_set_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_sync_set_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Assign a name and type to a sync object using char or Unicode string\n * @param[in] addr - pointer to the sync object. You should use a real pointer to your object\n * to make sure that the values don't clash with other object addresses\n * @param[in] objtype - null-terminated object type string. If NULL is passed, the object will\n * be assumed to be of generic \"User Synchronization\" type\n * @param[in] objname - null-terminated object name string. If NULL, no name will be assigned\n * to the object -- you can use the __itt_sync_rename call later to assign\n * the name\n * @param[in] typelen, namelen - a length of string for appropriate objtype and objname parameter\n * @param[in] attribute - one of [#__itt_attr_barrier, #__itt_attr_mutex] values which defines the\n * exact semantics of how prepare/acquired/releasing calls work.\n * @return __itt_err upon failure (name or namelen being null,name and namelen mismatched)\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nint LIBITTAPI __itt_notify_sync_nameA(void *addr, const char *objtype, int typelen, const char *objname, int namelen, int attribute);\nint LIBITTAPI __itt_notify_sync_nameW(void *addr, const wchar_t *objtype, int typelen, const wchar_t *objname, int namelen, int attribute);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_notify_sync_name __itt_notify_sync_nameW\n#else\n# define __itt_notify_sync_name __itt_notify_sync_nameA\n#endif\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nint LIBITTAPI __itt_notify_sync_name(void *addr, const char *objtype, int typelen, const char *objname, int namelen, int attribute);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, int, notify_sync_nameA, (void *addr, const char *objtype, int typelen, const char *objname, int namelen, int attribute))\nITT_STUB(LIBITTAPI, int, notify_sync_nameW, (void *addr, const wchar_t *objtype, int typelen, const wchar_t *objname, int namelen, int attribute))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, int, notify_sync_name, (void *addr, const char *objtype, int typelen, const char *objname, int namelen, int attribute))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_notify_sync_nameA ITTNOTIFY_DATA(notify_sync_nameA)\n#define __itt_notify_sync_nameA_ptr ITTNOTIFY_NAME(notify_sync_nameA)\n#define __itt_notify_sync_nameW ITTNOTIFY_DATA(notify_sync_nameW)\n#define __itt_notify_sync_nameW_ptr ITTNOTIFY_NAME(notify_sync_nameW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_notify_sync_name ITTNOTIFY_DATA(notify_sync_name)\n#define __itt_notify_sync_name_ptr ITTNOTIFY_NAME(notify_sync_name)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_notify_sync_nameA(addr, objtype, typelen, objname, namelen, attribute)\n#define __itt_notify_sync_nameA_ptr 0\n#define __itt_notify_sync_nameW(addr, objtype, typelen, objname, namelen, attribute)\n#define __itt_notify_sync_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_notify_sync_name(addr, objtype, typelen, objname, namelen, attribute)\n#define __itt_notify_sync_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_notify_sync_nameA_ptr 0\n#define __itt_notify_sync_nameW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_notify_sync_name_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Enter spin loop on user-defined sync object\n */\nvoid LIBITTAPI __itt_notify_sync_prepare(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, notify_sync_prepare, (void *addr))\n#define __itt_notify_sync_prepare ITTNOTIFY_VOID(notify_sync_prepare)\n#define __itt_notify_sync_prepare_ptr ITTNOTIFY_NAME(notify_sync_prepare)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_notify_sync_prepare(addr)\n#define __itt_notify_sync_prepare_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_notify_sync_prepare_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Quit spin loop without acquiring spin object\n */\nvoid LIBITTAPI __itt_notify_sync_cancel(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, notify_sync_cancel, (void *addr))\n#define __itt_notify_sync_cancel ITTNOTIFY_VOID(notify_sync_cancel)\n#define __itt_notify_sync_cancel_ptr ITTNOTIFY_NAME(notify_sync_cancel)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_notify_sync_cancel(addr)\n#define __itt_notify_sync_cancel_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_notify_sync_cancel_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Successful spin loop completion (sync object acquired)\n */\nvoid LIBITTAPI __itt_notify_sync_acquired(void *addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, notify_sync_acquired, (void *addr))\n#define __itt_notify_sync_acquired ITTNOTIFY_VOID(notify_sync_acquired)\n#define __itt_notify_sync_acquired_ptr ITTNOTIFY_NAME(notify_sync_acquired)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_notify_sync_acquired(addr)\n#define __itt_notify_sync_acquired_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_notify_sync_acquired_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Start sync object releasing code. Is called before the lock release call.\n */\nvoid LIBITTAPI __itt_notify_sync_releasing(void* addr);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, notify_sync_releasing, (void *addr))\n#define __itt_notify_sync_releasing ITTNOTIFY_VOID(notify_sync_releasing)\n#define __itt_notify_sync_releasing_ptr ITTNOTIFY_NAME(notify_sync_releasing)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_notify_sync_releasing(addr)\n#define __itt_notify_sync_releasing_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_notify_sync_releasing_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} legacy_sync group */\n\n#ifndef _ITTNOTIFY_H_\n/**\n * @defgroup legacy_events Events\n * @ingroup legacy\n * Events group\n * @{\n */\n\n/** @brief user event type */\ntypedef int __itt_event;\n\n/**\n * @brief Create an event notification\n * @note name or namelen being null/name and namelen not matching, user event feature not enabled\n * @return non-zero event identifier upon success and __itt_err otherwise\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_event LIBITTAPI __itt_event_createA(const char *name, int namelen);\n__itt_event LIBITTAPI __itt_event_createW(const wchar_t *name, int namelen);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_event_create __itt_event_createW\n# define __itt_event_create_ptr __itt_event_createW_ptr\n#else\n# define __itt_event_create __itt_event_createA\n# define __itt_event_create_ptr __itt_event_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_event LIBITTAPI __itt_event_create(const char *name, int namelen);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen))\nITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA ITTNOTIFY_DATA(event_createA)\n#define __itt_event_createA_ptr ITTNOTIFY_NAME(event_createA)\n#define __itt_event_createW ITTNOTIFY_DATA(event_createW)\n#define __itt_event_createW_ptr ITTNOTIFY_NAME(event_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create ITTNOTIFY_DATA(event_create)\n#define __itt_event_create_ptr ITTNOTIFY_NAME(event_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA(name, namelen) (__itt_event)0\n#define __itt_event_createA_ptr 0\n#define __itt_event_createW(name, namelen) (__itt_event)0\n#define __itt_event_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create(name, namelen) (__itt_event)0\n#define __itt_event_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_event_createA_ptr 0\n#define __itt_event_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_event_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an event occurrence.\n * @return __itt_err upon failure (invalid event id/user event feature not enabled)\n */\nint LIBITTAPI __itt_event_start(__itt_event event);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(LIBITTAPI, int, event_start, (__itt_event event))\n#define __itt_event_start ITTNOTIFY_DATA(event_start)\n#define __itt_event_start_ptr ITTNOTIFY_NAME(event_start)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_event_start(event) (int)0\n#define __itt_event_start_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_event_start_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @brief Record an event end occurrence.\n * @note It is optional if events do not have durations.\n * @return __itt_err upon failure (invalid event id/user event feature not enabled)\n */\nint LIBITTAPI __itt_event_end(__itt_event event);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(LIBITTAPI, int, event_end, (__itt_event event))\n#define __itt_event_end ITTNOTIFY_DATA(event_end)\n#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_event_end(event) (int)0\n#define __itt_event_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_event_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} legacy_events group */\n#endif /* _ITTNOTIFY_H_ */\n\n/**\n * @defgroup legacy_memory Memory Accesses\n * @ingroup legacy\n */\n\n/**\n * @deprecated Legacy API\n * @brief Inform the tool of memory accesses on reading\n */\nvoid LIBITTAPI __itt_memory_read(void *addr, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, memory_read, (void *addr, size_t size))\n#define __itt_memory_read ITTNOTIFY_VOID(memory_read)\n#define __itt_memory_read_ptr ITTNOTIFY_NAME(memory_read)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_memory_read(addr, size)\n#define __itt_memory_read_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_memory_read_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Inform the tool of memory accesses on writing\n */\nvoid LIBITTAPI __itt_memory_write(void *addr, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, memory_write, (void *addr, size_t size))\n#define __itt_memory_write ITTNOTIFY_VOID(memory_write)\n#define __itt_memory_write_ptr ITTNOTIFY_NAME(memory_write)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_memory_write(addr, size)\n#define __itt_memory_write_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_memory_write_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief Inform the tool of memory accesses on updating\n */\nvoid LIBITTAPI __itt_memory_update(void *address, size_t size);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(LIBITTAPI, void, memory_update, (void *addr, size_t size))\n#define __itt_memory_update ITTNOTIFY_VOID(memory_update)\n#define __itt_memory_update_ptr ITTNOTIFY_NAME(memory_update)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_memory_update(addr, size)\n#define __itt_memory_update_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_memory_update_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} legacy_memory group */\n\n/**\n * @defgroup legacy_state Thread and Object States\n * @ingroup legacy\n */\n\n/** @brief state type */\ntypedef int __itt_state_t;\n\n/** @cond exclude_from_documentation */\ntypedef enum __itt_obj_state {\n __itt_obj_state_err = 0,\n __itt_obj_state_clr = 1,\n __itt_obj_state_set = 2,\n __itt_obj_state_use = 3\n} __itt_obj_state_t;\n\ntypedef enum __itt_thr_state {\n __itt_thr_state_err = 0,\n __itt_thr_state_clr = 1,\n __itt_thr_state_set = 2\n} __itt_thr_state_t;\n\ntypedef enum __itt_obj_prop {\n __itt_obj_prop_watch = 1,\n __itt_obj_prop_ignore = 2,\n __itt_obj_prop_sharable = 3\n} __itt_obj_prop_t;\n\ntypedef enum __itt_thr_prop {\n __itt_thr_prop_quiet = 1\n} __itt_thr_prop_t;\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief managing thread and object states\n */\n__itt_state_t LIBITTAPI __itt_state_get(void);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_state_t, state_get, (void))\n#define __itt_state_get ITTNOTIFY_DATA(state_get)\n#define __itt_state_get_ptr ITTNOTIFY_NAME(state_get)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_state_get(void) (__itt_state_t)0\n#define __itt_state_get_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_state_get_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief managing thread and object states\n */\n__itt_state_t LIBITTAPI __itt_state_set(__itt_state_t s);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_state_t, state_set, (__itt_state_t s))\n#define __itt_state_set ITTNOTIFY_DATA(state_set)\n#define __itt_state_set_ptr ITTNOTIFY_NAME(state_set)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_state_set(s) (__itt_state_t)0\n#define __itt_state_set_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_state_set_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief managing thread and object modes\n */\n__itt_thr_state_t LIBITTAPI __itt_thr_mode_set(__itt_thr_prop_t p, __itt_thr_state_t s);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_thr_state_t, thr_mode_set, (__itt_thr_prop_t p, __itt_thr_state_t s))\n#define __itt_thr_mode_set ITTNOTIFY_DATA(thr_mode_set)\n#define __itt_thr_mode_set_ptr ITTNOTIFY_NAME(thr_mode_set)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_thr_mode_set(p, s) (__itt_thr_state_t)0\n#define __itt_thr_mode_set_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_thr_mode_set_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/**\n * @deprecated Legacy API\n * @brief managing thread and object modes\n */\n__itt_obj_state_t LIBITTAPI __itt_obj_mode_set(__itt_obj_prop_t p, __itt_obj_state_t s);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUB(ITTAPI, __itt_obj_state_t, obj_mode_set, (__itt_obj_prop_t p, __itt_obj_state_t s))\n#define __itt_obj_mode_set ITTNOTIFY_DATA(obj_mode_set)\n#define __itt_obj_mode_set_ptr ITTNOTIFY_NAME(obj_mode_set)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_obj_mode_set(p, s) (__itt_obj_state_t)0\n#define __itt_obj_mode_set_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_obj_mode_set_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} legacy_state group */\n\n/**\n * @defgroup frames Frames\n * @ingroup legacy\n * Frames group\n * @{\n */\n/**\n * @brief opaque structure for frame identification\n */\ntypedef struct __itt_frame_t *__itt_frame;\n\n/**\n * @brief Create a global frame with given domain\n */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n__itt_frame ITTAPI __itt_frame_createA(const char *domain);\n__itt_frame ITTAPI __itt_frame_createW(const wchar_t *domain);\n#if defined(UNICODE) || defined(_UNICODE)\n# define __itt_frame_create __itt_frame_createW\n# define __itt_frame_create_ptr __itt_frame_createW_ptr\n#else /* UNICODE */\n# define __itt_frame_create __itt_frame_createA\n# define __itt_frame_create_ptr __itt_frame_createA_ptr\n#endif /* UNICODE */\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n__itt_frame ITTAPI __itt_frame_create(const char *domain);\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\nITT_STUB(ITTAPI, __itt_frame, frame_createA, (const char *domain))\nITT_STUB(ITTAPI, __itt_frame, frame_createW, (const wchar_t *domain))\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\nITT_STUB(ITTAPI, __itt_frame, frame_create, (const char *domain))\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_frame_createA ITTNOTIFY_DATA(frame_createA)\n#define __itt_frame_createA_ptr ITTNOTIFY_NAME(frame_createA)\n#define __itt_frame_createW ITTNOTIFY_DATA(frame_createW)\n#define __itt_frame_createW_ptr ITTNOTIFY_NAME(frame_createW)\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_frame_create ITTNOTIFY_DATA(frame_create)\n#define __itt_frame_create_ptr ITTNOTIFY_NAME(frame_create)\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#else /* INTEL_NO_ITTNOTIFY_API */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_frame_createA(domain)\n#define __itt_frame_createA_ptr 0\n#define __itt_frame_createW(domain)\n#define __itt_frame_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_frame_create(domain)\n#define __itt_frame_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#if ITT_PLATFORM==ITT_PLATFORM_WIN\n#define __itt_frame_createA_ptr 0\n#define __itt_frame_createW_ptr 0\n#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#define __itt_frame_create_ptr 0\n#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n\n/** @brief Record a frame begin occurrence. */\nvoid ITTAPI __itt_frame_begin(__itt_frame frame);\n/** @brief Record a frame end occurrence. */\nvoid ITTAPI __itt_frame_end (__itt_frame frame);\n\n/** @cond exclude_from_documentation */\n#ifndef INTEL_NO_MACRO_BODY\n#ifndef INTEL_NO_ITTNOTIFY_API\nITT_STUBV(ITTAPI, void, frame_begin, (__itt_frame frame))\nITT_STUBV(ITTAPI, void, frame_end, (__itt_frame frame))\n#define __itt_frame_begin ITTNOTIFY_VOID(frame_begin)\n#define __itt_frame_begin_ptr ITTNOTIFY_NAME(frame_begin)\n#define __itt_frame_end ITTNOTIFY_VOID(frame_end)\n#define __itt_frame_end_ptr ITTNOTIFY_NAME(frame_end)\n#else /* INTEL_NO_ITTNOTIFY_API */\n#define __itt_frame_begin(frame)\n#define __itt_frame_begin_ptr 0\n#define __itt_frame_end(frame)\n#define __itt_frame_end_ptr 0\n#endif /* INTEL_NO_ITTNOTIFY_API */\n#else /* INTEL_NO_MACRO_BODY */\n#define __itt_frame_begin_ptr 0\n#define __itt_frame_end_ptr 0\n#endif /* INTEL_NO_MACRO_BODY */\n/** @endcond */\n/** @} frames group */\n\n#ifdef __cplusplus\n}\n#endif /* __cplusplus */\n\n#endif /* _LEGACY_ITTNOTIFY_H_ */\n" }, { "path": "src/tbb/src/tbb/version.cpp", "content": "/*\n Copyright (c) 2020-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/version.h\"\n\nextern \"C\" int TBB_runtime_interface_version() {\n return TBB_INTERFACE_VERSION;\n}\n\nextern \"C\" const char* TBB_runtime_version() {\n static const char version_str[] = TBB_VERSION_STRING;\n return version_str;\n}\n" }, { "path": "src/tbb/src/tbb/waiters.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_waiters_H\n#define _TBB_waiters_H\n\n#include \"oneapi/tbb/detail/_task.h\"\n#include \"scheduler_common.h\"\n#include \"arena.h\"\n#include \"threading_control.h\"\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\ninline d1::task* get_self_recall_task(arena_slot& slot);\n\nclass waiter_base {\npublic:\n waiter_base(arena& a, int yields_multiplier = 1) : my_arena(a), my_backoff(int(a.my_num_slots), yields_multiplier) {}\n\n bool pause() {\n if (my_backoff.pause()) {\n my_arena.out_of_work();\n return true;\n }\n\n return false;\n }\n\n void reset_wait() {\n my_backoff.reset_wait();\n }\n\nprotected:\n arena& my_arena;\n stealing_loop_backoff my_backoff;\n};\n\nclass outermost_worker_waiter : public waiter_base {\npublic:\n using waiter_base::waiter_base;\n\n bool continue_execution(arena_slot& slot, d1::task*& t) const {\n __TBB_ASSERT(t == nullptr, nullptr);\n\n if (is_worker_should_leave(slot)) {\n if (!governor::hybrid_cpu()) {\n static constexpr std::chrono::microseconds worker_wait_leave_duration(1000);\n static_assert(worker_wait_leave_duration > std::chrono::steady_clock::duration(1), \"Clock resolution is not enough for measured interval.\");\n\n for (auto t1 = std::chrono::steady_clock::now(), t2 = t1;\n std::chrono::duration_cast(t2 - t1) < worker_wait_leave_duration;\n t2 = std::chrono::steady_clock::now())\n {\n if (!my_arena.is_empty() && !my_arena.is_recall_requested()) {\n return true;\n }\n\n if (my_arena.my_threading_control->is_any_other_client_active()) {\n break;\n }\n d0::yield();\n }\n }\n // Leave dispatch loop\n return false;\n }\n\n t = get_self_recall_task(slot);\n return true;\n }\n\n void pause(arena_slot&) {\n waiter_base::pause();\n }\n\n\n d1::wait_context* wait_ctx() {\n return nullptr;\n }\n\n static bool postpone_execution(d1::task&) {\n return false;\n }\n\nprivate:\n using base_type = waiter_base;\n\n bool is_worker_should_leave(arena_slot& slot) const {\n bool is_top_priority_arena = my_arena.is_top_priority();\n bool is_task_pool_empty = slot.task_pool.load(std::memory_order_relaxed) == EmptyTaskPool;\n\n if (is_top_priority_arena) {\n // Worker in most priority arena do not leave arena, until all work in task_pool is done\n if (is_task_pool_empty && my_arena.is_recall_requested()) {\n return true;\n }\n } else {\n if (my_arena.is_recall_requested()) {\n // If worker has work in task pool, we must notify other threads,\n // because can appear missed wake up of other threads\n if (!is_task_pool_empty) {\n my_arena.advertise_new_work();\n }\n return true;\n }\n }\n\n return false;\n }\n};\n\nclass sleep_waiter : public waiter_base {\nprotected:\n using waiter_base::waiter_base;\n\n template \n void sleep(std::uintptr_t uniq_tag, Pred wakeup_condition) {\n my_arena.get_waiting_threads_monitor().wait(wakeup_condition,\n market_context{uniq_tag, &my_arena});\n reset_wait();\n }\n};\n\nclass external_waiter : public sleep_waiter {\npublic:\n external_waiter(arena& a, d1::wait_context& wo)\n : sleep_waiter(a, /*yields_multiplier*/10), my_wait_ctx(wo)\n {}\n\n bool continue_execution(arena_slot& slot, d1::task*& t) const {\n __TBB_ASSERT(t == nullptr, nullptr);\n if (!my_wait_ctx.continue_execution())\n return false;\n t = get_self_recall_task(slot);\n return true;\n }\n\n void pause(arena_slot&) {\n if (!sleep_waiter::pause()) {\n return;\n }\n\n auto wakeup_condition = [&] { return !my_arena.is_empty() || !my_wait_ctx.continue_execution(); };\n\n sleep(std::uintptr_t(&my_wait_ctx), wakeup_condition);\n }\n\n d1::wait_context* wait_ctx() {\n return &my_wait_ctx;\n }\n\n static bool postpone_execution(d1::task&) {\n return false;\n }\n\nprivate:\n d1::wait_context& my_wait_ctx;\n};\n\n#if __TBB_RESUMABLE_TASKS\n\nclass coroutine_waiter : public sleep_waiter {\npublic:\n using sleep_waiter::sleep_waiter;\n\n bool continue_execution(arena_slot& slot, d1::task*& t) const {\n __TBB_ASSERT(t == nullptr, nullptr);\n t = get_self_recall_task(slot);\n return true;\n }\n\n void pause(arena_slot& slot) {\n if (!sleep_waiter::pause()) {\n return;\n }\n\n suspend_point_type* sp = slot.default_task_dispatcher().m_suspend_point;\n\n auto wakeup_condition = [&] { return !my_arena.is_empty() || sp->m_is_owner_recalled.load(std::memory_order_relaxed); };\n\n sleep(std::uintptr_t(sp), wakeup_condition);\n }\n\n d1::wait_context* wait_ctx() {\n return nullptr;\n }\n\n static bool postpone_execution(d1::task& t) {\n return task_accessor::is_resume_task(t);\n }\n};\n\n#endif // __TBB_RESUMABLE_TASKS\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#endif // _TBB_waiters_H\n" }, { "path": "src/tbb/src/tbbbind/CMakeLists.txt", "content": "# Copyright (c) 2020-2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset(CMAKE_SKIP_BUILD_RPATH TRUE)\n\nfunction(tbbbind_build TBBBIND_NAME REQUIRED_HWLOC_TARGET)\n if (NOT TARGET ${REQUIRED_HWLOC_TARGET})\n message(STATUS \"HWLOC target ${REQUIRED_HWLOC_TARGET} doesn't exist.\"\n \" The ${TBBBIND_NAME} target cannot be created\")\n return()\n endif()\n add_library(${TBBBIND_NAME} tbb_bind.cpp)\n\n if (WIN32)\n target_sources(${TBBBIND_NAME} PRIVATE tbb_bind.rc)\n endif()\n\n add_library(TBB::${TBBBIND_NAME} ALIAS ${TBBBIND_NAME})\n\n target_compile_definitions(${TBBBIND_NAME}\n PUBLIC\n $<$:TBB_USE_DEBUG>\n PRIVATE\n __TBBBIND_BUILD)\n target_include_directories(${TBBBIND_NAME}\n PUBLIC\n $\n $\n ${HWLOC_INCLUDE_DIRS} # pkg-config defined\n )\n\n target_compile_options(${TBBBIND_NAME}\n PRIVATE\n ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC.\n ${TBB_MMD_FLAG}\n ${TBB_DSE_FLAG}\n ${TBB_WARNING_LEVEL}\n ${TBB_LIB_COMPILE_FLAGS}\n ${TBB_COMMON_COMPILE_FLAGS}\n )\n\n # Avoid use of target_link_libraries here as it changes /DEF option to \\DEF on Windows.\n set_target_properties(${TBBBIND_NAME} PROPERTIES\n DEFINE_SYMBOL \"\"\n )\n\n tbb_handle_ipo(${TBBBIND_NAME})\n\n if (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives\n set_target_properties(${TBBBIND_NAME} PROPERTIES\n LINK_FLAGS \"${TBB_LINK_DEF_FILE_FLAG}\\\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbbind.def\\\"\"\n LINK_DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbbind.def\"\n )\n endif()\n\n # Prefer using target_link_options instead of target_link_libraries to specify link options because\n # target_link_libraries may incorrectly handle some options (on Windows, for example).\n if (COMMAND target_link_options)\n target_link_options(${TBBBIND_NAME}\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\n else()\n target_link_libraries(${TBBBIND_NAME}\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\n endif()\n\n target_link_libraries(${TBBBIND_NAME}\n PUBLIC\n ${REQUIRED_HWLOC_TARGET}\n PRIVATE\n ${TBB_LIB_LINK_LIBS}\n ${TBB_COMMON_LINK_LIBS}\n )\n\n tbb_install_target(${TBBBIND_NAME})\n\nendfunction()\n\nif (NOT DEFINED HWLOC_TARGET_EXPLICITLY_DEFINED AND TARGET PkgConfig::HWLOC)\n message(STATUS \"The ${TBBBIND_LIBRARY_NAME} target will be configured using the HWLOC ${HWLOC_VERSION}\")\n tbbbind_build(${TBBBIND_LIBRARY_NAME} PkgConfig::HWLOC)\nelse()\n tbbbind_build(tbbbind HWLOC::hwloc_1_11)\n tbbbind_build(tbbbind_2_0 HWLOC::hwloc_2 )\n tbbbind_build(tbbbind_2_5 HWLOC::hwloc_2_5 )\nendif()\n\n" }, { "path": "src/tbb/src/tbbbind/def/lin32-tbbbind.def", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n__TBB_internal_initialize_system_topology;\n__TBB_internal_apply_affinity;\n__TBB_internal_restore_affinity;\n__TBB_internal_allocate_binding_handler;\n__TBB_internal_deallocate_binding_handler;\n__TBB_internal_get_default_concurrency;\n__TBB_internal_destroy_system_topology;\n};\n" }, { "path": "src/tbb/src/tbbbind/def/lin64-tbbbind.def", "content": "/*\n Copyright (c) 2019-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n__TBB_internal_initialize_system_topology;\n__TBB_internal_apply_affinity;\n__TBB_internal_restore_affinity;\n__TBB_internal_allocate_binding_handler;\n__TBB_internal_deallocate_binding_handler;\n__TBB_internal_get_default_concurrency;\n__TBB_internal_destroy_system_topology;\n};\n" }, { "path": "src/tbb/src/tbbbind/def/mac64-tbbbind.def", "content": "# Copyright (c) 2023 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n___TBB_internal_initialize_system_topology\n___TBB_internal_get_default_concurrency\n___TBB_internal_destroy_system_topology\n\n" }, { "path": "src/tbb/src/tbbbind/def/win32-tbbbind.def", "content": "; Copyright (c) 2019-2021 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\nEXPORTS\n\n__TBB_internal_initialize_system_topology\n__TBB_internal_apply_affinity\n__TBB_internal_restore_affinity\n__TBB_internal_allocate_binding_handler\n__TBB_internal_deallocate_binding_handler\n__TBB_internal_get_default_concurrency\n__TBB_internal_destroy_system_topology\n" }, { "path": "src/tbb/src/tbbbind/def/win64-tbbbind.def", "content": "; Copyright (c) 2019-2021 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\nEXPORTS\n\n__TBB_internal_initialize_system_topology\n__TBB_internal_apply_affinity\n__TBB_internal_restore_affinity\n__TBB_internal_allocate_binding_handler\n__TBB_internal_deallocate_binding_handler\n__TBB_internal_get_default_concurrency\n__TBB_internal_destroy_system_topology\n" }, { "path": "src/tbb/src/tbbbind/tbb_bind.cpp", "content": "/*\n Copyright (c) 2019-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \n#include \n\n#include \"../tbb/assert_impl.h\" // Out-of-line TBB assertion handling routines are instantiated here.\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"oneapi/tbb/detail/_config.h\"\n\n#if _MSC_VER && !__INTEL_COMPILER && !__clang__\n// #pragma warning( push )\n// #pragma warning( disable : 4100 )\n#elif _MSC_VER && __clang__\n// #pragma GCC diagnostic push\n// #pragma GCC diagnostic ignored \"-Wunused-parameter\"\n#endif\n#include \n#if _MSC_VER && !__INTEL_COMPILER && !__clang__\n// #pragma warning( pop )\n#elif _MSC_VER && __clang__\n// #pragma GCC diagnostic pop\n#endif\n\n#define __TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT (HWLOC_API_VERSION >= 0x20400)\n#define __TBBBIND_HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING_PRESENT (HWLOC_API_VERSION >= 0x20500)\n\n// Most of hwloc calls returns negative exit code on error.\n// This macro tracks error codes that are returned from the hwloc interfaces.\n#define assertion_hwloc_wrapper(command, ...) \\\n __TBB_ASSERT_EX( (command(__VA_ARGS__)) >= 0, \"Error occurred during call to hwloc API.\");\n\nnamespace tbb {\nnamespace detail {\nnamespace r1 {\n\n//------------------------------------------------------------------------\n// Information about the machine's hardware TBB is happen to work on\n//------------------------------------------------------------------------\nclass system_topology {\n friend class binding_handler;\n\n // Common topology members\n hwloc_topology_t topology{nullptr};\n hwloc_cpuset_t process_cpu_affinity_mask{nullptr};\n hwloc_nodeset_t process_node_affinity_mask{nullptr};\n std::size_t number_of_processors_groups{1};\n\n // NUMA API related topology members\n std::vector numa_affinity_masks_list{};\n std::vector numa_indexes_list{};\n int numa_nodes_count{0};\n\n // Hybrid CPUs API related topology members\n std::vector core_types_affinity_masks_list{};\n std::vector core_types_indexes_list{};\n\n enum init_stages { uninitialized,\n started,\n topology_allocated,\n topology_loaded,\n topology_parsed } initialization_state;\n\n // Binding threads that locate in another Windows Processor groups\n // is allowed only if machine topology contains several Windows Processors groups\n // and process affinity mask wasn't limited manually (affinity mask cannot violates\n // processors group boundaries).\n bool intergroup_binding_allowed(std::size_t groups_num) { return groups_num > 1; }\n\nprivate:\n void topology_initialization(std::size_t groups_num) {\n initialization_state = started;\n\n // Parse topology\n if ( hwloc_topology_init( &topology ) == 0 ) {\n initialization_state = topology_allocated;\n#if __TBBBIND_HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING_PRESENT\n unsigned long flags = 0;\n if (groups_num > 1) {\n // HWLOC x86 backend might interfere with process affinity mask on\n // Windows systems with multiple processor groups.\n flags = HWLOC_TOPOLOGY_FLAG_DONT_CHANGE_BINDING;\n } else {\n flags = HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM | HWLOC_TOPOLOGY_FLAG_RESTRICT_TO_CPUBINDING;\n }\n if (hwloc_topology_set_flags(topology, flags) != 0) {\n return;\n }\n#endif\n if ( hwloc_topology_load( topology ) == 0 ) {\n initialization_state = topology_loaded;\n }\n }\n if ( initialization_state != topology_loaded )\n return;\n\n#if __TBB_CPUBIND_PRESENT\n // Getting process affinity mask\n if ( intergroup_binding_allowed(groups_num) ) {\n process_cpu_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset (topology));\n process_node_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_nodeset(topology));\n } else {\n process_cpu_affinity_mask = hwloc_bitmap_alloc();\n process_node_affinity_mask = hwloc_bitmap_alloc();\n\n assertion_hwloc_wrapper(hwloc_get_cpubind, topology, process_cpu_affinity_mask, 0);\n hwloc_cpuset_to_nodeset(topology, process_cpu_affinity_mask, process_node_affinity_mask);\n }\n#else\n process_cpu_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset (topology));\n process_node_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_nodeset(topology));\n#endif\n\n number_of_processors_groups = groups_num;\n }\n\n void numa_topology_parsing() {\n // Fill parameters with stubs if topology parsing is broken.\n if ( initialization_state != topology_loaded ) {\n numa_nodes_count = 1;\n numa_indexes_list.push_back(-1);\n return;\n }\n\n // If system contains no NUMA nodes, HWLOC 1.11 returns an infinitely filled bitmap.\n // hwloc_bitmap_weight() returns negative value for such bitmaps, so we use this check\n // to change way of topology initialization.\n numa_nodes_count = hwloc_bitmap_weight(process_node_affinity_mask);\n if (numa_nodes_count <= 0) {\n // numa_nodes_count may be empty if the process affinity mask is empty too (invalid case)\n // or if some internal HWLOC error occurred.\n // So we place -1 as index in this case.\n numa_indexes_list.push_back(numa_nodes_count == 0 ? -1 : 0);\n numa_nodes_count = 1;\n\n numa_affinity_masks_list.push_back(hwloc_bitmap_dup(process_cpu_affinity_mask));\n } else {\n // Get NUMA logical indexes list\n unsigned counter = 0;\n int i = 0;\n int max_numa_index = -1;\n numa_indexes_list.resize(numa_nodes_count);\n hwloc_obj_t node_buffer;\n hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) {\n node_buffer = hwloc_get_numanode_obj_by_os_index(topology, i);\n numa_indexes_list[counter] = static_cast(node_buffer->logical_index);\n\n if ( numa_indexes_list[counter] > max_numa_index ) {\n max_numa_index = numa_indexes_list[counter];\n }\n\n counter++;\n } hwloc_bitmap_foreach_end();\n __TBB_ASSERT(max_numa_index >= 0, \"Maximal NUMA index must not be negative\");\n\n // Fill concurrency and affinity masks lists\n numa_affinity_masks_list.resize(max_numa_index + 1);\n int index = 0;\n hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) {\n node_buffer = hwloc_get_numanode_obj_by_os_index(topology, i);\n index = static_cast(node_buffer->logical_index);\n\n hwloc_cpuset_t& current_mask = numa_affinity_masks_list[index];\n current_mask = hwloc_bitmap_dup(node_buffer->cpuset);\n\n hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask);\n __TBB_ASSERT(!hwloc_bitmap_iszero(current_mask), \"hwloc detected unavailable NUMA node\");\n } hwloc_bitmap_foreach_end();\n }\n }\n\n void core_types_topology_parsing() {\n // Fill parameters with stubs if topology parsing is broken.\n if ( initialization_state != topology_loaded ) {\n core_types_indexes_list.push_back(-1);\n return;\n }\n#if __TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT\n __TBB_ASSERT(hwloc_get_api_version() >= 0x20400, \"Hybrid CPUs support interfaces required HWLOC >= 2.4\");\n // Parsing the hybrid CPU topology\n int core_types_number = hwloc_cpukinds_get_nr(topology, 0);\n bool core_types_parsing_broken = core_types_number <= 0;\n if (!core_types_parsing_broken) {\n core_types_affinity_masks_list.resize(core_types_number);\n int efficiency{-1};\n\n for (int core_type = 0; core_type < core_types_number; ++core_type) {\n hwloc_cpuset_t& current_mask = core_types_affinity_masks_list[core_type];\n current_mask = hwloc_bitmap_alloc();\n\n if (!hwloc_cpukinds_get_info(topology, core_type, current_mask, &efficiency, nullptr, nullptr, 0)\n && efficiency >= 0\n ) {\n hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask);\n\n if (hwloc_bitmap_weight(current_mask) > 0) {\n core_types_indexes_list.push_back(core_type);\n }\n __TBB_ASSERT(hwloc_bitmap_weight(current_mask) >= 0, \"Infinivitely filled core type mask\");\n } else {\n core_types_parsing_broken = true;\n break;\n }\n }\n }\n#else /*!__TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT*/\n bool core_types_parsing_broken{true};\n#endif /*__TBBBIND_HWLOC_HYBRID_CPUS_INTERFACES_PRESENT*/\n\n if (core_types_parsing_broken) {\n for (auto& core_type_mask : core_types_affinity_masks_list) {\n hwloc_bitmap_free(core_type_mask);\n }\n core_types_affinity_masks_list.resize(1);\n core_types_indexes_list.resize(1);\n\n core_types_affinity_masks_list[0] = hwloc_bitmap_dup(process_cpu_affinity_mask);\n core_types_indexes_list[0] = -1;\n }\n }\n\n void enforce_hwloc_2_5_runtime_linkage() {\n // Without the call of this function HWLOC 2.4 can be successfully loaded during the tbbbind_2_5 loading.\n // It is possible since tbbbind_2_5 don't use any new entry points that were introduced in HWLOC 2.5\n // But tbbbind_2_5 compiles with HWLOC 2.5 header, therefore such situation requires binary forward compatibility\n // which are not guaranteed by the HWLOC library. To enforce linkage tbbbind_2_5 only with HWLOC >= 2.5 version\n // this function calls the interface that is available in the HWLOC 2.5 only.\n#if HWLOC_API_VERSION >= 0x20500\n auto some_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, nullptr);\n hwloc_get_obj_with_same_locality(topology, some_core, HWLOC_OBJ_CORE, nullptr, nullptr, 0);\n#endif\n }\n\n \n void initialize( std::size_t groups_num ) {\n if ( initialization_state != uninitialized )\n return;\n\n topology_initialization(groups_num);\n numa_topology_parsing();\n core_types_topology_parsing();\n\n enforce_hwloc_2_5_runtime_linkage();\n\n if (initialization_state == topology_loaded)\n initialization_state = topology_parsed;\n }\n\n static system_topology* instance_ptr;\npublic:\n typedef hwloc_cpuset_t affinity_mask;\n typedef hwloc_const_cpuset_t const_affinity_mask;\n\n bool is_topology_parsed() { return initialization_state == topology_parsed; }\n\n static void construct( std::size_t groups_num ) {\n if (instance_ptr == nullptr) {\n instance_ptr = new system_topology();\n instance_ptr->initialize(groups_num);\n }\n }\n\n static system_topology& instance() {\n __TBB_ASSERT(instance_ptr != nullptr, \"Getting instance of non-constructed topology\");\n return *instance_ptr;\n }\n\n static void destroy() {\n __TBB_ASSERT(instance_ptr != nullptr, \"Destroying non-constructed topology\");\n delete instance_ptr;\n }\n\n ~system_topology() {\n if ( is_topology_parsed() ) {\n for (auto& numa_node_mask : numa_affinity_masks_list) {\n hwloc_bitmap_free(numa_node_mask);\n }\n\n for (auto& core_type_mask : core_types_affinity_masks_list) {\n hwloc_bitmap_free(core_type_mask);\n }\n\n hwloc_bitmap_free(process_node_affinity_mask);\n hwloc_bitmap_free(process_cpu_affinity_mask);\n }\n\n if ( initialization_state >= topology_allocated ) {\n hwloc_topology_destroy(topology);\n }\n\n initialization_state = uninitialized;\n }\n\n void fill_topology_information(\n int& _numa_nodes_count, int*& _numa_indexes_list,\n int& _core_types_count, int*& _core_types_indexes_list\n ) {\n __TBB_ASSERT(is_topology_parsed(), \"Trying to get access to uninitialized system_topology\");\n _numa_nodes_count = numa_nodes_count;\n _numa_indexes_list = numa_indexes_list.data();\n\n _core_types_count = (int)core_types_indexes_list.size();\n _core_types_indexes_list = core_types_indexes_list.data();\n }\n\n void fill_constraints_affinity_mask(affinity_mask input_mask, int numa_node_index, int core_type_index, int max_threads_per_core) {\n __TBB_ASSERT(is_topology_parsed(), \"Trying to get access to uninitialized system_topology\");\n __TBB_ASSERT(numa_node_index < (int)numa_affinity_masks_list.size(), \"Wrong NUMA node id\");\n __TBB_ASSERT(core_type_index < (int)core_types_affinity_masks_list.size(), \"Wrong core type id\");\n __TBB_ASSERT(max_threads_per_core == -1 || max_threads_per_core > 0, \"Wrong max_threads_per_core\");\n\n hwloc_cpuset_t constraints_mask = hwloc_bitmap_alloc();\n hwloc_cpuset_t core_mask = hwloc_bitmap_alloc();\n\n hwloc_bitmap_copy(constraints_mask, process_cpu_affinity_mask);\n if (numa_node_index >= 0) {\n hwloc_bitmap_and(constraints_mask, constraints_mask, numa_affinity_masks_list[numa_node_index]);\n }\n if (core_type_index >= 0) {\n hwloc_bitmap_and(constraints_mask, constraints_mask, core_types_affinity_masks_list[core_type_index]);\n }\n if (max_threads_per_core > 0) {\n // clear input mask\n hwloc_bitmap_zero(input_mask);\n\n hwloc_obj_t current_core = nullptr;\n while ((current_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, current_core)) != nullptr) {\n hwloc_bitmap_and(core_mask, constraints_mask, current_core->cpuset);\n\n // fit the core mask to required bits number\n int current_threads_per_core = 0;\n for (int id = hwloc_bitmap_first(core_mask); id != -1; id = hwloc_bitmap_next(core_mask, id)) {\n if (++current_threads_per_core > max_threads_per_core) {\n hwloc_bitmap_clr(core_mask, id);\n }\n }\n\n hwloc_bitmap_or(input_mask, input_mask, core_mask);\n }\n } else {\n hwloc_bitmap_copy(input_mask, constraints_mask);\n }\n\n hwloc_bitmap_free(core_mask);\n hwloc_bitmap_free(constraints_mask);\n }\n\n void fit_num_threads_per_core(affinity_mask result_mask, affinity_mask current_mask, affinity_mask constraints_mask) {\n hwloc_bitmap_zero(result_mask);\n hwloc_obj_t current_core = nullptr;\n while ((current_core = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_CORE, current_core)) != nullptr) {\n if (hwloc_bitmap_intersects(current_mask, current_core->cpuset)) {\n hwloc_bitmap_or(result_mask, result_mask, current_core->cpuset);\n }\n }\n hwloc_bitmap_and(result_mask, result_mask, constraints_mask);\n }\n\n int get_default_concurrency(int numa_node_index, int core_type_index, int max_threads_per_core) {\n __TBB_ASSERT(is_topology_parsed(), \"Trying to get access to uninitialized system_topology\");\n\n hwloc_cpuset_t constraints_mask = hwloc_bitmap_alloc();\n fill_constraints_affinity_mask(constraints_mask, numa_node_index, core_type_index, max_threads_per_core);\n\n int default_concurrency = hwloc_bitmap_weight(constraints_mask);\n hwloc_bitmap_free(constraints_mask);\n return default_concurrency;\n }\n\n affinity_mask allocate_process_affinity_mask() {\n __TBB_ASSERT(is_topology_parsed(), \"Trying to get access to uninitialized system_topology\");\n return hwloc_bitmap_dup(process_cpu_affinity_mask);\n }\n\n void free_affinity_mask( affinity_mask mask_to_free ) {\n hwloc_bitmap_free(mask_to_free); // If bitmap is nullptr, no operation is performed.\n }\n\n void store_current_affinity_mask( affinity_mask current_mask ) {\n assertion_hwloc_wrapper(hwloc_get_cpubind, topology, current_mask, HWLOC_CPUBIND_THREAD);\n\n hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask);\n __TBB_ASSERT(!hwloc_bitmap_iszero(current_mask),\n \"Current affinity mask must intersects with process affinity mask\");\n }\n\n void set_affinity_mask( const_affinity_mask mask ) {\n if (hwloc_bitmap_weight(mask) > 0) {\n assertion_hwloc_wrapper(hwloc_set_cpubind, topology, mask, HWLOC_CPUBIND_THREAD);\n }\n }\n};\n\nsystem_topology* system_topology::instance_ptr{nullptr};\n\nclass binding_handler {\n // Following vector saves thread affinity mask on scheduler entry to return it to this thread \n // on scheduler exit.\n typedef std::vector affinity_masks_container;\n affinity_masks_container affinity_backup;\n system_topology::affinity_mask handler_affinity_mask;\n\n#ifdef _WIN32\n affinity_masks_container affinity_buffer;\n int my_numa_node_id;\n int my_core_type_id;\n int my_max_threads_per_core;\n#endif\n\npublic:\n binding_handler( std::size_t size, int numa_node_id, int core_type_id, int max_threads_per_core )\n : affinity_backup(size)\n#ifdef _WIN32\n , affinity_buffer(size)\n , my_numa_node_id(numa_node_id)\n , my_core_type_id(core_type_id)\n , my_max_threads_per_core(max_threads_per_core)\n#endif\n {\n for (std::size_t i = 0; i < size; ++i) {\n affinity_backup[i] = system_topology::instance().allocate_process_affinity_mask();\n#ifdef _WIN32\n affinity_buffer[i] = system_topology::instance().allocate_process_affinity_mask();\n#endif\n }\n handler_affinity_mask = system_topology::instance().allocate_process_affinity_mask();\n system_topology::instance().fill_constraints_affinity_mask\n (handler_affinity_mask, numa_node_id, core_type_id, max_threads_per_core);\n }\n\n ~binding_handler() {\n for (std::size_t i = 0; i < affinity_backup.size(); ++i) {\n system_topology::instance().free_affinity_mask(affinity_backup[i]);\n#ifdef _WIN32\n system_topology::instance().free_affinity_mask(affinity_buffer[i]);\n#endif\n }\n system_topology::instance().free_affinity_mask(handler_affinity_mask);\n }\n\n void apply_affinity( unsigned slot_num ) {\n auto& topology = system_topology::instance();\n __TBB_ASSERT(slot_num < affinity_backup.size(),\n \"The slot number is greater than the number of slots in the arena\");\n __TBB_ASSERT(topology.is_topology_parsed(),\n \"Trying to get access to uninitialized system_topology\");\n\n topology.store_current_affinity_mask(affinity_backup[slot_num]);\n\n#ifdef _WIN32\n // TBBBind supports only systems where NUMA nodes and core types do not cross the border\n // between several processor groups. So if a certain NUMA node or core type constraint\n // specified, then the constraints affinity mask will not cross the processor groups' border.\n\n // But if we have constraint based only on the max_threads_per_core setting, then the\n // constraints affinity mask does may cross the border between several processor groups\n // on machines with more then 64 hardware threads. That is why we need to use the special\n // function, which regulates the number of threads in the current threads mask.\n if (topology.number_of_processors_groups > 1 && my_max_threads_per_core != -1 &&\n (my_numa_node_id == -1 || topology.numa_indexes_list.size() == 1) &&\n (my_core_type_id == -1 || topology.core_types_indexes_list.size() == 1)\n ) {\n topology.fit_num_threads_per_core(affinity_buffer[slot_num], affinity_backup[slot_num], handler_affinity_mask);\n topology.set_affinity_mask(affinity_buffer[slot_num]);\n return;\n }\n#endif\n topology.set_affinity_mask(handler_affinity_mask);\n }\n\n void restore_previous_affinity_mask( unsigned slot_num ) {\n auto& topology = system_topology::instance();\n __TBB_ASSERT(topology.is_topology_parsed(),\n \"Trying to get access to uninitialized system_topology\");\n topology.set_affinity_mask(affinity_backup[slot_num]);\n };\n\n};\n\nextern \"C\" { // exported to TBB interfaces\n\nTBBBIND_EXPORT void __TBB_internal_initialize_system_topology(\n std::size_t groups_num,\n int& numa_nodes_count, int*& numa_indexes_list,\n int& core_types_count, int*& core_types_indexes_list\n) {\n system_topology::construct(groups_num);\n system_topology::instance().fill_topology_information(\n numa_nodes_count, numa_indexes_list,\n core_types_count, core_types_indexes_list\n );\n}\n\nTBBBIND_EXPORT binding_handler* __TBB_internal_allocate_binding_handler(int number_of_slots, int numa_id, int core_type_id, int max_threads_per_core) {\n __TBB_ASSERT(number_of_slots > 0, \"Trying to create numa handler for 0 threads.\");\n return new binding_handler(number_of_slots, numa_id, core_type_id, max_threads_per_core);\n}\n\nTBBBIND_EXPORT void __TBB_internal_deallocate_binding_handler(binding_handler* handler_ptr) {\n __TBB_ASSERT(handler_ptr != nullptr, \"Trying to deallocate nullptr pointer.\");\n delete handler_ptr;\n}\n\nTBBBIND_EXPORT void __TBB_internal_apply_affinity(binding_handler* handler_ptr, int slot_num) {\n __TBB_ASSERT(handler_ptr != nullptr, \"Trying to get access to uninitialized metadata.\");\n handler_ptr->apply_affinity(slot_num);\n}\n\nTBBBIND_EXPORT void __TBB_internal_restore_affinity(binding_handler* handler_ptr, int slot_num) {\n __TBB_ASSERT(handler_ptr != nullptr, \"Trying to get access to uninitialized metadata.\");\n handler_ptr->restore_previous_affinity_mask(slot_num);\n}\n\nTBBBIND_EXPORT int __TBB_internal_get_default_concurrency(int numa_id, int core_type_id, int max_threads_per_core) {\n return system_topology::instance().get_default_concurrency(numa_id, core_type_id, max_threads_per_core);\n}\n\nvoid __TBB_internal_destroy_system_topology() {\n return system_topology::destroy();\n}\n\n} // extern \"C\"\n\n} // namespace r1\n} // namespace detail\n} // namespace tbb\n\n#undef assertion_hwloc_wrapper\n" }, { "path": "src/tbb/src/tbbbind/tbb_bind.rc", "content": "// Copyright (c) 2005-2024 Intel Corporation\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Includes\n//\n#include \n#include \"../../include/oneapi/tbb/version.h\"\n\n/////////////////////////////////////////////////////////////////////////////\n// Neutral resources\n\n#ifdef _WIN32\nLANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL\n#pragma code_page(1252)\n#endif //_WIN32\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Version\n//\n#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH\n#define TBB_VERSION TBB_VERSION_STRING\n\nVS_VERSION_INFO VERSIONINFO\n FILEVERSION TBB_VERNUMBERS\n PRODUCTVERSION TBB_VERNUMBERS\n FILEFLAGSMASK 0x17L\n#ifdef _DEBUG\n FILEFLAGS 0x1L\n#else\n FILEFLAGS 0x0L\n#endif\n FILEOS 0x40004L\n FILETYPE 0x2L\n FILESUBTYPE 0x0L\nBEGIN\n BLOCK \"StringFileInfo\"\n BEGIN\n BLOCK \"000004b0\"\n BEGIN\n VALUE \"CompanyName\", \"Intel Corporation\\0\"\n VALUE \"FileDescription\", \"oneAPI Threading Building Blocks (oneTBB) library\\0\"\n VALUE \"FileVersion\", TBB_VERSION \"\\0\"\n VALUE \"LegalCopyright\", \"Copyright 2005-2024 Intel Corporation. All Rights Reserved.\\0\"\n VALUE \"LegalTrademarks\", \"\\0\"\n#ifndef TBB_USE_DEBUG\n VALUE \"OriginalFilename\", \"tbbbind.dll\\0\"\n#else\n VALUE \"OriginalFilename\", \"tbbbind_debug.dll\\0\"\n#endif\n VALUE \"ProductName\", \"oneAPI Threading Building Blocks (oneTBB)\\0\"\n VALUE \"ProductVersion\", TBB_VERSION \"\\0\"\n VALUE \"PrivateBuild\", \"\\0\"\n VALUE \"SpecialBuild\", \"\\0\"\n END\n END\n BLOCK \"VarFileInfo\"\n BEGIN\n VALUE \"Translation\", 0x0, 1200\n END\nEND\n" }, { "path": "src/tbb/src/tbbmalloc/CMakeLists.txt", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nadd_library(tbbmalloc \n backend.cpp\n backref.cpp\n frontend.cpp\n large_objects.cpp\n tbbmalloc.cpp\n ../tbb/itt_notify.cpp)\n \nif (WIN32)\n target_sources(tbbmalloc PRIVATE tbbmalloc.rc)\nendif()\n\nadd_library(TBB::tbbmalloc ALIAS tbbmalloc)\n\ntarget_compile_definitions(tbbmalloc\n PUBLIC\n $<$:TBB_USE_DEBUG>\n PRIVATE\n __TBBMALLOC_BUILD\n $<$>:__TBB_DYNAMIC_LOAD_ENABLED=0>\n $<$>:__TBB_SOURCE_DIRECTLY_INCLUDED=1>)\n\nif (NOT (\"${CMAKE_SYSTEM_PROCESSOR}\" MATCHES \"(armv7-a|aarch64|mips|arm64|riscv)\" OR\n \"${CMAKE_OSX_ARCHITECTURES}\" MATCHES \"arm64\" OR\n WINDOWS_STORE OR\n TBB_WINDOWS_DRIVER OR\n TBB_SANITIZE MATCHES \"thread\"))\n target_compile_definitions(tbbmalloc PRIVATE __TBB_USE_ITT_NOTIFY)\nendif()\n\ntarget_include_directories(tbbmalloc\n PUBLIC\n $\n $)\n\n# TODO: fix warnings\nif (MSVC)\n # signed unsigned mismatch, declaration hides class member\n set(TBB_WARNING_SUPPRESS ${TBB_WARNING_SUPPRESS} /wd4267 /wd4244 /wd4245 /wd4458)\nendif()\n\ntarget_compile_options(tbbmalloc\n PRIVATE\n ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC.\n ${TBB_MMD_FLAG}\n ${TBB_DSE_FLAG}\n ${TBB_WARNING_LEVEL}\n ${TBB_WARNING_SUPPRESS}\n ${TBB_LIB_COMPILE_FLAGS}\n ${TBBMALLOC_LIB_COMPILE_FLAGS}\n ${TBB_COMMON_COMPILE_FLAGS}\n)\n\nenable_language(C)\n\n# Avoid use of target_link_libraries here as it changes /DEF option to \\DEF on Windows.\nset_target_properties(tbbmalloc PROPERTIES\n DEFINE_SYMBOL \"\"\n LINKER_LANGUAGE C\n)\n\ntbb_handle_ipo(tbbmalloc)\n\nif (TBB_DEF_FILE_PREFIX) # If there's no prefix, assume we're using export directives\n set_target_properties(tbbmalloc PROPERTIES\n LINK_FLAGS \"${TBB_LINK_DEF_FILE_FLAG}\\\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbmalloc.def\\\"\"\n LINK_DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-tbbmalloc.def\"\n )\nendif()\n\nset(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES \"\")\n\n# Prefer using target_link_options instead of target_link_libraries to specify link options because\n# target_link_libraries may incorrectly handle some options (on Windows, for example).\nif (COMMAND target_link_options)\n target_link_options(tbbmalloc\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nelse()\n target_link_libraries(tbbmalloc\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nendif()\n\ntarget_link_libraries(tbbmalloc\n PRIVATE\n Threads::Threads\n ${TBB_LIB_LINK_LIBS}\n ${TBB_COMMON_LINK_LIBS}\n)\n\nif(TBB_BUILD_APPLE_FRAMEWORKS)\n set_target_properties(tbbmalloc PROPERTIES \n FRAMEWORK TRUE\n FRAMEWORK_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbbmalloc\n MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbbmalloc\n MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBBMALLOC_BINARY_VERSION}\n )\nendif()\n\ntbb_install_target(tbbmalloc)\n" }, { "path": "src/tbb/src/tbbmalloc/Customize.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _TBB_malloc_Customize_H_\n#define _TBB_malloc_Customize_H_\n\n// customizing MALLOC_ASSERT macro\n#define MALLOC_ASSERT(assertion, message) __TBB_ASSERT(assertion, message)\n#define MALLOC_ASSERT_EX(assertion, message) __TBB_ASSERT_EX(assertion, message)\n\n#ifndef MALLOC_DEBUG\n#define MALLOC_DEBUG TBB_USE_DEBUG\n#endif\n\n#include \"oneapi/tbb/detail/_utils.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n\n#include \"Synchronize.h\"\n\n#if __TBB_USE_ITT_NOTIFY\n#include \"../tbb/itt_notify.h\"\n#define MALLOC_ITT_SYNC_PREPARE(pointer) ITT_NOTIFY(sync_prepare, (pointer))\n#define MALLOC_ITT_SYNC_ACQUIRED(pointer) ITT_NOTIFY(sync_acquired, (pointer))\n#define MALLOC_ITT_SYNC_RELEASING(pointer) ITT_NOTIFY(sync_releasing, (pointer))\n#define MALLOC_ITT_SYNC_CANCEL(pointer) ITT_NOTIFY(sync_cancel, (pointer))\n#define MALLOC_ITT_FINI_ITTLIB() ITT_FINI_ITTLIB()\n#define MALLOC_ITT_RELEASE_RESOURCES() ITT_RELEASE_RESOURCES()\n#else\n#define MALLOC_ITT_SYNC_PREPARE(pointer) ((void)0)\n#define MALLOC_ITT_SYNC_ACQUIRED(pointer) ((void)0)\n#define MALLOC_ITT_SYNC_RELEASING(pointer) ((void)0)\n#define MALLOC_ITT_SYNC_CANCEL(pointer) ((void)0)\n#define MALLOC_ITT_FINI_ITTLIB() ((void)0)\n#define MALLOC_ITT_RELEASE_RESOURCES() ((void)0)\n#endif\n\ninline intptr_t BitScanRev(uintptr_t x) {\n return x == 0 ? -1 : static_cast(tbb::detail::log2(x));\n}\n\ntemplate\nstatic inline bool isAligned(T* arg, uintptr_t alignment) {\n return tbb::detail::is_aligned(arg,alignment);\n}\n\nstatic inline bool isPowerOfTwo(uintptr_t arg) {\n return tbb::detail::is_power_of_two(arg);\n}\nstatic inline bool isPowerOfTwoAtLeast(uintptr_t arg, uintptr_t power2) {\n return arg && tbb::detail::is_power_of_two_at_least(arg,power2);\n}\n\ninline void do_yield() {\n tbb::detail::yield();\n}\n\n#define USE_DEFAULT_MEMORY_MAPPING 1\n\n// To support malloc replacement\n#include \"../tbbmalloc_proxy/proxy.h\"\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED\n#define malloc_proxy __TBB_malloc_proxy\nextern \"C\" void * __TBB_malloc_proxy(size_t) __attribute__ ((weak));\n#elif MALLOC_ZONE_OVERLOAD_ENABLED\n// as there is no significant overhead, always suppose that proxy can be present\nconst bool malloc_proxy = true;\n#else\nconst bool malloc_proxy = false;\n#endif\n\nnamespace rml {\nnamespace internal {\n void init_tbbmalloc();\n} } // namespaces\n\n#define MALLOC_EXTRA_INITIALIZATION rml::internal::init_tbbmalloc()\n\n// Need these to work regardless of tools support.\nnamespace tbb {\nnamespace detail {\nnamespace d1 {\n\n enum notify_type {prepare=0, cancel, acquired, releasing};\n\n#if TBB_USE_PROFILING_TOOLS\n inline void call_itt_notify(notify_type t, void *ptr) {\n // unreferenced formal parameter warning\n detail::suppress_unused_warning(ptr);\n switch ( t ) {\n case prepare:\n MALLOC_ITT_SYNC_PREPARE( ptr );\n break;\n case cancel:\n MALLOC_ITT_SYNC_CANCEL( ptr );\n break;\n case acquired:\n MALLOC_ITT_SYNC_ACQUIRED( ptr );\n break;\n case releasing:\n MALLOC_ITT_SYNC_RELEASING( ptr );\n break;\n }\n }\n#else\n inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {}\n#endif // TBB_USE_PROFILING_TOOLS\n\n} // namespace d1\n} // namespace detail\n} // namespace tbb\n\n#include \"oneapi/tbb/detail/_aggregator.h\"\n\ntemplate \nstruct MallocAggregator {\n typedef tbb::detail::d1::aggregator_generic type;\n};\n\n//! aggregated_operation base class\ntemplate \nstruct MallocAggregatedOperation {\n typedef tbb::detail::d1::aggregated_operation type;\n};\n\n#endif /* _TBB_malloc_Customize_H_ */\n" }, { "path": "src/tbb/src/tbbmalloc/MapMemory.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _itt_shared_malloc_MapMemory_H\n#define _itt_shared_malloc_MapMemory_H\n\n#include \n\n#if __unix__ || __APPLE__ || __sun || __FreeBSD__\n\n#if __sun && !defined(_XPG4_2)\n // To have void* as mmap's 1st argument\n #define _XPG4_2 1\n #define XPG4_WAS_DEFINED 1\n#endif\n\n#include \n#if __unix__\n/* __TBB_MAP_HUGETLB is MAP_HUGETLB from system header linux/mman.h.\n The header is not included here, as on some Linux flavors inclusion of\n linux/mman.h leads to compilation error,\n while changing of MAP_HUGETLB is highly unexpected.\n*/\n#define __TBB_MAP_HUGETLB 0x40000\n#else\n#define __TBB_MAP_HUGETLB 0\n#endif\n\n#if XPG4_WAS_DEFINED\n #undef _XPG4_2\n #undef XPG4_WAS_DEFINED\n#endif\n\ninline void* mmap_impl(size_t map_size, void* map_hint = nullptr, int map_flags = 0) {\n#ifndef MAP_ANONYMOUS\n// macOS* defines MAP_ANON, which is deprecated in Linux*.\n#define MAP_ANONYMOUS MAP_ANON\n#endif /* MAP_ANONYMOUS */\n return mmap(map_hint, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | map_flags, -1, 0);\n}\n\ninline void* mmapTHP(size_t bytes) {\n // Initializes in zero-initialized data section\n static void* hint;\n\n // Optimistically try to use a last huge page aligned region end\n // as a hint for mmap.\n hint = hint ? (void*)((uintptr_t)hint - bytes) : hint;\n void* result = mmap_impl(bytes, hint);\n\n // Something went wrong\n if (result == MAP_FAILED) {\n hint = nullptr;\n return MAP_FAILED;\n }\n\n // Otherwise, fall back to the slow path - map oversized region\n // and trim excess parts.\n if (!isAligned(result, HUGE_PAGE_SIZE)) {\n // Undo previous try\n munmap(result, bytes);\n\n // Map oversized on huge page size region\n result = mmap_impl(bytes + HUGE_PAGE_SIZE);\n\n // Something went wrong\n if (result == MAP_FAILED) {\n hint = nullptr;\n return MAP_FAILED;\n }\n\n // Misalignment offset\n uintptr_t offset = 0;\n\n if (!isAligned(result, HUGE_PAGE_SIZE)) {\n // Trim excess head of a region if it is no aligned\n offset = HUGE_PAGE_SIZE - ((uintptr_t)result & (HUGE_PAGE_SIZE - 1));\n munmap(result, offset);\n\n // New region beginning\n result = (void*)((uintptr_t)result + offset);\n }\n\n // Trim excess tail of a region\n munmap((void*)((uintptr_t)result + bytes), HUGE_PAGE_SIZE - offset);\n }\n\n // Assume, that mmap virtual addresses grow down by default\n // So, set a hint as a result of a last successful allocation\n // and then use it minus requested size as a new mapping point.\n // TODO: Atomic store is meant here, fence not needed, but\n // currently we don't have such function.\n hint = result;\n\n MALLOC_ASSERT(isAligned(result, HUGE_PAGE_SIZE), \"Mapped address is not aligned on huge page size.\");\n\n return result;\n}\n\n#define MEMORY_MAPPING_USES_MALLOC 0\nvoid* MapMemory (size_t bytes, PageType pageType)\n{\n void* result = nullptr;\n int prevErrno = errno;\n\n switch (pageType) {\n case REGULAR:\n {\n result = mmap_impl(bytes);\n break;\n }\n case PREALLOCATED_HUGE_PAGE:\n {\n MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, \"Mapping size should be divisible by huge page size\");\n result = mmap_impl(bytes, nullptr, __TBB_MAP_HUGETLB);\n break;\n }\n case TRANSPARENT_HUGE_PAGE:\n {\n MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, \"Mapping size should be divisible by huge page size\");\n result = mmapTHP(bytes);\n break;\n }\n default:\n {\n MALLOC_ASSERT(false, \"Unknown page type\");\n }\n }\n\n if (result == MAP_FAILED) {\n errno = prevErrno;\n return nullptr;\n }\n\n return result;\n}\n\nint UnmapMemory(void *area, size_t bytes)\n{\n int prevErrno = errno;\n int ret = munmap(area, bytes);\n if (-1 == ret)\n errno = prevErrno;\n return ret;\n}\n\n#elif (_WIN32 || _WIN64) && !__TBB_WIN8UI_SUPPORT\n#include \n\n#define MEMORY_MAPPING_USES_MALLOC 0\nvoid* MapMemory (size_t bytes, PageType)\n{\n /* Is VirtualAlloc thread safe? */\n return VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);\n}\n\nint UnmapMemory(void *area, size_t /*bytes*/)\n{\n BOOL result = VirtualFree(area, 0, MEM_RELEASE);\n return !result;\n}\n\n#else\n\nvoid *ErrnoPreservingMalloc(size_t bytes)\n{\n int prevErrno = errno;\n void *ret = malloc( bytes );\n if (!ret)\n errno = prevErrno;\n return ret;\n}\n\n#define MEMORY_MAPPING_USES_MALLOC 1\nvoid* MapMemory (size_t bytes, PageType)\n{\n return ErrnoPreservingMalloc( bytes );\n}\n\nint UnmapMemory(void *area, size_t /*bytes*/)\n{\n free( area );\n return 0;\n}\n\n#endif /* OS dependent */\n\n#if MALLOC_CHECK_RECURSION && MEMORY_MAPPING_USES_MALLOC\n#error Impossible to protect against malloc recursion when memory mapping uses malloc.\n#endif\n\n#endif /* _itt_shared_malloc_MapMemory_H */\n" }, { "path": "src/tbb/src/tbbmalloc/Statistics.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#define MAX_THREADS 1024\n#define NUM_OF_BINS 30\n#define ThreadCommonCounters NUM_OF_BINS\n\nenum counter_type {\n allocBlockNew = 0,\n allocBlockPublic,\n allocBumpPtrUsed,\n allocFreeListUsed,\n allocPrivatized,\n examineEmptyEnough,\n examineNotEmpty,\n freeRestoreBumpPtr,\n freeByOtherThread,\n freeToActiveBlock,\n freeToInactiveBlock,\n freeBlockPublic,\n freeBlockBack,\n MaxCounters\n};\nenum common_counter_type {\n allocNewLargeObj = 0,\n allocCachedLargeObj,\n cacheLargeObj,\n freeLargeObj,\n lockPublicFreeList,\n freeToOtherThread\n};\n\n#if COLLECT_STATISTICS\n/* Statistics reporting callback registered via a static object dtor\n on Posix or DLL_PROCESS_DETACH on Windows.\n */\n\nstatic bool reportAllocationStatistics;\n\nstruct bin_counters {\n int counter[MaxCounters];\n};\n\nstatic bin_counters statistic[MAX_THREADS][NUM_OF_BINS+1]; //zero-initialized;\n\nstatic inline int STAT_increment(int thread, int bin, int ctr)\n{\n return reportAllocationStatistics && thread < MAX_THREADS ? ++(statistic[thread][bin].counter[ctr]) : 0;\n}\n\nstatic inline void initStatisticsCollection() {\n#if defined(MALLOCENV_COLLECT_STATISTICS)\n if (nullptr != getenv(MALLOCENV_COLLECT_STATISTICS))\n reportAllocationStatistics = true;\n#endif\n}\n\n#else\n#define STAT_increment(a,b,c) ((void)0)\n#endif /* COLLECT_STATISTICS */\n\n#if COLLECT_STATISTICS\nstatic inline void STAT_print(int thread)\n{\n if (!reportAllocationStatistics)\n return;\n\n char filename[100];\n#if USE_PTHREAD\n sprintf(filename, \"stat_ScalableMalloc_proc%04d_thr%04d.log\", getpid(), thread);\n#else\n sprintf(filename, \"stat_ScalableMalloc_thr%04d.log\", thread);\n#endif\n FILE* outfile = fopen(filename, \"w\");\n for(int i=0; i\n\n//! Stripped down version of spin_mutex.\n/** Instances of MallocMutex must be declared in memory that is zero-initialized.\n There are no constructors. This is a feature that lets it be\n used in situations where the mutex might be used while file-scope constructors\n are running.\n\n There are no methods \"acquire\" or \"release\". The scoped_lock must be used\n in a strict block-scoped locking pattern. Omitting these methods permitted\n further simplification. */\nclass MallocMutex : tbb::detail::no_copy {\n std::atomic_flag m_flag = ATOMIC_FLAG_INIT;\n\n void lock() {\n tbb::detail::atomic_backoff backoff;\n while (m_flag.test_and_set()) backoff.pause();\n }\n bool try_lock() {\n return !m_flag.test_and_set();\n }\n void unlock() {\n m_flag.clear(std::memory_order_release);\n }\n\npublic:\n class scoped_lock : tbb::detail::no_copy {\n MallocMutex& m_mutex;\n bool m_taken;\n\n public:\n scoped_lock(MallocMutex& m) : m_mutex(m), m_taken(true) {\n m.lock();\n }\n scoped_lock(MallocMutex& m, bool block, bool *locked) : m_mutex(m), m_taken(false) {\n if (block) {\n m.lock();\n m_taken = true;\n } else {\n m_taken = m.try_lock();\n }\n if (locked) *locked = m_taken;\n }\n\n scoped_lock(scoped_lock& other) = delete;\n scoped_lock& operator=(scoped_lock&) = delete;\n\n ~scoped_lock() {\n if (m_taken) {\n m_mutex.unlock();\n }\n }\n };\n friend class scoped_lock;\n};\n\ninline void SpinWaitWhileEq(const std::atomic& location, const intptr_t value) {\n tbb::detail::spin_wait_while_eq(location, value);\n}\n\n#if USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED\n\ninline void SpinWaitUntilEq(const std::atomic& location, const intptr_t value) {\n tbb::detail::spin_wait_until_eq(location, value);\n}\n\n#endif\n\nclass AtomicBackoff {\n tbb::detail::atomic_backoff backoff;\npublic:\n AtomicBackoff() {}\n void pause() { backoff.pause(); }\n};\n\n#endif /* __TBB_malloc_Synchronize_H_ */\n" }, { "path": "src/tbb/src/tbbmalloc/TypeDefinitions.h", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef _itt_shared_malloc_TypeDefinitions_H_\n#define _itt_shared_malloc_TypeDefinitions_H_\n\n// Define preprocessor symbols used to determine architecture\n#if _WIN32||_WIN64\n# if defined(_M_X64)||defined(__x86_64__) // the latter for MinGW support\n# define __ARCH_x86_64 1\n# elif defined(_M_IA64)\n# define __ARCH_ipf 1\n# elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support\n# define __ARCH_x86_32 1\n# elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) // the latter for MinGW support\n# define __ARCH_other 1\n# else\n# error Unknown processor architecture for Windows\n# endif\n# define USE_WINTHREAD 1\n#else /* Assume generic Unix */\n# if __x86_64__\n# define __ARCH_x86_64 1\n# elif __ia64__\n# define __ARCH_ipf 1\n# elif __i386__ || __i386\n# define __ARCH_x86_32 1\n# else\n# define __ARCH_other 1\n# endif\n# define USE_PTHREAD 1\n#endif\n\n// According to C99 standard INTPTR_MIN defined for C++\n// iff __STDC_LIMIT_MACROS pre-defined\n#ifndef __STDC_LIMIT_MACROS\n#define __STDC_LIMIT_MACROS 1\n#endif\n\n//! PROVIDE YOUR OWN Customize.h IF YOU FEEL NECESSARY\n#include \"Customize.h\"\n\n#include \"shared_utils.h\"\n\n#endif /* _itt_shared_malloc_TypeDefinitions_H_ */\n" }, { "path": "src/tbb/src/tbbmalloc/backend.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include /* for memset */\n#include \n#include \"tbbmalloc_internal.h\"\n\nnamespace rml {\nnamespace internal {\n\n/*********** Code to acquire memory from the OS or other executive ****************/\n\n/*\n syscall/malloc can set non-zero errno in case of failure,\n but later allocator might be able to find memory to fulfill the request.\n And we do not want changing of errno by successful scalable_malloc call.\n To support this, restore old errno in (get|free)RawMemory, and set errno\n in frontend just before returning to user code.\n Please note: every syscall/libc call used inside scalable_malloc that\n sets errno must be protected this way, not just memory allocation per se.\n*/\n\n#if USE_DEFAULT_MEMORY_MAPPING\n#include \"MapMemory.h\"\n#else\n/* assume MapMemory and UnmapMemory are customized */\n#endif\n\nvoid* getRawMemory (size_t size, PageType pageType) {\n return MapMemory(size, pageType);\n}\n\nint freeRawMemory (void *object, size_t size) {\n return UnmapMemory(object, size);\n}\n\n#if CHECK_ALLOCATION_RANGE\n\nvoid Backend::UsedAddressRange::registerAlloc(uintptr_t left, uintptr_t right)\n{\n MallocMutex::scoped_lock lock(mutex);\n if (left < leftBound.load(std::memory_order_relaxed))\n leftBound.store(left, std::memory_order_relaxed);\n if (right > rightBound.load(std::memory_order_relaxed))\n rightBound.store(right, std::memory_order_relaxed);\n MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed), ASSERT_TEXT);\n MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);\n MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) <= left && right <= rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);\n}\n\nvoid Backend::UsedAddressRange::registerFree(uintptr_t left, uintptr_t right)\n{\n MallocMutex::scoped_lock lock(mutex);\n if (leftBound.load(std::memory_order_relaxed) == left) {\n if (rightBound.load(std::memory_order_relaxed) == right) {\n leftBound.store(ADDRESS_UPPER_BOUND, std::memory_order_relaxed);\n rightBound.store(0, std::memory_order_relaxed);\n } else\n leftBound.store(right, std::memory_order_relaxed);\n } else if (rightBound.load(std::memory_order_relaxed) == right)\n rightBound.store(left, std::memory_order_relaxed);\n MALLOC_ASSERT((!rightBound.load(std::memory_order_relaxed) && leftBound.load(std::memory_order_relaxed) == ADDRESS_UPPER_BOUND)\n || leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);\n}\n#endif // CHECK_ALLOCATION_RANGE\n\n// Initialized in frontend inside defaultMemPool\nextern HugePagesStatus hugePages;\n\nvoid *Backend::allocRawMem(size_t &size)\n{\n void *res = nullptr;\n size_t allocSize = 0;\n\n if (extMemPool->userPool()) {\n if (extMemPool->fixedPool && bootsrapMemDone == bootsrapMemStatus.load(std::memory_order_acquire))\n return nullptr;\n MALLOC_ASSERT(bootsrapMemStatus != bootsrapMemNotDone,\n \"Backend::allocRawMem() called prematurely?\");\n // TODO: support for raw mem not aligned at sizeof(uintptr_t)\n // memory from fixed pool is asked once and only once\n allocSize = alignUpGeneric(size, extMemPool->granularity);\n res = (*extMemPool->rawAlloc)(extMemPool->poolId, allocSize);\n } else {\n // Align allocation on page size\n size_t pageSize = hugePages.isEnabled ? hugePages.getGranularity() : extMemPool->granularity;\n MALLOC_ASSERT(pageSize, \"Page size cannot be zero.\");\n allocSize = alignUpGeneric(size, pageSize);\n\n // If user requested huge pages and they are available, try to use preallocated ones firstly.\n // If there are none, lets check transparent huge pages support and use them instead.\n if (hugePages.isEnabled) {\n if (hugePages.isHPAvailable) {\n res = getRawMemory(allocSize, PREALLOCATED_HUGE_PAGE);\n }\n if (!res && hugePages.isTHPAvailable) {\n res = getRawMemory(allocSize, TRANSPARENT_HUGE_PAGE);\n }\n }\n\n if (!res) {\n res = getRawMemory(allocSize, REGULAR);\n }\n }\n\n if (res) {\n MALLOC_ASSERT(allocSize > 0, \"Invalid size of an allocated region.\");\n size = allocSize;\n if (!extMemPool->userPool())\n usedAddrRange.registerAlloc((uintptr_t)res, (uintptr_t)res+size);\n#if MALLOC_DEBUG\n volatile size_t curTotalSize = totalMemSize; // to read global value once\n MALLOC_ASSERT(curTotalSize+size > curTotalSize, \"Overflow allocation size.\");\n#endif\n totalMemSize.fetch_add(size);\n }\n\n return res;\n}\n\nbool Backend::freeRawMem(void *object, size_t size)\n{\n bool fail;\n#if MALLOC_DEBUG\n volatile size_t curTotalSize = totalMemSize; // to read global value once\n MALLOC_ASSERT(curTotalSize-size < curTotalSize, \"Negative allocation size.\");\n#endif\n totalMemSize.fetch_sub(size);\n if (extMemPool->userPool()) {\n MALLOC_ASSERT(!extMemPool->fixedPool, \"No free for fixed-size pools.\");\n fail = (*extMemPool->rawFree)(extMemPool->poolId, object, size);\n } else {\n usedAddrRange.registerFree((uintptr_t)object, (uintptr_t)object + size);\n fail = freeRawMemory(object, size);\n }\n // TODO: use result in all freeRawMem() callers\n return !fail;\n}\n\n/********* End memory acquisition code ********************************/\n\n// Protected object size. After successful locking returns size of locked block,\n// and releasing requires setting block size.\nclass GuardedSize : tbb::detail::no_copy {\n std::atomic value;\npublic:\n enum State {\n LOCKED,\n COAL_BLOCK, // block is coalescing now\n MAX_LOCKED_VAL = COAL_BLOCK,\n LAST_REGION_BLOCK, // used to mark last block in region\n // values after this are \"normal\" block sizes\n MAX_SPEC_VAL = LAST_REGION_BLOCK\n };\n\n void initLocked() { value.store(LOCKED, std::memory_order_release); } // TBB_REVAMP_TODO: was relaxed\n void makeCoalscing() {\n MALLOC_ASSERT(value.load(std::memory_order_relaxed) == LOCKED, ASSERT_TEXT);\n value.store(COAL_BLOCK, std::memory_order_release); // TBB_REVAMP_TODO: was relaxed\n }\n size_t tryLock(State state) {\n MALLOC_ASSERT(state <= MAX_LOCKED_VAL, ASSERT_TEXT);\n size_t sz = value.load(std::memory_order_acquire);\n for (;;) {\n if (sz <= MAX_LOCKED_VAL) {\n break;\n }\n if (value.compare_exchange_strong(sz, state)) {\n break;\n }\n }\n return sz;\n }\n void unlock(size_t size) {\n MALLOC_ASSERT(value.load(std::memory_order_relaxed) <= MAX_LOCKED_VAL, \"The lock is not locked\");\n MALLOC_ASSERT(size > MAX_LOCKED_VAL, ASSERT_TEXT);\n value.store(size, std::memory_order_release);\n }\n bool isLastRegionBlock() const { return value.load(std::memory_order_relaxed) == LAST_REGION_BLOCK; }\n friend void Backend::IndexedBins::verify();\n};\n\nstruct MemRegion {\n MemRegion *next, // keep all regions in any pool to release all them on\n *prev; // pool destroying, 2-linked list to release individual\n // regions.\n size_t allocSz, // got from pool callback\n blockSz; // initial and maximal inner block size\n MemRegionType type;\n};\n\n// this data must be unmodified while block is in use, so separate it\nclass BlockMutexes {\nprotected:\n GuardedSize myL, // lock for me\n leftL; // lock for left neighbor\n};\n\nclass FreeBlock : BlockMutexes {\npublic:\n static const size_t minBlockSize;\n friend void Backend::IndexedBins::verify();\n\n FreeBlock *prev, // in 2-linked list related to bin\n *next,\n *nextToFree; // used to form a queue during coalescing\n // valid only when block is in processing, i.e. one is not free and not\n size_t sizeTmp; // used outside of backend\n int myBin; // bin that is owner of the block\n bool slabAligned;\n bool blockInBin; // this block in myBin already\n\n FreeBlock *rightNeig(size_t sz) const {\n MALLOC_ASSERT(sz, ASSERT_TEXT);\n return (FreeBlock*)((uintptr_t)this+sz);\n }\n FreeBlock *leftNeig(size_t sz) const {\n MALLOC_ASSERT(sz, ASSERT_TEXT);\n return (FreeBlock*)((uintptr_t)this - sz);\n }\n\n void initHeader() { myL.initLocked(); leftL.initLocked(); }\n void setMeFree(size_t size) { myL.unlock(size); }\n size_t trySetMeUsed(GuardedSize::State s) { return myL.tryLock(s); }\n bool isLastRegionBlock() const { return myL.isLastRegionBlock(); }\n\n void setLeftFree(size_t sz) { leftL.unlock(sz); }\n size_t trySetLeftUsed(GuardedSize::State s) { return leftL.tryLock(s); }\n\n size_t tryLockBlock() {\n size_t rSz, sz = trySetMeUsed(GuardedSize::LOCKED);\n\n if (sz <= GuardedSize::MAX_LOCKED_VAL)\n return false;\n rSz = rightNeig(sz)->trySetLeftUsed(GuardedSize::LOCKED);\n if (rSz <= GuardedSize::MAX_LOCKED_VAL) {\n setMeFree(sz);\n return false;\n }\n MALLOC_ASSERT(rSz == sz, ASSERT_TEXT);\n return sz;\n }\n void markCoalescing(size_t blockSz) {\n myL.makeCoalscing();\n rightNeig(blockSz)->leftL.makeCoalscing();\n sizeTmp = blockSz;\n nextToFree = nullptr;\n }\n void markUsed() {\n myL.initLocked();\n rightNeig(sizeTmp)->leftL.initLocked();\n nextToFree = nullptr;\n }\n static void markBlocks(FreeBlock *fBlock, int num, size_t size) {\n for (int i=1; iinitHeader();\n }\n }\n};\n\n// Last block in any region. Its \"size\" field is GuardedSize::LAST_REGION_BLOCK,\n// This kind of blocks used to find region header\n// and have a possibility to return region back to OS\nstruct LastFreeBlock : public FreeBlock {\n MemRegion *memRegion;\n};\n\nconst size_t FreeBlock::minBlockSize = sizeof(FreeBlock);\n\ninline bool BackendSync::waitTillBlockReleased(intptr_t startModifiedCnt)\n{\n AtomicBackoff backoff;\n#if __TBB_MALLOC_BACKEND_STAT\n class ITT_Guard {\n void *ptr;\n public:\n ITT_Guard(void *p) : ptr(p) {\n MALLOC_ITT_SYNC_PREPARE(ptr);\n }\n ~ITT_Guard() {\n MALLOC_ITT_SYNC_ACQUIRED(ptr);\n }\n };\n ITT_Guard ittGuard(&inFlyBlocks);\n#endif\n intptr_t myBinsInFlyBlocks = inFlyBlocks.load(std::memory_order_acquire);\n intptr_t myCoalescQInFlyBlocks = backend->blocksInCoalescing();\n while (true) {\n MALLOC_ASSERT(myBinsInFlyBlocks>=0 && myCoalescQInFlyBlocks>=0, nullptr);\n\n intptr_t currBinsInFlyBlocks = inFlyBlocks.load(std::memory_order_acquire);\n intptr_t currCoalescQInFlyBlocks = backend->blocksInCoalescing();\n WhiteboxTestingYield();\n // Stop waiting iff:\n\n // 1) blocks were removed from processing, not added\n if (myBinsInFlyBlocks > currBinsInFlyBlocks\n // 2) released during delayed coalescing queue\n || myCoalescQInFlyBlocks > currCoalescQInFlyBlocks)\n break;\n // 3) if there are blocks in coalescing, and no progress in its processing,\n // try to scan coalescing queue and stop waiting, if changes were made\n // (if there are no changes and in-fly blocks exist, we continue\n // waiting to not increase load on coalescQ)\n if (currCoalescQInFlyBlocks > 0 && backend->scanCoalescQ(/*forceCoalescQDrop=*/false))\n break;\n // 4) when there are no blocks\n if (!currBinsInFlyBlocks && !currCoalescQInFlyBlocks) {\n // re-scan make sense only if bins were modified since scanned\n auto pool = backend->extMemPool;\n if (pool->hardCachesCleanupInProgress.load(std::memory_order_acquire) ||\n pool->softCachesCleanupInProgress.load(std::memory_order_acquire)) {\n backoff.pause();\n continue;\n }\n\n return startModifiedCnt != getNumOfMods();\n }\n myBinsInFlyBlocks = currBinsInFlyBlocks;\n myCoalescQInFlyBlocks = currCoalescQInFlyBlocks;\n backoff.pause();\n }\n return true;\n}\n\nvoid CoalRequestQ::putBlock(FreeBlock *fBlock)\n{\n MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, ASSERT_TEXT);\n fBlock->markUsed();\n // the block is in the queue, do not forget that it's here\n inFlyBlocks++;\n\n FreeBlock *myBlToFree = blocksToFree.load(std::memory_order_acquire);\n for (;;) {\n fBlock->nextToFree = myBlToFree;\n if (blocksToFree.compare_exchange_strong(myBlToFree, fBlock)) {\n return;\n }\n }\n}\n\nFreeBlock *CoalRequestQ::getAll()\n{\n for (;;) {\n FreeBlock *myBlToFree = blocksToFree.load(std::memory_order_acquire);\n\n if (!myBlToFree) {\n return nullptr;\n } else {\n if (blocksToFree.compare_exchange_strong(myBlToFree, nullptr)) {\n return myBlToFree;\n } else {\n continue;\n }\n }\n }\n}\n\ninline void CoalRequestQ::blockWasProcessed()\n{\n bkndSync->binsModified();\n int prev = inFlyBlocks.fetch_sub(1);\n tbb::detail::suppress_unused_warning(prev);\n MALLOC_ASSERT(prev > 0, ASSERT_TEXT);\n}\n\n// Try to get a block from a bin.\n// If the remaining free space would stay in the same bin,\n// split the block without removing it.\n// If the free space should go to other bin(s), remove the block.\n// alignedBin is true, if all blocks in the bin have slab-aligned right side.\nFreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, size_t size,\n bool needAlignedRes, bool alignedBin, bool wait, int *binLocked)\n{\n Bin *b = &freeBins[binIdx];\ntry_next:\n FreeBlock *fBlock = nullptr;\n if (!b->empty()) {\n bool locked = false;\n MallocMutex::scoped_lock scopedLock(b->tLock, wait, &locked);\n\n if (!locked) {\n if (binLocked) (*binLocked)++;\n return nullptr;\n }\n\n for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; curr = curr->next) {\n size_t szBlock = curr->tryLockBlock();\n if (!szBlock) {\n // block is locked, re-do bin lock, as there is no place to spin\n // while block coalescing\n goto try_next;\n }\n\n // GENERAL CASE\n if (alignedBin || !needAlignedRes) {\n size_t splitSz = szBlock - size;\n // If we got a block as split result, it must have a room for control structures.\n if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || !splitSz))\n fBlock = curr;\n } else {\n // SPECIAL CASE, to get aligned block from unaligned bin we have to cut the middle of a block\n // and return remaining left and right part. Possible only in fixed pool scenario, assert for this\n // is set inside splitBlock() function.\n\n void *newB = alignUp(curr, slabSize);\n uintptr_t rightNew = (uintptr_t)newB + size;\n uintptr_t rightCurr = (uintptr_t)curr + szBlock;\n // Check if the block size is sufficient,\n // and also left and right split results are either big enough or non-existent\n if (rightNew <= rightCurr\n && (newB == curr || ((uintptr_t)newB - (uintptr_t)curr) >= FreeBlock::minBlockSize)\n && (rightNew == rightCurr || (rightCurr - rightNew) >= FreeBlock::minBlockSize))\n fBlock = curr;\n }\n\n if (fBlock) {\n // consume must be called before result of removing from a bin is visible externally.\n sync->blockConsumed();\n // TODO: think about cases when block stays in the same bin\n b->removeBlock(fBlock);\n if (freeBins[binIdx].empty())\n bitMask.set(binIdx, false);\n fBlock->sizeTmp = szBlock;\n break;\n } else { // block size is not valid, search for next block in the bin\n curr->setMeFree(szBlock);\n curr->rightNeig(szBlock)->setLeftFree(szBlock);\n }\n }\n }\n return fBlock;\n}\n\nbool Backend::IndexedBins::tryReleaseRegions(int binIdx, Backend *backend)\n{\n Bin *b = &freeBins[binIdx];\n FreeBlock *fBlockList = nullptr;\n\n // got all blocks from the bin and re-do coalesce on them\n // to release single-block regions\ntry_next:\n if (!b->empty()) {\n MallocMutex::scoped_lock binLock(b->tLock);\n for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; ) {\n size_t szBlock = curr->tryLockBlock();\n if (!szBlock)\n goto try_next;\n\n FreeBlock *next = curr->next;\n\n b->removeBlock(curr);\n curr->sizeTmp = szBlock;\n curr->nextToFree = fBlockList;\n fBlockList = curr;\n curr = next;\n }\n }\n return backend->coalescAndPutList(fBlockList, /*forceCoalescQDrop=*/true,\n /*reportBlocksProcessed=*/false);\n}\n\nvoid Backend::Bin::removeBlock(FreeBlock *fBlock)\n{\n MALLOC_ASSERT(fBlock->next||fBlock->prev||fBlock== head.load(std::memory_order_relaxed),\n \"Detected that a block is not in the bin.\");\n if (head.load(std::memory_order_relaxed) == fBlock)\n head.store(fBlock->next, std::memory_order_relaxed);\n if (tail == fBlock)\n tail = fBlock->prev;\n if (fBlock->prev)\n fBlock->prev->next = fBlock->next;\n if (fBlock->next)\n fBlock->next->prev = fBlock->prev;\n}\n\nvoid Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t /* blockSz */, bool addToTail)\n{\n Bin *b = &freeBins[binIdx];\n fBlock->myBin = binIdx;\n fBlock->next = fBlock->prev = nullptr;\n {\n MallocMutex::scoped_lock scopedLock(b->tLock);\n if (addToTail) {\n fBlock->prev = b->tail;\n b->tail = fBlock;\n if (fBlock->prev)\n fBlock->prev->next = fBlock;\n if (!b->head.load(std::memory_order_relaxed))\n b->head.store(fBlock, std::memory_order_relaxed);\n } else {\n fBlock->next = b->head.load(std::memory_order_relaxed);\n b->head.store(fBlock, std::memory_order_relaxed);\n if (fBlock->next)\n fBlock->next->prev = fBlock;\n if (!b->tail)\n b->tail = fBlock;\n }\n }\n bitMask.set(binIdx, true);\n}\n\nbool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail)\n{\n bool locked = false;\n Bin *b = &freeBins[binIdx];\n fBlock->myBin = binIdx;\n if (addToTail) {\n fBlock->next = nullptr;\n {\n MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);\n if (!locked)\n return false;\n fBlock->prev = b->tail;\n b->tail = fBlock;\n if (fBlock->prev)\n fBlock->prev->next = fBlock;\n if (!b->head.load(std::memory_order_relaxed))\n b->head.store(fBlock, std::memory_order_relaxed);\n }\n } else {\n fBlock->prev = nullptr;\n {\n MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);\n if (!locked)\n return false;\n fBlock->next = b->head.load(std::memory_order_relaxed);\n b->head.store(fBlock, std::memory_order_relaxed);\n if (fBlock->next)\n fBlock->next->prev = fBlock;\n if (!b->tail)\n b->tail = fBlock;\n }\n }\n bitMask.set(binIdx, true);\n return true;\n}\n\nvoid Backend::IndexedBins::reset()\n{\n for (unsigned i=0; ifixedPool,\n \"Aligned block request from unaligned bin possible only in fixed pool scenario.\");\n\n // Space to use is in the middle\n FreeBlock *newBlock = alignUp(fBlock, slabSize);\n FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newBlock + totalSize);\n uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlock->sizeTmp;\n\n // Return free right part\n if ((uintptr_t)rightPart != fBlockEnd) {\n rightPart->initHeader(); // to prevent coalescing rightPart with fBlock\n size_t rightSize = fBlockEnd - (uintptr_t)rightPart;\n coalescAndPut(rightPart, rightSize, toAlignedBin(rightPart, rightSize));\n }\n // And free left part\n if (newBlock != fBlock) {\n newBlock->initHeader(); // to prevent coalescing fBlock with newB\n size_t leftSize = (uintptr_t)newBlock - (uintptr_t)fBlock;\n coalescAndPut(fBlock, leftSize, toAlignedBin(fBlock, leftSize));\n }\n fBlock = newBlock;\n } else if (size_t splitSize = fBlock->sizeTmp - totalSize) { // need to split the block\n // GENERAL CASE, cut the left or right part of the block\n FreeBlock *splitBlock = nullptr;\n if (needAlignedBlock) {\n // For slab aligned blocks cut the right side of the block\n // and return it to a requester, original block returns to backend\n splitBlock = fBlock;\n fBlock = (FreeBlock*)((uintptr_t)splitBlock + splitSize);\n fBlock->initHeader();\n } else {\n // For large object blocks cut original block and put free right part to backend\n splitBlock = (FreeBlock*)((uintptr_t)fBlock + totalSize);\n splitBlock->initHeader();\n }\n // Mark free block as it`s parent only when the requested type (needAlignedBlock)\n // and returned from Bins/OS block (isAligned) are equal (XOR operation used)\n bool markAligned = (blockIsAligned ^ needAlignedBlock) ? toAlignedBin(splitBlock, splitSize) : blockIsAligned;\n coalescAndPut(splitBlock, splitSize, markAligned);\n }\n MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), \"Expect to get aligned block, if one was requested.\");\n FreeBlock::markBlocks(fBlock, num, size);\n return fBlock;\n}\n\nsize_t Backend::getMaxBinnedSize() const\n{\n return hugePages.isEnabled && !inUserPool() ?\n maxBinned_HugePage : maxBinned_SmallPage;\n}\n\ninline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, size_t requestSize) const\n{\n return requestSize > oldMaxReq && requestSize < backend->getMaxBinnedSize();\n}\n\n// last chance to get memory\nFreeBlock *Backend::releaseMemInCaches(intptr_t startModifiedCnt,\n int *lockedBinsThreshold, int numOfLockedBins)\n{\n // something released from caches\n if (extMemPool->hardCachesCleanup(false))\n return (FreeBlock*)VALID_BLOCK_IN_BIN;\n\n if (bkndSync.waitTillBlockReleased(startModifiedCnt))\n return (FreeBlock*)VALID_BLOCK_IN_BIN;\n\n // OS can't give us more memory, but we have some in locked bins\n if (*lockedBinsThreshold && numOfLockedBins) {\n *lockedBinsThreshold = 0;\n return (FreeBlock*)VALID_BLOCK_IN_BIN;\n }\n return nullptr; // nothing found, give up\n}\n\nFreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt,\n int *lockedBinsThreshold, int numOfLockedBins,\n bool *splittableRet, bool needSlabRegion)\n{\n FreeBlock *block;\n // The block sizes can be divided into 3 groups:\n // 1. \"quite small\": popular object size, we are in bootstarp or something\n // like; request several regions.\n // 2. \"quite large\": we want to have several such blocks in the region\n // but not want several pre-allocated regions.\n // 3. \"huge\": exact fit, we allocate only one block and do not allow\n // any other allocations to placed in a region.\n // Dividing the block sizes in these groups we are trying to balance between\n // too small regions (that leads to fragmentation) and too large ones (that\n // leads to excessive address space consumption). If a region is \"too\n // large\", allocate only one, to prevent fragmentation. It supposedly\n // doesn't hurt performance, because the object requested by user is large.\n // Bounds for the groups are:\n const size_t maxBinned = getMaxBinnedSize();\n const size_t quiteSmall = maxBinned / 8;\n const size_t quiteLarge = maxBinned;\n\n if (blockSize >= quiteLarge) {\n // Do not interact with other threads via semaphores, as for exact fit\n // we can't share regions with them, memory requesting is individual.\n block = addNewRegion(blockSize, MEMREG_ONE_BLOCK, /*addToBin=*/false);\n if (!block)\n return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins);\n *splittableRet = false;\n } else {\n const size_t regSz_sizeBased = alignUp(4*maxRequestedSize, 1024*1024);\n // Another thread is modifying backend while we can't get the block.\n // Wait while it leaves and re-do the scan\n // before trying other ways to extend the backend.\n if (bkndSync.waitTillBlockReleased(startModifiedCnt)\n // semaphore is protecting adding more more memory from OS\n || memExtendingSema.wait())\n return (FreeBlock*)VALID_BLOCK_IN_BIN;\n\n if (startModifiedCnt != bkndSync.getNumOfMods()) {\n memExtendingSema.signal();\n return (FreeBlock*)VALID_BLOCK_IN_BIN;\n }\n\n if (blockSize < quiteSmall) {\n // For this size of blocks, add NUM_OF_REG \"advance\" regions in bin,\n // and return one as a result.\n // TODO: add to bin first, because other threads can use them right away.\n // This must be done carefully, because blocks in bins can be released\n // in releaseCachesToLimit().\n const unsigned NUM_OF_REG = 3;\n MemRegionType regType = needSlabRegion ? MEMREG_SLAB_BLOCKS : MEMREG_LARGE_BLOCKS;\n block = addNewRegion(regSz_sizeBased, regType, /*addToBin=*/false);\n if (block)\n for (unsigned idx=0; idxsoftCachesCleanup() &&\n (locTotalMemSize = totalMemSize.load(std::memory_order_acquire)) <=\n (locMemSoftLimit = memSoftLimit.load(std::memory_order_acquire)))\n return;\n // clean global large-object cache, if this is not enough, clean local caches\n // do this in several tries, because backend fragmentation can prevent\n // region from releasing\n for (int cleanLocal = 0; cleanLocal<2; cleanLocal++)\n while (cleanLocal ?\n extMemPool->allLocalCaches.cleanup(/*cleanOnlyUnused=*/true) :\n extMemPool->loc.decreasingCleanup())\n if ((locTotalMemSize = totalMemSize.load(std::memory_order_acquire)) <=\n (locMemSoftLimit = memSoftLimit.load(std::memory_order_acquire)))\n return;\n // last chance to match memSoftLimit\n extMemPool->hardCachesCleanup(true);\n}\n\nint Backend::IndexedBins::getMinNonemptyBin(unsigned startBin) const\n{\n int p = bitMask.getMinTrue(startBin);\n return p == -1 ? Backend::freeBinsNum : p;\n}\n\nFreeBlock *Backend::IndexedBins::findBlock(int nativeBin, BackendSync *sync, size_t size,\n bool needAlignedBlock, bool alignedBin, int *numOfLockedBins)\n{\n for (int i=getMinNonemptyBin(nativeBin); i<(int)freeBinsNum; i=getMinNonemptyBin(i+1))\n if (FreeBlock *block = getFromBin(i, sync, size, needAlignedBlock, alignedBin, /*wait=*/false, numOfLockedBins))\n return block;\n\n return nullptr;\n}\n\nvoid Backend::requestBootstrapMem()\n{\n if (bootsrapMemDone == bootsrapMemStatus.load(std::memory_order_acquire))\n return;\n MallocMutex::scoped_lock lock( bootsrapMemStatusMutex );\n if (bootsrapMemDone == bootsrapMemStatus)\n return;\n MALLOC_ASSERT(bootsrapMemNotDone == bootsrapMemStatus, ASSERT_TEXT);\n bootsrapMemStatus = bootsrapMemInitializing;\n // request some rather big region during bootstrap in advance\n // ok to get nullptr here, as later we re-do a request with more modest size\n addNewRegion(2*1024*1024, MEMREG_SLAB_BLOCKS, /*addToBin=*/true);\n bootsrapMemStatus = bootsrapMemDone;\n}\n\n// try to allocate size Byte block in available bins\n// needAlignedRes is true if result must be slab-aligned\nFreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock)\n{\n FreeBlock *block = nullptr;\n const size_t totalReqSize = num*size;\n // no splitting after requesting new region, asks exact size\n const int nativeBin = sizeToBin(totalReqSize);\n\n requestBootstrapMem();\n // If we found 2 or less locked bins, it's time to ask more memory from OS.\n // But nothing can be asked from fixed pool. And we prefer wait, not ask\n // for more memory, if block is quite large.\n int lockedBinsThreshold = extMemPool->fixedPool || size>=maxBinned_SmallPage? 0 : 2;\n\n // Find maximal requested size limited by getMaxBinnedSize()\n AtomicUpdate(maxRequestedSize, totalReqSize, MaxRequestComparator(this));\n scanCoalescQ(/*forceCoalescQDrop=*/false);\n\n bool splittable = true;\n for (;;) {\n const intptr_t startModifiedCnt = bkndSync.getNumOfMods();\n int numOfLockedBins;\n intptr_t cleanCnt;\n do {\n cleanCnt = backendCleanCnt.load(std::memory_order_acquire);\n numOfLockedBins = 0;\n if (needAlignedBlock) {\n block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,\n /*alignedBin=*/true, &numOfLockedBins);\n if (!block && extMemPool->fixedPool)\n block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,\n /*alignedBin=*/false, &numOfLockedBins);\n } else {\n block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,\n /*alignedBin=*/false, &numOfLockedBins);\n if (!block && extMemPool->fixedPool)\n block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,\n /*alignedBin=*/true, &numOfLockedBins);\n }\n } while (!block && (numOfLockedBins>lockedBinsThreshold || cleanCnt % 2 == 1 ||\n cleanCnt != backendCleanCnt.load(std::memory_order_acquire)));\n\n if (block)\n break;\n\n bool retScanCoalescQ = scanCoalescQ(/*forceCoalescQDrop=*/true);\n bool retSoftCachesCleanup = extMemPool->softCachesCleanup();\n if (!(retScanCoalescQ || retSoftCachesCleanup)) {\n // bins are not updated,\n // only remaining possibility is to ask for more memory\n block = askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold,\n numOfLockedBins, &splittable, needAlignedBlock);\n if (!block)\n return nullptr;\n if (block != (FreeBlock*)VALID_BLOCK_IN_BIN) {\n // size can be increased in askMemFromOS, that's why >=\n MALLOC_ASSERT(block->sizeTmp >= size, ASSERT_TEXT);\n break;\n }\n // valid block somewhere in bins, let's find it\n block = nullptr;\n }\n }\n MALLOC_ASSERT(block, ASSERT_TEXT);\n if (splittable) {\n // At this point we have to be sure that slabAligned attribute describes the right block state\n block = splitBlock(block, num, size, block->slabAligned, needAlignedBlock);\n }\n // matched blockConsumed() from startUseBlock()\n bkndSync.blockReleased();\n\n return block;\n}\n\nLargeMemoryBlock *Backend::getLargeBlock(size_t size)\n{\n LargeMemoryBlock *lmb =\n (LargeMemoryBlock*)genericGetBlock(1, size, /*needAlignedRes=*/false);\n if (lmb) {\n lmb->unalignedSize = size;\n if (extMemPool->userPool())\n extMemPool->lmbList.add(lmb);\n }\n return lmb;\n}\n\nBlockI *Backend::getSlabBlock(int num) {\n BlockI *b = (BlockI*)genericGetBlock(num, slabSize, /*slabAligned=*/true);\n MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT);\n return b;\n}\n\nvoid Backend::putSlabBlock(BlockI *block) {\n genericPutBlock((FreeBlock *)block, slabSize, /*slabAligned=*/true);\n}\n\nvoid *Backend::getBackRefSpace(size_t size, bool *rawMemUsed)\n{\n // This block is released only at shutdown, so it can prevent\n // a entire region releasing when it's received from the backend,\n // so prefer getRawMemory using.\n if (void *ret = getRawMemory(size, REGULAR)) {\n *rawMemUsed = true;\n return ret;\n }\n void *ret = genericGetBlock(1, size, /*needAlignedRes=*/false);\n if (ret) *rawMemUsed = false;\n return ret;\n}\n\nvoid Backend::putBackRefSpace(void *b, size_t size, bool rawMemUsed)\n{\n if (rawMemUsed)\n freeRawMemory(b, size);\n // ignore not raw mem, as it released on region releasing\n}\n\nvoid Backend::removeBlockFromBin(FreeBlock *fBlock)\n{\n if (fBlock->myBin != Backend::NO_BIN) {\n if (fBlock->slabAligned)\n freeSlabAlignedBins.lockRemoveBlock(fBlock->myBin, fBlock);\n else\n freeLargeBlockBins.lockRemoveBlock(fBlock->myBin, fBlock);\n }\n}\n\nvoid Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned)\n{\n bkndSync.blockConsumed();\n coalescAndPut(fBlock, blockSz, slabAligned);\n bkndSync.blockReleased();\n}\n\nvoid AllLargeBlocksList::add(LargeMemoryBlock *lmb)\n{\n MallocMutex::scoped_lock scoped_cs(largeObjLock);\n lmb->gPrev = nullptr;\n lmb->gNext = loHead;\n if (lmb->gNext)\n lmb->gNext->gPrev = lmb;\n loHead = lmb;\n}\n\nvoid AllLargeBlocksList::remove(LargeMemoryBlock *lmb)\n{\n MallocMutex::scoped_lock scoped_cs(largeObjLock);\n if (loHead == lmb)\n loHead = lmb->gNext;\n if (lmb->gNext)\n lmb->gNext->gPrev = lmb->gPrev;\n if (lmb->gPrev)\n lmb->gPrev->gNext = lmb->gNext;\n}\n\nvoid Backend::putLargeBlock(LargeMemoryBlock *lmb)\n{\n if (extMemPool->userPool())\n extMemPool->lmbList.remove(lmb);\n genericPutBlock((FreeBlock *)lmb, lmb->unalignedSize, false);\n}\n\nvoid Backend::returnLargeObject(LargeMemoryBlock *lmb)\n{\n removeBackRef(lmb->backRefIdx);\n putLargeBlock(lmb);\n STAT_increment(getThreadId(), ThreadCommonCounters, freeLargeObj);\n}\n\n#if BACKEND_HAS_MREMAP\nvoid *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment)\n{\n // no remap for user pools and for object too small that living in bins\n if (inUserPool() || min(oldSize, newSize)extMemPool->granularity)\n return nullptr;\n const LargeMemoryBlock* lmbOld = ((LargeObjectHdr *)ptr - 1)->memoryBlock;\n const size_t oldUnalignedSize = lmbOld->unalignedSize;\n FreeBlock *oldFBlock = (FreeBlock *)lmbOld;\n FreeBlock *right = oldFBlock->rightNeig(oldUnalignedSize);\n // in every region only one block can have LAST_REGION_BLOCK on right,\n // so don't need no synchronization\n if (!right->isLastRegionBlock())\n return nullptr;\n\n MemRegion *oldRegion = static_cast(right)->memRegion;\n MALLOC_ASSERT( oldRegion < ptr, ASSERT_TEXT );\n const size_t oldRegionSize = oldRegion->allocSz;\n if (oldRegion->type != MEMREG_ONE_BLOCK)\n return nullptr; // we are not single in the region\n const size_t userOffset = (uintptr_t)ptr - (uintptr_t)oldRegion;\n const size_t alignedSize = LargeObjectCache::alignToBin(newSize + userOffset);\n const size_t requestSize =\n alignUp(sizeof(MemRegion) + alignedSize + sizeof(LastFreeBlock), extMemPool->granularity);\n if (requestSize < alignedSize) // is wrapped around?\n return nullptr;\n regionList.remove(oldRegion);\n\n // The deallocation should be registered in address range before mremap to\n // prevent a race condition with allocation on another thread.\n // (OS can reuse the memory and registerAlloc will be missed on another thread)\n usedAddrRange.registerFree((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize);\n\n void *ret = mremap(oldRegion, oldRegion->allocSz, requestSize, MREMAP_MAYMOVE);\n if (MAP_FAILED == ret) { // can't remap, revert and leave\n regionList.add(oldRegion);\n usedAddrRange.registerAlloc((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize);\n return nullptr;\n }\n MemRegion *region = (MemRegion*)ret;\n MALLOC_ASSERT(region->type == MEMREG_ONE_BLOCK, ASSERT_TEXT);\n region->allocSz = requestSize;\n region->blockSz = alignedSize;\n\n FreeBlock *fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion),\n largeObjectAlignment);\n\n regionList.add(region);\n startUseBlock(region, fBlock, /*addToBin=*/false);\n MALLOC_ASSERT(fBlock->sizeTmp == region->blockSz, ASSERT_TEXT);\n // matched blockConsumed() in startUseBlock().\n // TODO: get rid of useless pair blockConsumed()/blockReleased()\n bkndSync.blockReleased();\n\n // object must start at same offset from region's start\n void *object = (void*)((uintptr_t)region + userOffset);\n MALLOC_ASSERT(isAligned(object, alignment), ASSERT_TEXT);\n LargeObjectHdr *header = (LargeObjectHdr*)object - 1;\n setBackRef(header->backRefIdx, header);\n\n LargeMemoryBlock *lmb = (LargeMemoryBlock*)fBlock;\n lmb->unalignedSize = region->blockSz;\n lmb->objectSize = newSize;\n lmb->backRefIdx = header->backRefIdx;\n header->memoryBlock = lmb;\n MALLOC_ASSERT((uintptr_t)lmb + lmb->unalignedSize >=\n (uintptr_t)object + lmb->objectSize, \"An object must fit to the block.\");\n\n usedAddrRange.registerAlloc((uintptr_t)region, (uintptr_t)region + requestSize);\n totalMemSize.fetch_add(region->allocSz - oldRegionSize);\n\n return object;\n}\n#endif /* BACKEND_HAS_MREMAP */\n\nvoid Backend::releaseRegion(MemRegion *memRegion)\n{\n regionList.remove(memRegion);\n freeRawMem(memRegion, memRegion->allocSz);\n}\n\n// coalesce fBlock with its neighborhood\nFreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion)\n{\n FreeBlock *resBlock = fBlock;\n size_t resSize = fBlock->sizeTmp;\n MemRegion *memRegion = nullptr;\n\n fBlock->markCoalescing(resSize);\n resBlock->blockInBin = false;\n\n // coalescing with left neighbor\n size_t leftSz = fBlock->trySetLeftUsed(GuardedSize::COAL_BLOCK);\n if (leftSz != GuardedSize::LOCKED) {\n if (leftSz == GuardedSize::COAL_BLOCK) {\n coalescQ.putBlock(fBlock);\n return nullptr;\n } else {\n FreeBlock *left = fBlock->leftNeig(leftSz);\n size_t lSz = left->trySetMeUsed(GuardedSize::COAL_BLOCK);\n if (lSz <= GuardedSize::MAX_LOCKED_VAL) {\n fBlock->setLeftFree(leftSz); // rollback\n coalescQ.putBlock(fBlock);\n return nullptr;\n } else {\n MALLOC_ASSERT(lSz == leftSz, \"Invalid header\");\n left->blockInBin = true;\n resBlock = left;\n resSize += leftSz;\n resBlock->sizeTmp = resSize;\n }\n }\n }\n // coalescing with right neighbor\n FreeBlock *right = fBlock->rightNeig(fBlock->sizeTmp);\n size_t rightSz = right->trySetMeUsed(GuardedSize::COAL_BLOCK);\n if (rightSz != GuardedSize::LOCKED) {\n // LastFreeBlock is on the right side\n if (GuardedSize::LAST_REGION_BLOCK == rightSz) {\n right->setMeFree(GuardedSize::LAST_REGION_BLOCK);\n memRegion = static_cast(right)->memRegion;\n } else if (GuardedSize::COAL_BLOCK == rightSz) {\n if (resBlock->blockInBin) {\n resBlock->blockInBin = false;\n removeBlockFromBin(resBlock);\n }\n coalescQ.putBlock(resBlock);\n return nullptr;\n } else {\n size_t rSz = right->rightNeig(rightSz)->\n trySetLeftUsed(GuardedSize::COAL_BLOCK);\n if (rSz <= GuardedSize::MAX_LOCKED_VAL) {\n right->setMeFree(rightSz); // rollback\n if (resBlock->blockInBin) {\n resBlock->blockInBin = false;\n removeBlockFromBin(resBlock);\n }\n coalescQ.putBlock(resBlock);\n return nullptr;\n } else {\n MALLOC_ASSERT(rSz == rightSz, \"Invalid header\");\n removeBlockFromBin(right);\n resSize += rightSz;\n\n // Is LastFreeBlock on the right side of right?\n FreeBlock *nextRight = right->rightNeig(rightSz);\n size_t nextRightSz = nextRight->\n trySetMeUsed(GuardedSize::COAL_BLOCK);\n if (nextRightSz > GuardedSize::MAX_LOCKED_VAL) {\n if (nextRightSz == GuardedSize::LAST_REGION_BLOCK)\n memRegion = static_cast(nextRight)->memRegion;\n\n nextRight->setMeFree(nextRightSz);\n }\n }\n }\n }\n if (memRegion) {\n MALLOC_ASSERT((uintptr_t)memRegion + memRegion->allocSz >=\n (uintptr_t)right + sizeof(LastFreeBlock), ASSERT_TEXT);\n MALLOC_ASSERT((uintptr_t)memRegion < (uintptr_t)resBlock, ASSERT_TEXT);\n *mRegion = memRegion;\n } else\n *mRegion = nullptr;\n resBlock->sizeTmp = resSize;\n return resBlock;\n}\n\nbool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, bool reportBlocksProcessed)\n{\n bool regionReleased = false;\n\n for (FreeBlock *helper; list;\n list = helper,\n // matches block enqueue in CoalRequestQ::putBlock()\n reportBlocksProcessed? coalescQ.blockWasProcessed() : (void)0) {\n MemRegion *memRegion;\n bool addToTail = false;\n\n helper = list->nextToFree;\n FreeBlock *toRet = doCoalesc(list, &memRegion);\n if (!toRet)\n continue;\n\n if (memRegion && memRegion->blockSz == toRet->sizeTmp\n && !extMemPool->fixedPool) {\n if (extMemPool->regionsAreReleaseable()) {\n // release the region, because there is no used blocks in it\n if (toRet->blockInBin)\n removeBlockFromBin(toRet);\n releaseRegion(memRegion);\n regionReleased = true;\n continue;\n } else // add block from empty region to end of bin,\n addToTail = true; // preserving for exact fit\n }\n size_t currSz = toRet->sizeTmp;\n int bin = sizeToBin(currSz);\n bool toAligned = extMemPool->fixedPool ? toAlignedBin(toRet, currSz) : toRet->slabAligned;\n bool needAddToBin = true;\n\n if (toRet->blockInBin) {\n // Does it stay in same bin?\n if (toRet->myBin == bin && toRet->slabAligned == toAligned)\n needAddToBin = false;\n else {\n toRet->blockInBin = false;\n removeBlockFromBin(toRet);\n }\n }\n\n // Does not stay in same bin, or bin-less; add it\n if (needAddToBin) {\n toRet->prev = toRet->next = toRet->nextToFree = nullptr;\n toRet->myBin = NO_BIN;\n toRet->slabAligned = toAligned;\n\n // If the block is too small to fit in any bin, keep it bin-less.\n // It's not a leak because the block later can be coalesced.\n if (currSz >= minBinnedSize) {\n toRet->sizeTmp = currSz;\n IndexedBins *target = toRet->slabAligned ? &freeSlabAlignedBins : &freeLargeBlockBins;\n if (forceCoalescQDrop) {\n target->addBlock(bin, toRet, toRet->sizeTmp, addToTail);\n } else if (!target->tryAddBlock(bin, toRet, addToTail)) {\n coalescQ.putBlock(toRet);\n continue;\n }\n }\n toRet->sizeTmp = 0;\n }\n // Free (possibly coalesced) free block.\n // Adding to bin must be done before this point,\n // because after a block is free it can be coalesced, and\n // using its pointer became unsafe.\n // Remember that coalescing is not done under any global lock.\n toRet->setMeFree(currSz);\n toRet->rightNeig(currSz)->setLeftFree(currSz);\n }\n return regionReleased;\n}\n\n// Coalesce fBlock and add it back to a bin;\n// processing delayed coalescing requests.\nvoid Backend::coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned)\n{\n fBlock->sizeTmp = blockSz;\n fBlock->nextToFree = nullptr;\n fBlock->slabAligned = slabAligned;\n\n coalescAndPutList(fBlock, /*forceCoalescQDrop=*/false, /*reportBlocksProcessed=*/false);\n}\n\nbool Backend::scanCoalescQ(bool forceCoalescQDrop)\n{\n FreeBlock *currCoalescList = coalescQ.getAll();\n\n if (currCoalescList)\n // reportBlocksProcessed=true informs that the blocks leave coalescQ,\n // matches blockConsumed() from CoalRequestQ::putBlock()\n coalescAndPutList(currCoalescList, forceCoalescQDrop,\n /*reportBlocksProcessed=*/true);\n // returns status of coalescQ.getAll(), as an indication of possible changes in backend\n // TODO: coalescAndPutList() may report is some new free blocks became available or not\n return currCoalescList;\n}\n\nFreeBlock *Backend::findBlockInRegion(MemRegion *region, size_t exactBlockSize)\n{\n FreeBlock *fBlock;\n size_t blockSz;\n uintptr_t fBlockEnd,\n lastFreeBlock = (uintptr_t)region + region->allocSz - sizeof(LastFreeBlock);\n\n static_assert(sizeof(LastFreeBlock) % sizeof(uintptr_t) == 0,\n \"Atomic applied on LastFreeBlock, and we put it at the end of region, that\"\n \" is uintptr_t-aligned, so no unaligned atomic operations are possible.\");\n // right bound is slab-aligned, keep LastFreeBlock after it\n if (region->type == MEMREG_SLAB_BLOCKS) {\n fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), sizeof(uintptr_t));\n fBlockEnd = alignDown(lastFreeBlock, slabSize);\n } else {\n fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment);\n fBlockEnd = (uintptr_t)fBlock + exactBlockSize;\n MALLOC_ASSERT(fBlockEnd <= lastFreeBlock, ASSERT_TEXT);\n }\n if (fBlockEnd <= (uintptr_t)fBlock)\n return nullptr; // allocSz is too small\n blockSz = fBlockEnd - (uintptr_t)fBlock;\n // TODO: extend getSlabBlock to support degradation, i.e. getting less blocks\n // then requested, and then relax this check\n // (now all or nothing is implemented, check according to this)\n if (blockSz < numOfSlabAllocOnMiss*slabSize)\n return nullptr;\n\n region->blockSz = blockSz;\n return fBlock;\n}\n\n// startUseBlock may add the free block to a bin, the block can be used and\n// even released after this, so the region must be added to regionList already\nvoid Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin)\n{\n size_t blockSz = region->blockSz;\n fBlock->initHeader();\n fBlock->setMeFree(blockSz);\n\n LastFreeBlock *lastBl = static_cast(fBlock->rightNeig(blockSz));\n // to not get unaligned atomics during LastFreeBlock access\n MALLOC_ASSERT(isAligned(lastBl, sizeof(uintptr_t)), nullptr);\n lastBl->initHeader();\n lastBl->setMeFree(GuardedSize::LAST_REGION_BLOCK);\n lastBl->setLeftFree(blockSz);\n lastBl->myBin = NO_BIN;\n lastBl->memRegion = region;\n\n if (addToBin) {\n unsigned targetBin = sizeToBin(blockSz);\n // during adding advance regions, register bin for a largest block in region\n advRegBins.registerBin(targetBin);\n if (region->type == MEMREG_SLAB_BLOCKS) {\n fBlock->slabAligned = true;\n freeSlabAlignedBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false);\n } else {\n fBlock->slabAligned = false;\n freeLargeBlockBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false);\n }\n } else {\n // to match with blockReleased() in genericGetBlock\n bkndSync.blockConsumed();\n // Understand our alignment for correct splitBlock operation\n fBlock->slabAligned = region->type == MEMREG_SLAB_BLOCKS ? true : false;\n fBlock->sizeTmp = fBlock->tryLockBlock();\n MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, \"Locking must be successful\");\n }\n}\n\nvoid MemRegionList::add(MemRegion *r)\n{\n r->prev = nullptr;\n MallocMutex::scoped_lock lock(regionListLock);\n r->next = head;\n head = r;\n if (head->next)\n head->next->prev = head;\n}\n\nvoid MemRegionList::remove(MemRegion *r)\n{\n MallocMutex::scoped_lock lock(regionListLock);\n if (head == r)\n head = head->next;\n if (r->next)\n r->next->prev = r->prev;\n if (r->prev)\n r->prev->next = r->next;\n}\n\n#if __TBB_MALLOC_BACKEND_STAT\nint MemRegionList::reportStat(FILE *f)\n{\n int regNum = 0;\n MallocMutex::scoped_lock lock(regionListLock);\n for (MemRegion *curr = head; curr; curr = curr->next) {\n fprintf(f, \"%p: max block %lu B, \", curr, curr->blockSz);\n regNum++;\n }\n return regNum;\n}\n#endif\n\nFreeBlock *Backend::addNewRegion(size_t size, MemRegionType memRegType, bool addToBin)\n{\n static_assert(sizeof(BlockMutexes) <= sizeof(BlockI), \"Header must be not overwritten in used blocks\");\n MALLOC_ASSERT(FreeBlock::minBlockSize > GuardedSize::MAX_SPEC_VAL,\n \"Block length must not conflict with special values of GuardedSize\");\n // If the region is not \"for slabs\" we should reserve some space for\n // a region header, the worst case alignment and the last block mark.\n const size_t requestSize = memRegType == MEMREG_SLAB_BLOCKS ? size :\n size + sizeof(MemRegion) + largeObjectAlignment\n + FreeBlock::minBlockSize + sizeof(LastFreeBlock);\n\n size_t rawSize = requestSize;\n MemRegion *region = (MemRegion*)allocRawMem(rawSize);\n if (!region) {\n MALLOC_ASSERT(rawSize==requestSize, \"getRawMem has not allocated memory but changed the allocated size.\");\n return nullptr;\n }\n if (rawSize < sizeof(MemRegion)) {\n if (!extMemPool->fixedPool)\n freeRawMem(region, rawSize);\n return nullptr;\n }\n\n region->type = memRegType;\n region->allocSz = rawSize;\n FreeBlock *fBlock = findBlockInRegion(region, size);\n if (!fBlock) {\n if (!extMemPool->fixedPool)\n freeRawMem(region, rawSize);\n return nullptr;\n }\n regionList.add(region);\n startUseBlock(region, fBlock, addToBin);\n bkndSync.binsModified();\n return addToBin? (FreeBlock*)VALID_BLOCK_IN_BIN : fBlock;\n}\n\nvoid Backend::init(ExtMemoryPool *extMemoryPool)\n{\n extMemPool = extMemoryPool;\n usedAddrRange.init();\n coalescQ.init(&bkndSync);\n bkndSync.init(this);\n}\n\nvoid Backend::reset()\n{\n MALLOC_ASSERT(extMemPool->userPool(), \"Only user pool can be reset.\");\n // no active threads are allowed in backend while reset() called\n verify();\n\n freeLargeBlockBins.reset();\n freeSlabAlignedBins.reset();\n advRegBins.reset();\n\n for (MemRegion *curr = regionList.head; curr; curr = curr->next) {\n FreeBlock *fBlock = findBlockInRegion(curr, curr->blockSz);\n MALLOC_ASSERT(fBlock, \"A memory region unexpectedly got smaller\");\n startUseBlock(curr, fBlock, /*addToBin=*/true);\n }\n}\n\nbool Backend::destroy()\n{\n bool noError = true;\n // no active threads are allowed in backend while destroy() called\n verify();\n if (!inUserPool()) {\n freeLargeBlockBins.reset();\n freeSlabAlignedBins.reset();\n }\n while (regionList.head) {\n MemRegion *helper = regionList.head->next;\n noError &= freeRawMem(regionList.head, regionList.head->allocSz);\n regionList.head = helper;\n }\n return noError;\n}\n\nbool Backend::clean()\n{\n scanCoalescQ(/*forceCoalescQDrop=*/false);\n // Backend::clean is always called under synchronization so only one thread can\n // enter to this method at once.\n // backendCleanCnt%2== 1 means that clean operation is in progress\n backendCleanCnt.fetch_add(1, std::memory_order_acq_rel);\n bool res = false;\n // We can have several blocks occupying a whole region,\n // because such regions are added in advance (see askMemFromOS() and reset()),\n // and never used. Release them all.\n for (int i = advRegBins.getMinUsedBin(0); i != -1; i = advRegBins.getMinUsedBin(i+1)) {\n if (i == freeSlabAlignedBins.getMinNonemptyBin(i))\n res |= freeSlabAlignedBins.tryReleaseRegions(i, this);\n if (i == freeLargeBlockBins.getMinNonemptyBin(i))\n res |= freeLargeBlockBins.tryReleaseRegions(i, this);\n }\n backendCleanCnt.fetch_add(1, std::memory_order_acq_rel);\n return res;\n}\n\nvoid Backend::IndexedBins::verify()\n{\n#if MALLOC_DEBUG\n for (int i=0; i<(int)freeBinsNum; i++) {\n for (FreeBlock *fb = freeBins[i].head.load(std::memory_order_relaxed); fb; fb=fb->next) {\n uintptr_t mySz = fb->myL.value;\n MALLOC_ASSERT(mySz>GuardedSize::MAX_SPEC_VAL, ASSERT_TEXT);\n FreeBlock *right = (FreeBlock*)((uintptr_t)fb + mySz);\n suppress_unused_warning(right);\n MALLOC_ASSERT(right->myL.value<=GuardedSize::MAX_SPEC_VAL, ASSERT_TEXT);\n MALLOC_ASSERT(right->leftL.value==mySz, ASSERT_TEXT);\n MALLOC_ASSERT(fb->leftL.value<=GuardedSize::MAX_SPEC_VAL, ASSERT_TEXT);\n }\n }\n#endif\n}\n\n// For correct operation, it must be called when no other threads\n// is changing backend.\nvoid Backend::verify()\n{\n#if MALLOC_DEBUG\n scanCoalescQ(/*forceCoalescQDrop=*/false);\n#endif // MALLOC_DEBUG\n\n freeLargeBlockBins.verify();\n freeSlabAlignedBins.verify();\n}\n\n#if __TBB_MALLOC_BACKEND_STAT\nsize_t Backend::Bin::countFreeBlocks()\n{\n size_t cnt = 0;\n {\n MallocMutex::scoped_lock lock(tLock);\n for (FreeBlock *fb = head; fb; fb = fb->next)\n cnt++;\n }\n return cnt;\n}\n\nsize_t Backend::Bin::reportFreeBlocks(FILE *f)\n{\n size_t totalSz = 0;\n MallocMutex::scoped_lock lock(tLock);\n for (FreeBlock *fb = head; fb; fb = fb->next) {\n size_t sz = fb->tryLockBlock();\n fb->setMeFree(sz);\n fb->rightNeig(sz)->setLeftFree(sz);\n fprintf(f, \" [%p;%p]\", fb, (void*)((uintptr_t)fb+sz));\n totalSz += sz;\n }\n return totalSz;\n}\n\nvoid Backend::IndexedBins::reportStat(FILE *f)\n{\n size_t totalSize = 0;\n\n for (int i=0; i inFlyBlocks; // to another\n std::atomic binsModifications; // incremented on every bin modification\n Backend *backend;\npublic:\n void init(Backend *b) { backend = b; }\n void blockConsumed() { inFlyBlocks++; }\n void binsModified() { binsModifications++; }\n void blockReleased() {\n#if __TBB_MALLOC_BACKEND_STAT\n MALLOC_ITT_SYNC_RELEASING(&inFlyBlocks);\n#endif\n binsModifications++;\n intptr_t prev = inFlyBlocks.fetch_sub(1);\n MALLOC_ASSERT(prev > 0, ASSERT_TEXT);\n suppress_unused_warning(prev);\n }\n intptr_t getNumOfMods() const { return binsModifications.load(std::memory_order_acquire); }\n // return true if need re-do the blocks search\n inline bool waitTillBlockReleased(intptr_t startModifiedCnt);\n};\n\nclass CoalRequestQ { // queue of free blocks that coalescing was delayed\nprivate:\n std::atomic blocksToFree;\n BackendSync *bkndSync;\n // counted blocks in blocksToFree and that are leaved blocksToFree\n // and still in active coalescing\n std::atomic inFlyBlocks;\npublic:\n void init(BackendSync *bSync) { bkndSync = bSync; }\n FreeBlock *getAll(); // return current list of blocks and make queue empty\n void putBlock(FreeBlock *fBlock);\n inline void blockWasProcessed();\n intptr_t blocksInFly() const { return inFlyBlocks.load(std::memory_order_acquire); }\n};\n\nclass MemExtendingSema {\n std::atomic active;\npublic:\n bool wait() {\n bool rescanBins = false;\n // up to 3 threads can add more memory from OS simultaneously,\n // rest of threads have to wait\n intptr_t prevCnt = active.load(std::memory_order_acquire);\n for (;;) {\n if (prevCnt < 3) {\n if (active.compare_exchange_strong(prevCnt, prevCnt + 1)) {\n break;\n }\n } else {\n SpinWaitWhileEq(active, prevCnt);\n rescanBins = true;\n break;\n }\n }\n return rescanBins;\n }\n void signal() { active.fetch_sub(1); }\n};\n\nenum MemRegionType {\n // The region holds only slabs\n MEMREG_SLAB_BLOCKS = 0,\n // The region can hold several large object blocks\n MEMREG_LARGE_BLOCKS,\n // The region holds only one block with a requested size\n MEMREG_ONE_BLOCK\n};\n\nclass MemRegionList {\n MallocMutex regionListLock;\npublic:\n MemRegion *head;\n void add(MemRegion *r);\n void remove(MemRegion *r);\n int reportStat(FILE *f);\n};\n\nclass Backend {\nprivate:\n/* Blocks in range [minBinnedSize; getMaxBinnedSize()] are kept in bins,\n one region can contains several blocks. Larger blocks are allocated directly\n and one region always contains one block.\n*/\n enum {\n minBinnedSize = 8*1024UL,\n /* If huge pages are available, maxBinned_HugePage used.\n If not, maxBinned_SmallPage is the threshold.\n TODO: use pool's granularity for upper bound setting.*/\n maxBinned_SmallPage = 1024*1024UL,\n // TODO: support other page sizes\n maxBinned_HugePage = 4*1024*1024UL\n };\n enum {\n VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result\n };\npublic:\n // Backend bins step is the same as CacheStep for large object cache\n static const size_t freeBinsStep = LargeObjectCache::LargeBSProps::CacheStep;\n static const unsigned freeBinsNum = (maxBinned_HugePage-minBinnedSize)/freeBinsStep + 1;\n\n // if previous access missed per-thread slabs pool,\n // allocate numOfSlabAllocOnMiss blocks in advance\n static const int numOfSlabAllocOnMiss = 2;\n\n enum {\n NO_BIN = -1,\n // special bin for blocks >= maxBinned_HugePage, blocks go to this bin\n // when pool is created with keepAllMemory policy\n // TODO: currently this bin is scanned using \"1st fit\", as it accumulates\n // blocks of different sizes, \"best fit\" is preferred in terms of fragmentation\n HUGE_BIN = freeBinsNum-1\n };\n\n // Bin keeps 2-linked list of free blocks. It must be 2-linked\n // because during coalescing a block it's removed from a middle of the list.\n struct Bin {\n std::atomic head;\n FreeBlock* tail;\n MallocMutex tLock;\n\n void removeBlock(FreeBlock *fBlock);\n void reset() {\n head.store(nullptr, std::memory_order_relaxed);\n tail = nullptr;\n }\n bool empty() const { return !head.load(std::memory_order_relaxed); }\n\n size_t countFreeBlocks();\n size_t reportFreeBlocks(FILE *f);\n void reportStat(FILE *f);\n };\n\n typedef BitMaskMin BitMaskBins;\n\n // array of bins supplemented with bitmask for fast finding of non-empty bins\n class IndexedBins {\n BitMaskBins bitMask;\n Bin freeBins[Backend::freeBinsNum];\n FreeBlock *getFromBin(int binIdx, BackendSync *sync, size_t size,\n bool needAlignedBlock, bool alignedBin, bool wait, int *resLocked);\n public:\n FreeBlock *findBlock(int nativeBin, BackendSync *sync, size_t size,\n bool needAlignedBlock, bool alignedBin,int *numOfLockedBins);\n bool tryReleaseRegions(int binIdx, Backend *backend);\n void lockRemoveBlock(int binIdx, FreeBlock *fBlock);\n void addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail);\n bool tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail);\n int getMinNonemptyBin(unsigned startBin) const;\n void verify();\n void reset();\n void reportStat(FILE *f);\n };\n\nprivate:\n class AdvRegionsBins {\n BitMaskBins bins;\n public:\n void registerBin(int regBin) { bins.set(regBin, 1); }\n int getMinUsedBin(int start) const { return bins.getMinTrue(start); }\n void reset() { bins.reset(); }\n };\n // auxiliary class to atomic maximum request finding\n class MaxRequestComparator {\n const Backend *backend;\n public:\n MaxRequestComparator(const Backend *be) : backend(be) {}\n inline bool operator()(size_t oldMaxReq, size_t requestSize) const;\n };\n\n#if CHECK_ALLOCATION_RANGE\n // Keep min and max of all addresses requested from OS,\n // use it for checking memory possibly allocated by replaced allocators\n // and for debugging purposes. Valid only for default memory pool.\n class UsedAddressRange {\n static const uintptr_t ADDRESS_UPPER_BOUND = UINTPTR_MAX;\n\n std::atomic leftBound,\n rightBound;\n MallocMutex mutex;\n public:\n // rightBound is zero-initialized\n void init() { leftBound.store(ADDRESS_UPPER_BOUND, std::memory_order_relaxed); }\n void registerAlloc(uintptr_t left, uintptr_t right);\n void registerFree(uintptr_t left, uintptr_t right);\n // as only left and right bounds are kept, we can return true\n // for pointer not allocated by us, if more than single region\n // was requested from OS\n bool inRange(void *ptr) const {\n const uintptr_t p = (uintptr_t)ptr;\n return leftBound.load(std::memory_order_relaxed)<=p &&\n p<=rightBound.load(std::memory_order_relaxed);\n }\n };\n#else\n class UsedAddressRange {\n public:\n void init() { }\n void registerAlloc(uintptr_t, uintptr_t) {}\n void registerFree(uintptr_t, uintptr_t) {}\n bool inRange(void *) const { return true; }\n };\n#endif\n\n ExtMemoryPool *extMemPool;\n // used for release every region on pool destroying\n MemRegionList regionList;\n\n CoalRequestQ coalescQ; // queue of coalescing requests\n BackendSync bkndSync;\n // semaphore protecting adding more more memory from OS\n MemExtendingSema memExtendingSema;\n //size_t totalMemSize,\n // memSoftLimit;\n std::atomic totalMemSize;\n std::atomic memSoftLimit;\n UsedAddressRange usedAddrRange;\n // to keep 1st allocation large than requested, keep bootstrapping status\n enum {\n bootsrapMemNotDone = 0,\n bootsrapMemInitializing,\n bootsrapMemDone\n };\n std::atomic bootsrapMemStatus;\n MallocMutex bootsrapMemStatusMutex;\n\n // Using of maximal observed requested size allows decrease\n // memory consumption for small requests and decrease fragmentation\n // for workloads when small and large allocation requests are mixed.\n // TODO: decrease, not only increase it\n std::atomic maxRequestedSize;\n\n // register bins related to advance regions\n AdvRegionsBins advRegBins;\n // Storage for split FreeBlocks\n IndexedBins freeLargeBlockBins,\n freeSlabAlignedBins;\n\n std::atomic backendCleanCnt;\n // Our friends\n friend class BackendSync;\n\n /******************************** Backend methods ******************************/\n\n /*--------------------------- Coalescing functions ----------------------------*/\n void coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned);\n bool coalescAndPutList(FreeBlock *head, bool forceCoalescQDrop, bool reportBlocksProcessed);\n\n // Main coalescing operation\n FreeBlock *doCoalesc(FreeBlock *fBlock, MemRegion **memRegion);\n\n // Queue for conflicted blocks during coalescing\n bool scanCoalescQ(bool forceCoalescQDrop);\n intptr_t blocksInCoalescing() const { return coalescQ.blocksInFly(); }\n\n /*--------------------- FreeBlock backend accessors ---------------------------*/\n FreeBlock *genericGetBlock(int num, size_t size, bool slabAligned);\n void genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned);\n\n // Split the block and return remaining parts to backend if possible\n FreeBlock *splitBlock(FreeBlock *fBlock, int num, size_t size, bool isAligned, bool needAlignedBlock);\n\n void removeBlockFromBin(FreeBlock *fBlock);\n\n // TODO: combine with returnLargeObject\n void putLargeBlock(LargeMemoryBlock *lmb);\n\n /*------------------- Starting point for OS allocation ------------------------*/\n void requestBootstrapMem();\n FreeBlock *askMemFromOS(size_t totalReqSize, intptr_t startModifiedCnt,\n int *lockedBinsThreshold, int numOfLockedBins,\n bool *splittable, bool needSlabRegion);\n\n /*---------------------- Memory regions allocation ----------------------------*/\n FreeBlock *addNewRegion(size_t size, MemRegionType type, bool addToBin);\n void releaseRegion(MemRegion *region);\n\n // TODO: combine in one initMemoryRegion function\n FreeBlock *findBlockInRegion(MemRegion *region, size_t exactBlockSize);\n void startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin);\n\n /*------------------------- Raw memory accessors ------------------------------*/\n void *allocRawMem(size_t &size);\n bool freeRawMem(void *object, size_t size);\n\n /*------------------------------ Cleanup functions ----------------------------*/\n // Clean all memory from all caches (extMemPool hard cleanup)\n FreeBlock *releaseMemInCaches(intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins);\n // Soft heap limit (regular cleanup, then maybe hard cleanup)\n void releaseCachesToLimit();\n\n /*---------------------------------- Utility ----------------------------------*/\n // TODO: move inside IndexedBins class\n static int sizeToBin(size_t size) {\n if (size >= maxBinned_HugePage)\n return HUGE_BIN;\n else if (size < minBinnedSize)\n return NO_BIN;\n\n int bin = (size - minBinnedSize)/freeBinsStep;\n\n MALLOC_ASSERT(bin < HUGE_BIN, \"Invalid size.\");\n return bin;\n }\n static bool toAlignedBin(FreeBlock *block, size_t size) {\n return isAligned((char*)block + size, slabSize) && size >= slabSize;\n }\n\npublic:\n /*--------------------- Init, reset, destroy, verify -------------------------*/\n void init(ExtMemoryPool *extMemoryPool);\n bool destroy();\n\n void verify();\n void reset();\n bool clean(); // clean on caches cleanup\n\n /*------------------------- Slab block request --------------------------------*/\n BlockI *getSlabBlock(int num);\n void putSlabBlock(BlockI *block);\n\n /*-------------------------- Large object request -----------------------------*/\n LargeMemoryBlock *getLargeBlock(size_t size);\n // TODO: make consistent with getLargeBlock\n void returnLargeObject(LargeMemoryBlock *lmb);\n\n /*-------------------------- Backreference memory request ----------------------*/\n void *getBackRefSpace(size_t size, bool *rawMemUsed);\n void putBackRefSpace(void *b, size_t size, bool rawMemUsed);\n\n /*----------------------------- Remap object ----------------------------------*/\n void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment);\n\n /*---------------------------- Validation -------------------------------------*/\n bool inUserPool() const;\n bool ptrCanBeValid(void *ptr) const { return usedAddrRange.inRange(ptr); }\n\n /*-------------------------- Configuration API --------------------------------*/\n // Soft heap limit\n void setRecommendedMaxSize(size_t softLimit) {\n memSoftLimit = softLimit;\n releaseCachesToLimit();\n }\n\n /*------------------------------- Info ----------------------------------------*/\n size_t getMaxBinnedSize() const;\n\n /*-------------------------- Testing, statistics ------------------------------*/\n#if __TBB_MALLOC_WHITEBOX_TEST\n size_t getTotalMemSize() const { return totalMemSize.load(std::memory_order_relaxed); }\n#endif\n#if __TBB_MALLOC_BACKEND_STAT\n void reportStat(FILE *f);\nprivate:\n static size_t binToSize(int bin) {\n MALLOC_ASSERT(bin <= HUGE_BIN, \"Invalid bin.\");\n\n return bin*freeBinsStep + minBinnedSize;\n }\n#endif\n};\n\n#endif // __TBB_backend_H\n" }, { "path": "src/tbb/src/tbbmalloc/backref.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"tbbmalloc_internal.h\"\n#include /* for placement new */\n\nnamespace rml {\nnamespace internal {\n\n\n/********* backreferences ***********************/\n/* Each slab block and each large memory object header contains BackRefIdx\n * that points out in some BackRefBlock which points back to this block or header.\n */\nstruct BackRefBlock : public BlockI {\n BackRefBlock *nextForUse; // the next in the chain of blocks with free items\n FreeObject *bumpPtr; // bump pointer moves from the end to the beginning of the block\n FreeObject *freeList;\n // list of all blocks that were allocated from raw mem (i.e., not from backend)\n BackRefBlock *nextRawMemBlock;\n std::atomic allocatedCount; // the number of objects allocated\n BackRefIdx::main_t myNum; // the index in the main\n MallocMutex blockMutex;\n // true if this block has been added to the listForUse chain,\n // modifications protected by mainMutex\n std::atomic addedToForUse;\n\n BackRefBlock(const BackRefBlock *blockToUse, intptr_t num) :\n nextForUse(nullptr), bumpPtr((FreeObject*)((uintptr_t)blockToUse + slabSize - sizeof(void*))),\n freeList(nullptr), nextRawMemBlock(nullptr), allocatedCount(0), myNum(num),\n addedToForUse(false) {\n memset(static_cast(&blockMutex), 0, sizeof(MallocMutex));\n\n MALLOC_ASSERT(!(num >> CHAR_BIT*sizeof(BackRefIdx::main_t)),\n \"index in BackRefMain must fit to BackRefIdx::main\");\n }\n // clean all but header\n void zeroSet() { memset(static_cast(this+1), 0, BackRefBlock::bytes-sizeof(BackRefBlock)); }\n static const int bytes = slabSize;\n};\n\n// max number of backreference pointers in slab block\nstatic const int BR_MAX_CNT = (BackRefBlock::bytes-sizeof(BackRefBlock))/sizeof(void*);\n\nstruct BackRefMain {\n/* On 64-bit systems a slab block can hold up to ~2K back pointers to slab blocks\n * or large objects, so it can address at least 32MB. The main array of 256KB\n * holds 32K pointers to such blocks, addressing ~1 TB.\n * On 32-bit systems there is ~4K back pointers in a slab block, so ~64MB can be addressed.\n * The main array of 8KB holds 2K pointers to leaves, so ~128 GB can addressed.\n */\n static const size_t bytes = sizeof(uintptr_t)>4? 256*1024 : 8*1024;\n static const int dataSz;\n/* space is reserved for main table and 4 leaves\n taking into account VirtualAlloc allocation granularity */\n static const int leaves = 4;\n static const size_t mainSize = BackRefMain::bytes+leaves*BackRefBlock::bytes;\n // The size of memory request for a few more leaf blocks;\n // selected to match VirtualAlloc granularity\n static const size_t blockSpaceSize = 64*1024;\n\n Backend *backend;\n std::atomic active; // if defined, use it for allocations\n std::atomic listForUse; // the chain of data blocks with free items\n BackRefBlock *allRawMemBlocks;\n std::atomic lastUsed; // index of the last used block\n bool rawMemUsed;\n MallocMutex requestNewSpaceMutex;\n BackRefBlock *backRefBl[1]; // the real size of the array is dataSz\n\n BackRefBlock *findFreeBlock();\n void addToForUseList(BackRefBlock *bl);\n void initEmptyBackRefBlock(BackRefBlock *newBl);\n bool requestNewSpace();\n};\n\nconst int BackRefMain::dataSz\n = 1+(BackRefMain::bytes-sizeof(BackRefMain))/sizeof(BackRefBlock*);\n\nstatic MallocMutex mainMutex;\nstatic std::atomic backRefMain;\n\nbool initBackRefMain(Backend *backend)\n{\n bool rawMemUsed;\n BackRefMain *main =\n (BackRefMain*)backend->getBackRefSpace(BackRefMain::mainSize,\n &rawMemUsed);\n if (! main)\n return false;\n main->backend = backend;\n main->listForUse.store(nullptr, std::memory_order_relaxed);\n main->allRawMemBlocks = nullptr;\n main->rawMemUsed = rawMemUsed;\n main->lastUsed = -1;\n memset(static_cast(&main->requestNewSpaceMutex), 0, sizeof(MallocMutex));\n for (int i=0; izeroSet();\n main->initEmptyBackRefBlock(bl);\n if (i)\n main->addToForUseList(bl);\n else // active leaf is not needed in listForUse\n main->active.store(bl, std::memory_order_relaxed);\n }\n // backRefMain is read in getBackRef, so publish it in consistent state\n backRefMain.store(main, std::memory_order_release);\n return true;\n}\n\n#if __TBB_SOURCE_DIRECTLY_INCLUDED\nvoid destroyBackRefMain(Backend *backend)\n{\n if (backRefMain.load(std::memory_order_acquire)) { // Is initBackRefMain() called?\n for (BackRefBlock *curr = backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks; curr; ) {\n BackRefBlock *next = curr->nextRawMemBlock;\n // allRawMemBlocks list is only for raw mem blocks\n backend->putBackRefSpace(curr, BackRefMain::blockSpaceSize,\n /*rawMemUsed=*/true);\n curr = next;\n }\n backend->putBackRefSpace(backRefMain.load(std::memory_order_relaxed), BackRefMain::mainSize,\n backRefMain.load(std::memory_order_relaxed)->rawMemUsed);\n }\n}\n#endif\n\nvoid BackRefMain::addToForUseList(BackRefBlock *bl)\n{\n bl->nextForUse = listForUse.load(std::memory_order_relaxed);\n listForUse.store(bl, std::memory_order_relaxed);\n bl->addedToForUse.store(true, std::memory_order_relaxed);\n}\n\nvoid BackRefMain::initEmptyBackRefBlock(BackRefBlock *newBl)\n{\n intptr_t nextLU = lastUsed+1;\n new (newBl) BackRefBlock(newBl, nextLU);\n MALLOC_ASSERT(nextLU < dataSz, nullptr);\n backRefBl[nextLU] = newBl;\n // lastUsed is read in getBackRef, and access to backRefBl[lastUsed]\n // is possible only after checking backref against current lastUsed\n lastUsed.store(nextLU, std::memory_order_release);\n}\n\nbool BackRefMain::requestNewSpace()\n{\n bool isRawMemUsed;\n static_assert(!(blockSpaceSize % BackRefBlock::bytes),\n \"Must request space for whole number of blocks.\");\n\n if (BackRefMain::dataSz <= lastUsed + 1) // no space in main\n return false;\n\n // only one thread at a time may add blocks\n MallocMutex::scoped_lock newSpaceLock(requestNewSpaceMutex);\n\n if (listForUse.load(std::memory_order_relaxed)) // double check that only one block is available\n return true;\n BackRefBlock *newBl = (BackRefBlock*)backend->getBackRefSpace(blockSpaceSize, &isRawMemUsed);\n if (!newBl) return false;\n\n // touch a page for the 1st time without taking mainMutex ...\n for (BackRefBlock *bl = newBl; (uintptr_t)bl < (uintptr_t)newBl + blockSpaceSize;\n bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes)) {\n bl->zeroSet();\n }\n\n MallocMutex::scoped_lock lock(mainMutex); // ... and share under lock\n\n const size_t numOfUnusedIdxs = BackRefMain::dataSz - lastUsed - 1;\n if (numOfUnusedIdxs <= 0) { // no space in main under lock, roll back\n backend->putBackRefSpace(newBl, blockSpaceSize, isRawMemUsed);\n return false;\n }\n // It's possible that only part of newBl is used, due to lack of indices in main.\n // This is OK as such underutilization is possible only once for backreferneces table.\n int blocksToUse = min(numOfUnusedIdxs, blockSpaceSize / BackRefBlock::bytes);\n\n // use the first block in the batch to maintain the list of \"raw\" memory\n // to be released at shutdown\n if (isRawMemUsed) {\n newBl->nextRawMemBlock = backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks;\n backRefMain.load(std::memory_order_relaxed)->allRawMemBlocks = newBl;\n }\n for (BackRefBlock *bl = newBl; blocksToUse>0; bl = (BackRefBlock*)((uintptr_t)bl + BackRefBlock::bytes), blocksToUse--) {\n initEmptyBackRefBlock(bl);\n if (active.load(std::memory_order_relaxed)->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT) {\n active.store(bl, std::memory_order_release); // active leaf is not needed in listForUse\n } else {\n addToForUseList(bl);\n }\n }\n return true;\n}\n\nBackRefBlock *BackRefMain::findFreeBlock()\n{\n BackRefBlock* active_block = active.load(std::memory_order_acquire);\n MALLOC_ASSERT(active_block, ASSERT_TEXT);\n\n if (active_block->allocatedCount.load(std::memory_order_relaxed) < BR_MAX_CNT)\n return active_block;\n\n if (listForUse.load(std::memory_order_relaxed)) { // use released list\n MallocMutex::scoped_lock lock(mainMutex);\n\n if (active_block->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT) {\n active_block = listForUse.load(std::memory_order_relaxed);\n if (active_block) {\n active.store(active_block, std::memory_order_release);\n listForUse.store(active_block->nextForUse, std::memory_order_relaxed);\n MALLOC_ASSERT(active_block->addedToForUse.load(std::memory_order_relaxed), ASSERT_TEXT);\n active_block->addedToForUse.store(false, std::memory_order_relaxed);\n }\n }\n } else // allocate new data node\n if (!requestNewSpace())\n return nullptr;\n return active.load(std::memory_order_acquire); // reread because of requestNewSpace\n}\n\nvoid *getBackRef(BackRefIdx backRefIdx)\n{\n // !backRefMain means no initialization done, so it can't be valid memory\n // see addEmptyBackRefBlock for fences around lastUsed\n if (!(backRefMain.load(std::memory_order_acquire))\n || backRefIdx.getMain() > (backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_acquire))\n || backRefIdx.getOffset() >= BR_MAX_CNT)\n {\n return nullptr;\n }\n std::atomic& backRefEntry = *(std::atomic*)(\n (uintptr_t)backRefMain.load(std::memory_order_relaxed)->backRefBl[backRefIdx.getMain()]\n + sizeof(BackRefBlock) + backRefIdx.getOffset() * sizeof(std::atomic)\n );\n return backRefEntry.load(std::memory_order_relaxed);\n}\n\nvoid setBackRef(BackRefIdx backRefIdx, void *newPtr)\n{\n MALLOC_ASSERT(backRefIdx.getMain()<=backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_relaxed)\n && backRefIdx.getOffset()*)((uintptr_t)backRefMain.load(std::memory_order_relaxed)->backRefBl[backRefIdx.getMain()]\n + sizeof(BackRefBlock) + backRefIdx.getOffset() * sizeof(void*)))->store(newPtr, std::memory_order_relaxed);\n}\n\nBackRefIdx BackRefIdx::newBackRef(bool largeObj)\n{\n BackRefBlock *blockToUse;\n void **toUse;\n BackRefIdx res;\n bool lastBlockFirstUsed = false;\n\n do {\n MALLOC_ASSERT(backRefMain.load(std::memory_order_relaxed), ASSERT_TEXT);\n blockToUse = backRefMain.load(std::memory_order_relaxed)->findFreeBlock();\n if (!blockToUse)\n return BackRefIdx();\n toUse = nullptr;\n { // the block is locked to find a reference\n MallocMutex::scoped_lock lock(blockToUse->blockMutex);\n\n if (blockToUse->freeList) {\n toUse = (void**)blockToUse->freeList;\n blockToUse->freeList = blockToUse->freeList->next;\n MALLOC_ASSERT(!blockToUse->freeList ||\n ((uintptr_t)blockToUse->freeList>=(uintptr_t)blockToUse\n && (uintptr_t)blockToUse->freeList <\n (uintptr_t)blockToUse + slabSize), ASSERT_TEXT);\n } else if (blockToUse->allocatedCount.load(std::memory_order_relaxed) < BR_MAX_CNT) {\n toUse = (void**)blockToUse->bumpPtr;\n blockToUse->bumpPtr =\n (FreeObject*)((uintptr_t)blockToUse->bumpPtr - sizeof(void*));\n if (blockToUse->allocatedCount.load(std::memory_order_relaxed) == BR_MAX_CNT-1) {\n MALLOC_ASSERT((uintptr_t)blockToUse->bumpPtr\n < (uintptr_t)blockToUse+sizeof(BackRefBlock),\n ASSERT_TEXT);\n blockToUse->bumpPtr = nullptr;\n }\n }\n if (toUse) {\n if (!blockToUse->allocatedCount.load(std::memory_order_relaxed) &&\n !backRefMain.load(std::memory_order_relaxed)->listForUse.load(std::memory_order_relaxed)) {\n lastBlockFirstUsed = true;\n }\n blockToUse->allocatedCount.store(blockToUse->allocatedCount.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed);\n }\n } // end of lock scope\n } while (!toUse);\n // The first thread that uses the last block requests new space in advance;\n // possible failures are ignored.\n if (lastBlockFirstUsed)\n backRefMain.load(std::memory_order_relaxed)->requestNewSpace();\n\n res.main = blockToUse->myNum;\n uintptr_t offset =\n ((uintptr_t)toUse - ((uintptr_t)blockToUse + sizeof(BackRefBlock)))/sizeof(void*);\n // Is offset too big?\n MALLOC_ASSERT(!(offset >> 15), ASSERT_TEXT);\n res.offset = offset;\n if (largeObj) res.largeObj = largeObj;\n\n return res;\n}\n\nvoid removeBackRef(BackRefIdx backRefIdx)\n{\n MALLOC_ASSERT(!backRefIdx.isInvalid(), ASSERT_TEXT);\n MALLOC_ASSERT(backRefIdx.getMain()<=backRefMain.load(std::memory_order_relaxed)->lastUsed.load(std::memory_order_relaxed)\n && backRefIdx.getOffset()backRefBl[backRefIdx.getMain()];\n std::atomic& backRefEntry = *(std::atomic*)((uintptr_t)currBlock + sizeof(BackRefBlock)\n + backRefIdx.getOffset()*sizeof(std::atomic));\n MALLOC_ASSERT(((uintptr_t)&backRefEntry >(uintptr_t)currBlock &&\n (uintptr_t)&backRefEntry <(uintptr_t)currBlock + slabSize), ASSERT_TEXT);\n {\n MallocMutex::scoped_lock lock(currBlock->blockMutex);\n\n backRefEntry.store(currBlock->freeList, std::memory_order_relaxed);\n#if MALLOC_DEBUG\n uintptr_t backRefEntryValue = (uintptr_t)backRefEntry.load(std::memory_order_relaxed);\n MALLOC_ASSERT(!backRefEntryValue ||\n (backRefEntryValue > (uintptr_t)currBlock\n && backRefEntryValue < (uintptr_t)currBlock + slabSize), ASSERT_TEXT);\n#endif\n currBlock->freeList = (FreeObject*)&backRefEntry;\n currBlock->allocatedCount.store(currBlock->allocatedCount.load(std::memory_order_relaxed)-1, std::memory_order_relaxed);\n }\n // TODO: do we need double-check here?\n if (!currBlock->addedToForUse.load(std::memory_order_relaxed) &&\n currBlock!=backRefMain.load(std::memory_order_relaxed)->active.load(std::memory_order_relaxed)) {\n MallocMutex::scoped_lock lock(mainMutex);\n\n if (!currBlock->addedToForUse.load(std::memory_order_relaxed) &&\n currBlock!=backRefMain.load(std::memory_order_relaxed)->active.load(std::memory_order_relaxed))\n backRefMain.load(std::memory_order_relaxed)->addToForUseList(currBlock);\n }\n}\n\n/********* End of backreferences ***********************/\n\n} // namespace internal\n} // namespace rml\n\n" }, { "path": "src/tbb/src/tbbmalloc/def/lin32-tbbmalloc.def", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n\nscalable_calloc;\nscalable_free;\nscalable_malloc;\nscalable_realloc;\nscalable_posix_memalign;\nscalable_aligned_malloc;\nscalable_aligned_realloc;\nscalable_aligned_free;\nscalable_msize;\nscalable_allocation_mode;\nscalable_allocation_command;\n__TBB_malloc_safer_aligned_msize;\n__TBB_malloc_safer_aligned_realloc;\n__TBB_malloc_safer_free;\n__TBB_malloc_safer_msize;\n__TBB_malloc_safer_realloc;\n\n/* memory pool stuff */\n_ZN3rml10pool_resetEPNS_10MemoryPoolE;\n_ZN3rml11pool_createEiPKNS_13MemPoolPolicyE;\n_ZN3rml14pool_create_v1EiPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE;\n_ZN3rml11pool_mallocEPNS_10MemoryPoolEj;\n_ZN3rml12pool_destroyEPNS_10MemoryPoolE;\n_ZN3rml9pool_freeEPNS_10MemoryPoolEPv;\n_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvj;\n_ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvjj;\n_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEjj;\n_ZN3rml13pool_identifyEPv;\n_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv;\n\nlocal:\n\n/* TBB symbols */\n*3rml*;\n*3tbb*;\n*__TBB*;\n__itt_*;\nITT_DoOneTimeInitialization;\nTBB_runtime_interface_version;\n\n/* Intel Compiler (libirc) symbols */\n__intel_*;\n_intel_*;\nget_memcpy_largest_cachelinesize;\nget_memcpy_largest_cache_size;\nget_mem_ops_method;\ninit_mem_ops_method;\nirc__get_msg;\nirc__print;\noverride_mem_ops_method;\nset_memcpy_largest_cachelinesize;\nset_memcpy_largest_cache_size;\n\n};\n" }, { "path": "src/tbb/src/tbbmalloc/def/lin64-tbbmalloc.def", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\n\nscalable_calloc;\nscalable_free;\nscalable_malloc;\nscalable_realloc;\nscalable_posix_memalign;\nscalable_aligned_malloc;\nscalable_aligned_realloc;\nscalable_aligned_free;\nscalable_msize;\nscalable_allocation_mode;\nscalable_allocation_command;\n__TBB_malloc_safer_aligned_msize;\n__TBB_malloc_safer_aligned_realloc;\n__TBB_malloc_safer_free;\n__TBB_malloc_safer_msize;\n__TBB_malloc_safer_realloc;\n\n/* memory pool stuff */\n_ZN3rml11pool_createElPKNS_13MemPoolPolicyE;\n_ZN3rml14pool_create_v1ElPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE;\n_ZN3rml10pool_resetEPNS_10MemoryPoolE;\n_ZN3rml11pool_mallocEPNS_10MemoryPoolEm;\n_ZN3rml12pool_destroyEPNS_10MemoryPoolE;\n_ZN3rml9pool_freeEPNS_10MemoryPoolEPv;\n_ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm;\n_ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm;\n_ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm;\n_ZN3rml13pool_identifyEPv;\n_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv;\n\nlocal:\n\n/* TBB symbols */\n*3rml*;\n*3tbb*;\n*__TBB*;\n__itt_*;\nITT_DoOneTimeInitialization;\nTBB_runtime_interface_version;\n\n/* Intel Compiler (libirc) symbols */\n__intel_*;\n_intel_*;\nget_memcpy_largest_cachelinesize;\nget_memcpy_largest_cache_size;\nget_mem_ops_method;\ninit_mem_ops_method;\nirc__get_msg;\nirc__print;\noverride_mem_ops_method;\nset_memcpy_largest_cachelinesize;\nset_memcpy_largest_cache_size;\n\n};\n" }, { "path": "src/tbb/src/tbbmalloc/def/mac64-tbbmalloc.def", "content": "# Copyright (c) 2005-2021 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n_scalable_calloc\n_scalable_free\n_scalable_malloc\n_scalable_realloc\n_scalable_posix_memalign\n_scalable_aligned_malloc\n_scalable_aligned_realloc\n_scalable_aligned_free\n_scalable_msize\n_scalable_allocation_mode\n_scalable_allocation_command\n___TBB_malloc_safer_aligned_msize\n___TBB_malloc_safer_aligned_realloc\n___TBB_malloc_safer_free\n___TBB_malloc_safer_msize\n___TBB_malloc_safer_realloc\n___TBB_malloc_free_definite_size\n/* memory pool stuff */\n__ZN3rml11pool_createElPKNS_13MemPoolPolicyE\n__ZN3rml14pool_create_v1ElPKNS_13MemPoolPolicyEPPNS_10MemoryPoolE\n__ZN3rml10pool_resetEPNS_10MemoryPoolE\n__ZN3rml12pool_destroyEPNS_10MemoryPoolE\n__ZN3rml11pool_mallocEPNS_10MemoryPoolEm\n__ZN3rml9pool_freeEPNS_10MemoryPoolEPv\n__ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm\n__ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm\n__ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm\n__ZN3rml13pool_identifyEPv\n__ZN3rml10pool_msizeEPNS_10MemoryPoolEPv\n\n" }, { "path": "src/tbb/src/tbbmalloc/def/win32-tbbmalloc.def", "content": "; Copyright (c) 2005-2021 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\nEXPORTS\n\n; frontend.cpp\nscalable_calloc\nscalable_free\nscalable_malloc\nscalable_realloc\nscalable_posix_memalign\nscalable_aligned_malloc\nscalable_aligned_realloc\nscalable_aligned_free\nscalable_msize\nscalable_allocation_mode\nscalable_allocation_command\n__TBB_malloc_safer_free\n__TBB_malloc_safer_realloc\n__TBB_malloc_safer_msize\n__TBB_malloc_safer_aligned_msize\n__TBB_malloc_safer_aligned_realloc\n\n; memory pool stuff\n?pool_create@rml@@YAPAVMemoryPool@1@HPBUMemPoolPolicy@1@@Z\n?pool_create_v1@rml@@YA?AW4MemPoolError@1@HPBUMemPoolPolicy@1@PAPAVMemoryPool@1@@Z\n?pool_destroy@rml@@YA_NPAVMemoryPool@1@@Z\n?pool_malloc@rml@@YAPAXPAVMemoryPool@1@I@Z\n?pool_free@rml@@YA_NPAVMemoryPool@1@PAX@Z\n?pool_reset@rml@@YA_NPAVMemoryPool@1@@Z\n?pool_realloc@rml@@YAPAXPAVMemoryPool@1@PAXI@Z\n?pool_aligned_realloc@rml@@YAPAXPAVMemoryPool@1@PAXII@Z\n?pool_aligned_malloc@rml@@YAPAXPAVMemoryPool@1@II@Z\n?pool_identify@rml@@YAPAVMemoryPool@1@PAX@Z\n?pool_msize@rml@@YAIPAVMemoryPool@1@PAX@Z\n\n" }, { "path": "src/tbb/src/tbbmalloc/def/win64-tbbmalloc.def", "content": "; Copyright (c) 2005-2021 Intel Corporation\n;\n; Licensed under the Apache License, Version 2.0 (the \"License\");\n; you may not use this file except in compliance with the License.\n; You may obtain a copy of the License at\n;\n; http://www.apache.org/licenses/LICENSE-2.0\n;\n; Unless required by applicable law or agreed to in writing, software\n; distributed under the License is distributed on an \"AS IS\" BASIS,\n; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n; See the License for the specific language governing permissions and\n; limitations under the License.\n\nEXPORTS\n\n; frontend.cpp\nscalable_calloc\nscalable_free\nscalable_malloc\nscalable_realloc\nscalable_posix_memalign\nscalable_aligned_malloc\nscalable_aligned_realloc\nscalable_aligned_free\nscalable_msize\nscalable_allocation_mode\nscalable_allocation_command\n__TBB_malloc_safer_free\n__TBB_malloc_safer_realloc\n__TBB_malloc_safer_msize\n__TBB_malloc_safer_aligned_msize\n__TBB_malloc_safer_aligned_realloc\n\n; memory pool stuff\n?pool_create@rml@@YAPEAVMemoryPool@1@_JPEBUMemPoolPolicy@1@@Z\n?pool_create_v1@rml@@YA?AW4MemPoolError@1@_JPEBUMemPoolPolicy@1@PEAPEAVMemoryPool@1@@Z\n?pool_destroy@rml@@YA_NPEAVMemoryPool@1@@Z\n?pool_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K@Z\n?pool_free@rml@@YA_NPEAVMemoryPool@1@PEAX@Z\n?pool_reset@rml@@YA_NPEAVMemoryPool@1@@Z\n?pool_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K@Z\n?pool_aligned_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K2@Z\n?pool_aligned_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K1@Z\n?pool_identify@rml@@YAPEAVMemoryPool@1@PEAX@Z\n?pool_msize@rml@@YA_KPEAVMemoryPool@1@PEAX@Z\n\n" }, { "path": "src/tbb/src/tbbmalloc/frontend.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"tbbmalloc_internal.h\"\n#include \n#include /* for placement new */\n#include /* for memset */\n\n#include \"oneapi/tbb/version.h\"\n#include \"../tbb/environment.h\"\n#include \"../tbb/itt_notify.h\" // for __TBB_load_ittnotify()\n\n#if USE_PTHREAD\n #define TlsSetValue_func pthread_setspecific\n #define TlsGetValue_func pthread_getspecific\n #define GetMyTID() pthread_self()\n #include \n extern \"C\" { static void mallocThreadShutdownNotification(void*); }\n #if __sun || __SUNPRO_CC\n #define __asm__ asm\n #endif\n #include // sysconf(_SC_PAGESIZE)\n#elif USE_WINTHREAD\n #define GetMyTID() GetCurrentThreadId()\n#if __TBB_WIN8UI_SUPPORT\n #include\n #define TlsSetValue_func FlsSetValue\n #define TlsGetValue_func FlsGetValue\n #define TlsAlloc() FlsAlloc(nullptr)\n #define TLS_ALLOC_FAILURE FLS_OUT_OF_INDEXES\n #define TlsFree FlsFree\n#else\n #define TlsSetValue_func TlsSetValue\n #define TlsGetValue_func TlsGetValue\n #define TLS_ALLOC_FAILURE TLS_OUT_OF_INDEXES\n#endif\n#else\n #error Must define USE_PTHREAD or USE_WINTHREAD\n#endif\n\n#define FREELIST_NONBLOCKING 1\n\nnamespace rml {\nclass MemoryPool;\nnamespace internal {\n\nclass Block;\nclass MemoryPool;\n\n#if MALLOC_CHECK_RECURSION\n\ninline bool isMallocInitialized();\n\n#endif // MALLOC_CHECK_RECURSION\n\n/** Support for handling the special UNUSABLE pointer state **/\nconst intptr_t UNUSABLE = 0x1;\ninline bool isSolidPtr( void* ptr ) {\n return (UNUSABLE|(intptr_t)ptr)!=UNUSABLE;\n}\ninline bool isNotForUse( void* ptr ) {\n return (intptr_t)ptr==UNUSABLE;\n}\n\n/*\n * Block::objectSize value used to mark blocks allocated by startupAlloc\n */\nconst uint16_t startupAllocObjSizeMark = ~(uint16_t)0;\n\n/*\n * The following constant is used to define the size of struct Block, the block header.\n * The intent is to have the size of a Block multiple of the cache line size, this allows us to\n * get good alignment at the cost of some overhead equal to the amount of padding included in the Block.\n */\nconst int blockHeaderAlignment = estimatedCacheLineSize;\n\n/********* The data structures and global objects **************/\n\n/*\n * The malloc routines themselves need to be able to occasionally malloc some space,\n * in order to set up the structures used by the thread local structures. This\n * routine performs that functions.\n */\nclass BootStrapBlocks {\n MallocMutex bootStrapLock;\n Block *bootStrapBlock;\n Block *bootStrapBlockUsed;\n FreeObject *bootStrapObjectList;\npublic:\n void *allocate(MemoryPool *memPool, size_t size);\n void free(void* ptr);\n void reset();\n};\n\n#if USE_INTERNAL_TID\nclass ThreadId {\n static tls_key_t Tid_key;\n std::atomic ThreadCount;\n\n unsigned int id;\n\n static unsigned int tlsNumber() {\n unsigned int result = reinterpret_cast(TlsGetValue_func(Tid_key));\n if( !result ) {\n RecursiveMallocCallProtector scoped;\n // Thread-local value is zero -> first call from this thread,\n // need to initialize with next ID value (IDs start from 1)\n result = ++ThreadCount; // returned new value!\n TlsSetValue_func( Tid_key, reinterpret_cast(result) );\n }\n return result;\n }\npublic:\n static bool init() {\n#if USE_WINTHREAD\n Tid_key = TlsAlloc();\n if (Tid_key == TLS_ALLOC_FAILURE)\n return false;\n#else\n int status = pthread_key_create( &Tid_key, nullptr );\n if ( status ) {\n fprintf (stderr, \"The memory manager cannot create tls key during initialization\\n\");\n return false;\n }\n#endif /* USE_WINTHREAD */\n return true;\n }\n#if __TBB_SOURCE_DIRECTLY_INCLUDED\n static void destroy() {\n if( Tid_key ) {\n#if USE_WINTHREAD\n BOOL status = !(TlsFree( Tid_key )); // fail is zero\n#else\n int status = pthread_key_delete( Tid_key );\n#endif /* USE_WINTHREAD */\n if ( status )\n fprintf (stderr, \"The memory manager cannot delete tls key\\n\");\n Tid_key = 0;\n }\n }\n#endif\n\n ThreadId() : id(ThreadId::tlsNumber()) {}\n bool isCurrentThreadId() const { return id == ThreadId::tlsNumber(); }\n\n#if COLLECT_STATISTICS || MALLOC_TRACE\n friend unsigned int getThreadId() { return ThreadId::tlsNumber(); }\n#endif\n#if COLLECT_STATISTICS\n static unsigned getMaxThreadId() { return ThreadCount.load(std::memory_order_relaxed); }\n\n friend int STAT_increment(ThreadId tid, int bin, int ctr);\n#endif\n};\n\ntls_key_t ThreadId::Tid_key;\nintptr_t ThreadId::ThreadCount;\n\n#if COLLECT_STATISTICS\nint STAT_increment(ThreadId tid, int bin, int ctr)\n{\n return ::STAT_increment(tid.id, bin, ctr);\n}\n#endif\n\n#else // USE_INTERNAL_TID\n\nclass ThreadId {\n#if USE_PTHREAD\n std::atomic tid;\n#else\n std::atomic tid;\n#endif\npublic:\n ThreadId() : tid(GetMyTID()) {}\n ThreadId(ThreadId &other) = delete;\n ~ThreadId() = default;\n\n#if USE_PTHREAD\n bool isCurrentThreadId() const { return pthread_equal(pthread_self(), tid.load(std::memory_order_relaxed)); }\n#else\n bool isCurrentThreadId() const { return GetCurrentThreadId() == tid.load(std::memory_order_relaxed); }\n#endif\n ThreadId& operator=(const ThreadId& other) {\n tid.store(other.tid.load(std::memory_order_relaxed), std::memory_order_relaxed);\n return *this;\n }\n static bool init() { return true; }\n#if __TBB_SOURCE_DIRECTLY_INCLUDED\n static void destroy() {}\n#endif\n};\n\n#endif // USE_INTERNAL_TID\n\n/*********** Code to provide thread ID and a thread-local void pointer **********/\n\nbool TLSKey::init()\n{\n#if USE_WINTHREAD\n TLS_pointer_key = TlsAlloc();\n if (TLS_pointer_key == TLS_ALLOC_FAILURE)\n return false;\n#else\n int status = pthread_key_create( &TLS_pointer_key, mallocThreadShutdownNotification );\n if ( status )\n return false;\n#endif /* USE_WINTHREAD */\n return true;\n}\n\nbool TLSKey::destroy()\n{\n#if USE_WINTHREAD\n BOOL status1 = !(TlsFree(TLS_pointer_key)); // fail is zero\n#else\n int status1 = pthread_key_delete(TLS_pointer_key);\n#endif /* USE_WINTHREAD */\n MALLOC_ASSERT(!status1, \"The memory manager cannot delete tls key.\");\n return status1==0;\n}\n\ninline TLSData* TLSKey::getThreadMallocTLS() const\n{\n return (TLSData *)TlsGetValue_func( TLS_pointer_key );\n}\n\ninline void TLSKey::setThreadMallocTLS( TLSData * newvalue ) {\n RecursiveMallocCallProtector scoped;\n TlsSetValue_func( TLS_pointer_key, newvalue );\n}\n\n/* The 'next' field in the block header has to maintain some invariants:\n * it needs to be on a 16K boundary and the first field in the block.\n * Any value stored there needs to have the lower 14 bits set to 0\n * so that various assert work. This means that if you want to smash this memory\n * for debugging purposes you will need to obey this invariant.\n * The total size of the header needs to be a power of 2 to simplify\n * the alignment requirements. For now it is a 128 byte structure.\n * To avoid false sharing, the fields changed only locally are separated\n * from the fields changed by foreign threads.\n * Changing the size of the block header would require to change\n * some bin allocation sizes, in particular \"fitting\" sizes (see above).\n */\nclass Bin;\nclass StartupBlock;\n\nclass MemoryPool {\n // if no explicit grainsize, expect to see malloc in user's pAlloc\n // and set reasonable low granularity\n static const size_t defaultGranularity = estimatedCacheLineSize;\n\n MemoryPool() = delete; // deny\npublic:\n static MallocMutex memPoolListLock;\n\n // list of all active pools is used to release\n // all TLS data on thread termination or library unload\n MemoryPool *next,\n *prev;\n ExtMemoryPool extMemPool;\n BootStrapBlocks bootStrapBlocks;\n\n static void initDefaultPool();\n\n bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy);\n bool reset();\n bool destroy();\n void onThreadShutdown(TLSData *tlsData);\n\n inline TLSData *getTLS(bool create);\n void clearTLS() { extMemPool.tlsPointerKey.setThreadMallocTLS(nullptr); }\n\n Block *getEmptyBlock(size_t size);\n void returnEmptyBlock(Block *block, bool poolTheBlock);\n\n // get/put large object to/from local large object cache\n void *getFromLLOCache(TLSData *tls, size_t size, size_t alignment);\n void putToLLOCache(TLSData *tls, void *object);\n};\n\nstatic intptr_t defaultMemPool_space[sizeof(MemoryPool)/sizeof(intptr_t) +\n (sizeof(MemoryPool)%sizeof(intptr_t)? 1 : 0)];\nstatic MemoryPool *defaultMemPool = (MemoryPool*)defaultMemPool_space;\nconst size_t MemoryPool::defaultGranularity;\n// zero-initialized\nMallocMutex MemoryPool::memPoolListLock;\n// TODO: move huge page status to default pool, because that's its states\nHugePagesStatus hugePages;\nstatic bool usedBySrcIncluded = false;\n\n// Padding helpers\ntemplate\nstruct PaddingImpl {\n size_t __padding[padd];\n};\n\ntemplate<>\nstruct PaddingImpl<0> {};\n\ntemplate\nstruct Padding : PaddingImpl {};\n\n// Slab block is 16KB-aligned. To prevent false sharing, separate locally-accessed\n// fields and fields commonly accessed by not owner threads.\nclass GlobalBlockFields : public BlockI {\nprotected:\n std::atomic publicFreeList;\n std::atomic nextPrivatizable;\n MemoryPool *poolPtr;\n};\n\nclass LocalBlockFields : public GlobalBlockFields, Padding {\nprotected:\n Block *next;\n Block *previous; /* Use double linked list to speed up removal */\n FreeObject *bumpPtr; /* Bump pointer moves from the end to the beginning of a block */\n FreeObject *freeList;\n /* Pointer to local data for the owner thread. Used for fast finding tls\n when releasing object from a block that current thread owned.\n nullptr for orphaned blocks. */\n std::atomic tlsPtr;\n ThreadId ownerTid; /* the ID of the thread that owns or last owned the block */\n BackRefIdx backRefIdx;\n uint16_t allocatedCount; /* Number of objects allocated (obviously by the owning thread) */\n uint16_t objectSize;\n bool isFull;\n\n friend class FreeBlockPool;\n friend class StartupBlock;\n friend class LifoList;\n friend void *BootStrapBlocks::allocate(MemoryPool *, size_t);\n friend bool OrphanedBlocks::cleanup(Backend*);\n friend Block *MemoryPool::getEmptyBlock(size_t);\n};\n\n// Use inheritance to guarantee that a user data start on next cache line.\n// Can't use member for it, because when LocalBlockFields already on cache line,\n// we must have no additional memory consumption for all compilers.\nclass Block : public LocalBlockFields,\n Padding<2*blockHeaderAlignment - sizeof(LocalBlockFields)> {\npublic:\n bool empty() const {\n if (allocatedCount > 0) return false;\n MALLOC_ASSERT(!isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT);\n return true;\n }\n inline FreeObject* allocate();\n inline FreeObject *allocateFromFreeList();\n\n inline bool adjustFullness();\n void adjustPositionInBin(Bin* bin = nullptr);\n#if MALLOC_DEBUG\n bool freeListNonNull() { return freeList; }\n#endif\n void freePublicObject(FreeObject *objectToFree);\n inline void freeOwnObject(void *object);\n void reset();\n void privatizePublicFreeList( bool reset = true );\n void restoreBumpPtr();\n void privatizeOrphaned(TLSData *tls, unsigned index);\n bool readyToShare();\n void shareOrphaned(intptr_t binTag, unsigned index);\n unsigned int getSize() const {\n MALLOC_ASSERT(isStartupAllocObject() || objectSize0, msg);\n#endif\n // must not point to slab's header\n MALLOC_ASSERT((uintptr_t)object - (uintptr_t)this >= sizeof(Block), msg);\n if (startupAllocObjSizeMark == objectSize) // startup block\n MALLOC_ASSERT(object<=bumpPtr, msg);\n else {\n // non-startup objects are 8 Byte aligned\n MALLOC_ASSERT(isAligned(object, 8), \"Try to free invalid small object\");\n FreeObject *toFree = findObjectToFree(object);\n#if !__TBB_USE_THREAD_SANITIZER\n MALLOC_ASSERT(allocatedCount <= (slabSize-sizeof(Block))/objectSize\n && (!bumpPtr || object>bumpPtr), msg);\n // check against head of freeList, as this is mostly\n // expected after double free\n MALLOC_ASSERT(toFree != freeList, msg);\n#endif\n // check against head of publicFreeList, to detect double free\n // involving foreign thread\n MALLOC_ASSERT(toFree != publicFreeList.load(std::memory_order_relaxed), msg);\n }\n#else\n suppress_unused_warning(object);\n#endif\n }\n void initEmptyBlock(TLSData *tls, size_t size);\n size_t findObjectSize(void *object) const;\n MemoryPool *getMemPool() const { return poolPtr; } // do not use on the hot path!\n\nprotected:\n void cleanBlockHeader();\n\nprivate:\n static const float emptyEnoughRatio; /* Threshold on free space needed to \"reactivate\" a block */\n\n inline FreeObject *allocateFromBumpPtr();\n inline FreeObject *findAllocatedObject(const void *address) const;\n#if MALLOC_DEBUG\n inline bool isProperlyPlaced(const void *object) const;\n#endif\n inline void markOwned(TLSData *tls) {\n MALLOC_ASSERT(!tlsPtr.load(std::memory_order_relaxed), ASSERT_TEXT);\n ownerTid = ThreadId(); /* save the ID of the current thread */\n tlsPtr.store(tls, std::memory_order_relaxed);\n }\n inline void markOrphaned() {\n MALLOC_ASSERT(tlsPtr.load(std::memory_order_relaxed), ASSERT_TEXT);\n tlsPtr.store(nullptr, std::memory_order_relaxed);\n }\n\n friend class Bin;\n friend class TLSData;\n friend bool MemoryPool::destroy();\n};\n\nconst float Block::emptyEnoughRatio = 1.0 / 4.0;\n\nstatic_assert(sizeof(Block) <= 2*estimatedCacheLineSize,\n \"The class Block does not fit into 2 cache lines on this platform. \"\n \"Defining USE_INTERNAL_TID may help to fix it.\");\n\nclass Bin {\nprivate:\npublic:\n Block *activeBlk;\n std::atomic mailbox;\n MallocMutex mailLock;\n\npublic:\n inline Block* getActiveBlock() const { return activeBlk; }\n void resetActiveBlock() { activeBlk = nullptr; }\n inline void setActiveBlock(Block *block);\n inline Block* setPreviousBlockActive();\n Block* getPrivatizedFreeListBlock();\n void moveBlockToFront(Block *block);\n bool cleanPublicFreeLists();\n void processEmptyBlock(Block *block, bool poolTheBlock);\n void addPublicFreeListBlock(Block* block);\n\n void outofTLSBin(Block* block);\n void verifyTLSBin(size_t size) const;\n void pushTLSBin(Block* block);\n\n#if MALLOC_DEBUG\n void verifyInitState() const {\n MALLOC_ASSERT( !activeBlk, ASSERT_TEXT );\n MALLOC_ASSERT( !mailbox.load(std::memory_order_relaxed), ASSERT_TEXT );\n }\n#endif\n\n friend void Block::freePublicObject (FreeObject *objectToFree);\n};\n\n/********* End of the data structures **************/\n\n/*\n * There are bins for all 8 byte aligned objects less than this segregated size; 8 bins in total\n */\nconst uint32_t minSmallObjectIndex = 0;\nconst uint32_t numSmallObjectBins = 8;\nconst uint32_t maxSmallObjectSize = 64;\n\n/*\n * There are 4 bins between each couple of powers of 2 [64-128-256-...]\n * from maxSmallObjectSize till this size; 16 bins in total\n */\nconst uint32_t minSegregatedObjectIndex = minSmallObjectIndex+numSmallObjectBins;\nconst uint32_t numSegregatedObjectBins = 16;\nconst uint32_t maxSegregatedObjectSize = 1024;\n\n/*\n * And there are 5 bins with allocation sizes that are multiples of estimatedCacheLineSize\n * and selected to fit 9, 6, 4, 3, and 2 allocations in a block.\n */\nconst uint32_t minFittingIndex = minSegregatedObjectIndex+numSegregatedObjectBins;\nconst uint32_t numFittingBins = 5;\n\nconst uint32_t fittingAlignment = estimatedCacheLineSize;\n\n#define SET_FITTING_SIZE(N) ( (slabSize-sizeof(Block))/N ) & ~(fittingAlignment-1)\n// For blockSize=16*1024, sizeof(Block)=2*estimatedCacheLineSize and fittingAlignment=estimatedCacheLineSize,\n// the comments show the fitting sizes and the amounts left unused for estimatedCacheLineSize=64/128:\nconst uint32_t fittingSize1 = SET_FITTING_SIZE(9); // 1792/1792 128/000\nconst uint32_t fittingSize2 = SET_FITTING_SIZE(6); // 2688/2688 128/000\nconst uint32_t fittingSize3 = SET_FITTING_SIZE(4); // 4032/3968 128/256\nconst uint32_t fittingSize4 = SET_FITTING_SIZE(3); // 5376/5376 128/000\nconst uint32_t fittingSize5 = SET_FITTING_SIZE(2); // 8128/8064 000/000\n#undef SET_FITTING_SIZE\n\n/*\n * The total number of thread-specific Block-based bins\n */\nconst uint32_t numBlockBins = minFittingIndex+numFittingBins;\n\n/*\n * Objects of this size and larger are considered large objects.\n */\nconst uint32_t minLargeObjectSize = fittingSize5 + 1;\n\n/*\n * Per-thread pool of slab blocks. Idea behind it is to not share with other\n * threads memory that are likely in local cache(s) of our CPU.\n */\nclass FreeBlockPool {\nprivate:\n std::atomic head;\n int size;\n Backend *backend;\npublic:\n static const int POOL_HIGH_MARK = 32;\n static const int POOL_LOW_MARK = 8;\n\n class ResOfGet {\n ResOfGet() = delete;\n public:\n Block* block;\n bool lastAccMiss;\n ResOfGet(Block *b, bool lastMiss) : block(b), lastAccMiss(lastMiss) {}\n };\n\n // allocated in zero-initialized memory\n FreeBlockPool(Backend *bknd) : backend(bknd) {}\n ResOfGet getBlock();\n void returnBlock(Block *block);\n bool externalCleanup(); // can be called by another thread\n};\n\ntemplate\nclass LocalLOCImpl {\nprivate:\n static const size_t MAX_TOTAL_SIZE = 4*1024*1024;\n // TODO: can single-linked list be faster here?\n LargeMemoryBlock *tail; // need it when do releasing on overflow\n std::atomic head;\n size_t totalSize;\n int numOfBlocks;\npublic:\n bool put(LargeMemoryBlock *object, ExtMemoryPool *extMemPool);\n LargeMemoryBlock *get(size_t size);\n bool externalCleanup(ExtMemoryPool *extMemPool);\n#if __TBB_MALLOC_WHITEBOX_TEST\n LocalLOCImpl() : tail(nullptr), head(nullptr), totalSize(0), numOfBlocks(0) {}\n static size_t getMaxSize() { return MAX_TOTAL_SIZE; }\n static const int LOC_HIGH_MARK = HIGH_MARK;\n#else\n // no ctor, object must be created in zero-initialized memory\n#endif\n};\n\ntypedef LocalLOCImpl<8,32> LocalLOC; // set production code parameters\n\nclass TLSData : public TLSRemote {\n MemoryPool *memPool;\npublic:\n Bin bin[numBlockBinLimit];\n FreeBlockPool freeSlabBlocks;\n LocalLOC lloc;\n unsigned currCacheIdx;\nprivate:\n std::atomic unused;\npublic:\n TLSData(MemoryPool *mPool, Backend *bknd) : memPool(mPool), freeSlabBlocks(bknd), currCacheIdx(0) {}\n MemoryPool *getMemPool() const { return memPool; }\n Bin* getAllocationBin(size_t size);\n void release();\n bool externalCleanup(bool cleanOnlyUnused, bool cleanBins) {\n if (!unused.load(std::memory_order_relaxed) && cleanOnlyUnused) return false;\n // Heavy operation in terms of synchronization complexity,\n // should be called only for the current thread\n bool released = cleanBins ? cleanupBlockBins() : false;\n // both cleanups to be called, and the order is not important\n bool lloc_cleaned = lloc.externalCleanup(&memPool->extMemPool);\n bool free_slab_blocks_cleaned = freeSlabBlocks.externalCleanup();\n return released || lloc_cleaned || free_slab_blocks_cleaned;\n }\n bool cleanupBlockBins();\n void markUsed() { unused.store(false, std::memory_order_relaxed); } // called by owner when TLS touched\n void markUnused() { unused.store(true, std::memory_order_relaxed); } // can be called by not owner thread\n};\n\nTLSData *TLSKey::createTLS(MemoryPool *memPool, Backend *backend)\n{\n MALLOC_ASSERT( sizeof(TLSData) >= sizeof(Bin) * numBlockBins + sizeof(FreeBlockPool), ASSERT_TEXT );\n TLSData* tls = (TLSData*) memPool->bootStrapBlocks.allocate(memPool, sizeof(TLSData));\n if ( !tls )\n return nullptr;\n new(tls) TLSData(memPool, backend);\n /* the block contains zeroes after bootStrapMalloc, so bins are initialized */\n#if MALLOC_DEBUG\n for (uint32_t i = 0; i < numBlockBinLimit; i++)\n tls->bin[i].verifyInitState();\n#endif\n setThreadMallocTLS(tls);\n memPool->extMemPool.allLocalCaches.registerThread(tls);\n return tls;\n}\n\nbool TLSData::cleanupBlockBins()\n{\n bool released = false;\n for (uint32_t i = 0; i < numBlockBinLimit; i++) {\n released |= bin[i].cleanPublicFreeLists();\n // After cleaning public free lists, only the active block might be empty.\n // Do not use processEmptyBlock because it will just restore bumpPtr.\n Block *block = bin[i].getActiveBlock();\n if (block && block->empty()) {\n bin[i].outofTLSBin(block);\n memPool->returnEmptyBlock(block, /*poolTheBlock=*/false);\n released = true;\n }\n }\n return released;\n}\n\nbool ExtMemoryPool::releaseAllLocalCaches()\n{\n // Iterate all registered TLS data and clean LLOC and Slab pools\n bool released = allLocalCaches.cleanup(/*cleanOnlyUnused=*/false);\n\n // Bins privatization is done only for the current thread\n if (TLSData *tlsData = tlsPointerKey.getThreadMallocTLS())\n released |= tlsData->cleanupBlockBins();\n\n return released;\n}\n\nvoid AllLocalCaches::registerThread(TLSRemote *tls)\n{\n tls->prev = nullptr;\n MallocMutex::scoped_lock lock(listLock);\n MALLOC_ASSERT(head!=tls, ASSERT_TEXT);\n tls->next = head;\n if (head)\n head->prev = tls;\n head = tls;\n MALLOC_ASSERT(head->next!=head, ASSERT_TEXT);\n}\n\nvoid AllLocalCaches::unregisterThread(TLSRemote *tls)\n{\n MallocMutex::scoped_lock lock(listLock);\n MALLOC_ASSERT(head, \"Can't unregister thread: no threads are registered.\");\n if (head == tls)\n head = tls->next;\n if (tls->next)\n tls->next->prev = tls->prev;\n if (tls->prev)\n tls->prev->next = tls->next;\n MALLOC_ASSERT(!tls->next || tls->next->next!=tls->next, ASSERT_TEXT);\n}\n\nbool AllLocalCaches::cleanup(bool cleanOnlyUnused)\n{\n bool released = false;\n {\n MallocMutex::scoped_lock lock(listLock);\n for (TLSRemote *curr=head; curr; curr=curr->next)\n released |= static_cast(curr)->externalCleanup(cleanOnlyUnused, /*cleanBins=*/false);\n }\n return released;\n}\n\nvoid AllLocalCaches::markUnused()\n{\n bool locked = false;\n MallocMutex::scoped_lock lock(listLock, /*block=*/false, &locked);\n if (!locked) // not wait for marking if someone doing something with it\n return;\n\n for (TLSRemote *curr=head; curr; curr=curr->next)\n static_cast(curr)->markUnused();\n}\n\n#if MALLOC_CHECK_RECURSION\nMallocMutex RecursiveMallocCallProtector::rmc_mutex;\nstd::atomic RecursiveMallocCallProtector::owner_thread;\nstd::atomic RecursiveMallocCallProtector::autoObjPtr;\nbool RecursiveMallocCallProtector::mallocRecursionDetected;\n#if __FreeBSD__\nbool RecursiveMallocCallProtector::canUsePthread;\n#endif\n\n#endif\n\n/*********** End code to provide thread ID and a TLS pointer **********/\n\n// Parameter for isLargeObject, keeps our expectations on memory origin.\n// Assertions must use unknownMem to reliably report object invalidity.\nenum MemoryOrigin {\n ourMem, // allocated by TBB allocator\n unknownMem // can be allocated by system allocator or TBB allocator\n};\n\ntemplate\n#if __TBB_USE_THREAD_SANITIZER\n// We have a real race when accessing the large object header for\n// non large objects (e.g. small or foreign objects).\n// Therefore, we need to hide this access from the thread sanitizer\n__attribute__((no_sanitize(\"thread\")))\n#endif\nbool isLargeObject(void *object);\nstatic void *internalMalloc(size_t size);\nstatic void internalFree(void *object);\nstatic void *internalPoolMalloc(MemoryPool* mPool, size_t size);\nstatic bool internalPoolFree(MemoryPool *mPool, void *object, size_t size);\n\n#if !MALLOC_DEBUG\n#if __INTEL_COMPILER || _MSC_VER\n#define NOINLINE(decl) __declspec(noinline) decl\n#define ALWAYSINLINE(decl) __forceinline decl\n#elif __GNUC__\n#define NOINLINE(decl) decl __attribute__ ((noinline))\n#define ALWAYSINLINE(decl) decl __attribute__ ((always_inline))\n#else\n#define NOINLINE(decl) decl\n#define ALWAYSINLINE(decl) decl\n#endif\n\nstatic NOINLINE( bool doInitialization() );\nALWAYSINLINE( bool isMallocInitialized() );\n\n#undef ALWAYSINLINE\n#undef NOINLINE\n#endif /* !MALLOC_DEBUG */\n\n\n/********* Now some rough utility code to deal with indexing the size bins. **************/\n\n/*\n * Given a number return the highest non-zero bit in it. It is intended to work with 32-bit values only.\n * Moreover, on some platforms, for sake of simplicity and performance, it is narrowed to only serve for 64 to 1023.\n * This is enough for current algorithm of distribution of sizes among bins.\n * __TBB_Log2 is not used here to minimize dependencies on TBB specific sources.\n */\n#if _WIN64 && _MSC_VER>=1400 && !__INTEL_COMPILER\nextern \"C\" unsigned char _BitScanReverse( unsigned long* i, unsigned long w );\n#pragma intrinsic(_BitScanReverse)\n#endif\nstatic inline unsigned int highestBitPos(unsigned int n)\n{\n MALLOC_ASSERT( n>=64 && n<1024, ASSERT_TEXT ); // only needed for bsr array lookup, but always true\n unsigned int pos;\n#if __ARCH_x86_32||__ARCH_x86_64\n\n# if __unix__||__APPLE__||__MINGW32__\n __asm__ (\"bsr %1,%0\" : \"=r\"(pos) : \"r\"(n));\n# elif (_WIN32 && (!_WIN64 || __INTEL_COMPILER))\n __asm\n {\n bsr eax, n\n mov pos, eax\n }\n# elif _WIN64 && _MSC_VER>=1400\n _BitScanReverse((unsigned long*)&pos, (unsigned long)n);\n# else\n# error highestBitPos() not implemented for this platform\n# endif\n#elif __arm__\n __asm__ __volatile__\n (\n \"clz %0, %1\\n\"\n \"rsb %0, %0, %2\\n\"\n :\"=r\" (pos) :\"r\" (n), \"I\" (31)\n );\n#else\n static unsigned int bsr[16] = {0/*N/A*/,6,7,7,8,8,8,8,9,9,9,9,9,9,9,9};\n pos = bsr[ n>>6 ];\n#endif /* __ARCH_* */\n return pos;\n}\n\nunsigned int getSmallObjectIndex(unsigned int size)\n{\n unsigned int result = (size-1)>>3;\n constexpr bool is_64bit = (8 == sizeof(void*));\n if (is_64bit) {\n // For 64-bit malloc, 16 byte alignment is needed except for bin 0.\n if (result) result |= 1; // 0,1,3,5,7; bins 2,4,6 are not aligned to 16 bytes\n }\n return result;\n}\n\n/*\n * Depending on indexRequest, for a given size return either the index into the bin\n * for objects of this size, or the actual size of objects in this bin.\n * TODO: Change return type to unsigned short.\n */\ntemplate\nstatic unsigned int getIndexOrObjectSize (unsigned int size)\n{\n if (size <= maxSmallObjectSize) { // selection from 8/16/24/32/40/48/56/64\n unsigned int index = getSmallObjectIndex( size );\n /* Bin 0 is for 8 bytes, bin 1 is for 16, and so forth */\n return indexRequest ? index : (index+1)<<3;\n }\n else if (size <= maxSegregatedObjectSize ) { // 80/96/112/128 / 160/192/224/256 / 320/384/448/512 / 640/768/896/1024\n unsigned int order = highestBitPos(size-1); // which group of bin sizes?\n MALLOC_ASSERT( 6<=order && order<=9, ASSERT_TEXT );\n if (indexRequest)\n return minSegregatedObjectIndex - (4*6) - 4 + (4*order) + ((size-1)>>(order-2));\n else {\n unsigned int alignment = 128 >> (9-order); // alignment in the group\n MALLOC_ASSERT( alignment==16 || alignment==32 || alignment==64 || alignment==128, ASSERT_TEXT );\n return alignUp(size,alignment);\n }\n }\n else {\n if( size <= fittingSize3 ) {\n if( size <= fittingSize2 ) {\n if( size <= fittingSize1 )\n return indexRequest ? minFittingIndex : fittingSize1;\n else\n return indexRequest ? minFittingIndex+1 : fittingSize2;\n } else\n return indexRequest ? minFittingIndex+2 : fittingSize3;\n } else {\n if( size <= fittingSize5 ) {\n if( size <= fittingSize4 )\n return indexRequest ? minFittingIndex+3 : fittingSize4;\n else\n return indexRequest ? minFittingIndex+4 : fittingSize5;\n } else {\n MALLOC_ASSERT( 0,ASSERT_TEXT ); // this should not happen\n return ~0U;\n }\n }\n }\n}\n\nstatic unsigned int getIndex (unsigned int size)\n{\n return getIndexOrObjectSize(size);\n}\n\nstatic unsigned int getObjectSize (unsigned int size)\n{\n return getIndexOrObjectSize(size);\n}\n\n\nvoid *BootStrapBlocks::allocate(MemoryPool *memPool, size_t size)\n{\n FreeObject *result;\n\n MALLOC_ASSERT( size == sizeof(TLSData), ASSERT_TEXT );\n\n { // Lock with acquire\n MallocMutex::scoped_lock scoped_cs(bootStrapLock);\n\n if( bootStrapObjectList) {\n result = bootStrapObjectList;\n bootStrapObjectList = bootStrapObjectList->next;\n } else {\n if (!bootStrapBlock) {\n bootStrapBlock = memPool->getEmptyBlock(size);\n if (!bootStrapBlock) return nullptr;\n }\n result = bootStrapBlock->bumpPtr;\n bootStrapBlock->bumpPtr = (FreeObject *)((uintptr_t)bootStrapBlock->bumpPtr - bootStrapBlock->objectSize);\n if ((uintptr_t)bootStrapBlock->bumpPtr < (uintptr_t)bootStrapBlock+sizeof(Block)) {\n bootStrapBlock->bumpPtr = nullptr;\n bootStrapBlock->next = bootStrapBlockUsed;\n bootStrapBlockUsed = bootStrapBlock;\n bootStrapBlock = nullptr;\n }\n }\n } // Unlock with release\n memset (result, 0, size);\n return (void*)result;\n}\n\nvoid BootStrapBlocks::free(void* ptr)\n{\n MALLOC_ASSERT( ptr, ASSERT_TEXT );\n { // Lock with acquire\n MallocMutex::scoped_lock scoped_cs(bootStrapLock);\n ((FreeObject*)ptr)->next = bootStrapObjectList;\n bootStrapObjectList = (FreeObject*)ptr;\n } // Unlock with release\n}\n\nvoid BootStrapBlocks::reset()\n{\n bootStrapBlock = bootStrapBlockUsed = nullptr;\n bootStrapObjectList = nullptr;\n}\n\n#if !(FREELIST_NONBLOCKING)\nstatic MallocMutex publicFreeListLock; // lock for changes of publicFreeList\n#endif\n\n/********* End rough utility code **************/\n\n/* LifoList assumes zero initialization so a vector of it can be created\n * by just allocating some space with no call to constructor.\n * On Linux, it seems to be necessary to avoid linking with C++ libraries.\n *\n * By usage convention there is no race on the initialization. */\nLifoList::LifoList( ) : top(nullptr)\n{\n // MallocMutex assumes zero initialization\n memset(static_cast(&lock), 0, sizeof(MallocMutex));\n}\n\nvoid LifoList::push(Block *block)\n{\n MallocMutex::scoped_lock scoped_cs(lock);\n block->next = top.load(std::memory_order_relaxed);\n top.store(block, std::memory_order_relaxed);\n}\n\nBlock *LifoList::pop()\n{\n Block* block = nullptr;\n if (top.load(std::memory_order_relaxed)) {\n MallocMutex::scoped_lock scoped_cs(lock);\n block = top.load(std::memory_order_relaxed);\n if (block) {\n top.store(block->next, std::memory_order_relaxed);\n }\n }\n return block;\n}\n\nBlock *LifoList::grab()\n{\n Block *block = nullptr;\n if (top.load(std::memory_order_relaxed)) {\n MallocMutex::scoped_lock scoped_cs(lock);\n block = top.load(std::memory_order_relaxed);\n top.store(nullptr, std::memory_order_relaxed);\n }\n return block;\n}\n\n/********* Thread and block related code *************/\n\ntemplate void AllLargeBlocksList::releaseAll(Backend *backend) {\n LargeMemoryBlock *next, *lmb = loHead;\n loHead = nullptr;\n\n for (; lmb; lmb = next) {\n next = lmb->gNext;\n if (poolDestroy) {\n // as it's pool destruction, no need to return object to backend,\n // only remove backrefs, as they are global\n removeBackRef(lmb->backRefIdx);\n } else {\n // clean g(Next|Prev) to prevent removing lmb\n // from AllLargeBlocksList inside returnLargeObject\n lmb->gNext = lmb->gPrev = nullptr;\n backend->returnLargeObject(lmb);\n }\n }\n}\n\nTLSData* MemoryPool::getTLS(bool create)\n{\n TLSData* tls = extMemPool.tlsPointerKey.getThreadMallocTLS();\n if (create && !tls)\n tls = extMemPool.tlsPointerKey.createTLS(this, &extMemPool.backend);\n return tls;\n}\n\n/*\n * Return the bin for the given size.\n */\ninline Bin* TLSData::getAllocationBin(size_t size)\n{\n return bin + getIndex(size);\n}\n\n/* Return an empty uninitialized block in a non-blocking fashion. */\nBlock *MemoryPool::getEmptyBlock(size_t size)\n{\n TLSData* tls = getTLS(/*create=*/false);\n // try to use per-thread cache, if TLS available\n FreeBlockPool::ResOfGet resOfGet = tls?\n tls->freeSlabBlocks.getBlock() : FreeBlockPool::ResOfGet(nullptr, false);\n Block *result = resOfGet.block;\n\n if (!result) { // not found in local cache, asks backend for slabs\n int num = resOfGet.lastAccMiss? Backend::numOfSlabAllocOnMiss : 1;\n BackRefIdx backRefIdx[Backend::numOfSlabAllocOnMiss];\n\n result = static_cast(extMemPool.backend.getSlabBlock(num));\n if (!result) return nullptr;\n\n if (!extMemPool.userPool())\n for (int i=0; ibackRefIdx) BackRefIdx();\n } else {\n setBackRef(backRefIdx[i], b);\n b->backRefIdx = backRefIdx[i];\n }\n b->tlsPtr.store(tls, std::memory_order_relaxed);\n b->poolPtr = this;\n // all but first one go to per-thread pool\n if (i > 0) {\n MALLOC_ASSERT(tls, ASSERT_TEXT);\n tls->freeSlabBlocks.returnBlock(b);\n }\n }\n }\n MALLOC_ASSERT(result, ASSERT_TEXT);\n result->initEmptyBlock(tls, size);\n STAT_increment(getThreadId(), getIndex(result->objectSize), allocBlockNew);\n return result;\n}\n\nvoid MemoryPool::returnEmptyBlock(Block *block, bool poolTheBlock)\n{\n block->reset();\n if (poolTheBlock) {\n getTLS(/*create=*/false)->freeSlabBlocks.returnBlock(block);\n } else {\n // slab blocks in user's pools do not have valid backRefIdx\n if (!extMemPool.userPool())\n removeBackRef(*(block->getBackRefIdx()));\n extMemPool.backend.putSlabBlock(block);\n }\n}\n\nbool ExtMemoryPool::init(intptr_t poolId, rawAllocType rawAlloc,\n rawFreeType rawFree, size_t granularity,\n bool keepAllMemory, bool fixedPool)\n{\n this->poolId = poolId;\n this->rawAlloc = rawAlloc;\n this->rawFree = rawFree;\n this->granularity = granularity;\n this->keepAllMemory = keepAllMemory;\n this->fixedPool = fixedPool;\n this->delayRegsReleasing = false;\n if (!initTLS())\n return false;\n loc.init(this);\n backend.init(this);\n MALLOC_ASSERT(isPoolValid(), nullptr);\n return true;\n}\n\nbool ExtMemoryPool::initTLS() { return tlsPointerKey.init(); }\n\nbool MemoryPool::init(intptr_t poolId, const MemPoolPolicy *policy)\n{\n if (!extMemPool.init(poolId, policy->pAlloc, policy->pFree,\n policy->granularity? policy->granularity : defaultGranularity,\n policy->keepAllMemory, policy->fixedPool))\n return false;\n {\n MallocMutex::scoped_lock lock(memPoolListLock);\n next = defaultMemPool->next;\n defaultMemPool->next = this;\n prev = defaultMemPool;\n if (next)\n next->prev = this;\n }\n return true;\n}\n\nbool MemoryPool::reset()\n{\n MALLOC_ASSERT(extMemPool.userPool(), \"No reset for the system pool.\");\n // memory is not releasing during pool reset\n // TODO: mark regions to release unused on next reset()\n extMemPool.delayRegionsReleasing(true);\n\n bootStrapBlocks.reset();\n extMemPool.lmbList.releaseAll(&extMemPool.backend);\n if (!extMemPool.reset())\n return false;\n\n if (!extMemPool.initTLS())\n return false;\n extMemPool.delayRegionsReleasing(false);\n return true;\n}\n\nbool MemoryPool::destroy()\n{\n#if __TBB_MALLOC_LOCACHE_STAT\n extMemPool.loc.reportStat(stdout);\n#endif\n#if __TBB_MALLOC_BACKEND_STAT\n extMemPool.backend.reportStat(stdout);\n#endif\n {\n MallocMutex::scoped_lock lock(memPoolListLock);\n // remove itself from global pool list\n if (prev)\n prev->next = next;\n if (next)\n next->prev = prev;\n }\n // slab blocks in non-default pool do not have backreferences,\n // only large objects do\n if (extMemPool.userPool())\n extMemPool.lmbList.releaseAll(&extMemPool.backend);\n else {\n // only one non-userPool() is supported now\n MALLOC_ASSERT(this==defaultMemPool, nullptr);\n // There and below in extMemPool.destroy(), do not restore initial state\n // for user pool, because it's just about to be released. But for system\n // pool restoring, we do not want to do zeroing of it on subsequent reload.\n bootStrapBlocks.reset();\n extMemPool.orphanedBlocks.reset();\n }\n return extMemPool.destroy();\n}\n\nvoid MemoryPool::onThreadShutdown(TLSData *tlsData)\n{\n if (tlsData) { // might be called for \"empty\" TLS\n tlsData->release();\n bootStrapBlocks.free(tlsData);\n clearTLS();\n }\n}\n\n#if MALLOC_DEBUG\nvoid Bin::verifyTLSBin (size_t size) const\n{\n/* The debug version verifies the TLSBin as needed */\n uint32_t objSize = getObjectSize(size);\n\n if (activeBlk) {\n MALLOC_ASSERT( activeBlk->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( activeBlk->objectSize == objSize, ASSERT_TEXT );\n#if MALLOC_DEBUG>1\n for (Block* temp = activeBlk->next; temp; temp=temp->next) {\n MALLOC_ASSERT( temp!=activeBlk, ASSERT_TEXT );\n MALLOC_ASSERT( temp->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( temp->objectSize == objSize, ASSERT_TEXT );\n MALLOC_ASSERT( temp->previous->next == temp, ASSERT_TEXT );\n if (temp->next) {\n MALLOC_ASSERT( temp->next->previous == temp, ASSERT_TEXT );\n }\n }\n for (Block* temp = activeBlk->previous; temp; temp=temp->previous) {\n MALLOC_ASSERT( temp!=activeBlk, ASSERT_TEXT );\n MALLOC_ASSERT( temp->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( temp->objectSize == objSize, ASSERT_TEXT );\n MALLOC_ASSERT( temp->next->previous == temp, ASSERT_TEXT );\n if (temp->previous) {\n MALLOC_ASSERT( temp->previous->next == temp, ASSERT_TEXT );\n }\n }\n#endif /* MALLOC_DEBUG>1 */\n }\n}\n#else /* MALLOC_DEBUG */\ninline void Bin::verifyTLSBin (size_t) const { }\n#endif /* MALLOC_DEBUG */\n\n/*\n * Add a block to the start of this tls bin list.\n */\nvoid Bin::pushTLSBin(Block* block)\n{\n /* The objectSize should be defined and not a parameter\n because the function is applied to partially filled blocks as well */\n unsigned int size = block->objectSize;\n\n MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( block->objectSize != 0, ASSERT_TEXT );\n MALLOC_ASSERT( block->next == nullptr, ASSERT_TEXT );\n MALLOC_ASSERT( block->previous == nullptr, ASSERT_TEXT );\n\n MALLOC_ASSERT( this, ASSERT_TEXT );\n verifyTLSBin(size);\n\n block->next = activeBlk;\n if( activeBlk ) {\n block->previous = activeBlk->previous;\n activeBlk->previous = block;\n if( block->previous )\n block->previous->next = block;\n } else {\n activeBlk = block;\n }\n\n verifyTLSBin(size);\n}\n\n/*\n * Take a block out of its tls bin (e.g. before removal).\n */\nvoid Bin::outofTLSBin(Block* block)\n{\n unsigned int size = block->objectSize;\n\n MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( block->objectSize != 0, ASSERT_TEXT );\n\n MALLOC_ASSERT( this, ASSERT_TEXT );\n verifyTLSBin(size);\n\n if (block == activeBlk) {\n activeBlk = block->previous? block->previous : block->next;\n }\n /* Unlink the block */\n if (block->previous) {\n MALLOC_ASSERT( block->previous->next == block, ASSERT_TEXT );\n block->previous->next = block->next;\n }\n if (block->next) {\n MALLOC_ASSERT( block->next->previous == block, ASSERT_TEXT );\n block->next->previous = block->previous;\n }\n block->next = nullptr;\n block->previous = nullptr;\n\n verifyTLSBin(size);\n}\n\nBlock* Bin::getPrivatizedFreeListBlock()\n{\n Block* block;\n MALLOC_ASSERT( this, ASSERT_TEXT );\n // if this method is called, active block usage must be unsuccessful\n MALLOC_ASSERT( (!activeBlk && !mailbox.load(std::memory_order_relaxed)) || (activeBlk && activeBlk->isFull), ASSERT_TEXT );\n\n// the counter should be changed STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList);\n if (!mailbox.load(std::memory_order_acquire)) // hotpath is empty mailbox\n return nullptr;\n else { // mailbox is not empty, take lock and inspect it\n MallocMutex::scoped_lock scoped_cs(mailLock);\n block = mailbox.load(std::memory_order_relaxed);\n if( block ) {\n MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT );\n MALLOC_ASSERT( !isNotForUse(block->nextPrivatizable.load(std::memory_order_relaxed)), ASSERT_TEXT );\n mailbox.store(block->nextPrivatizable.load(std::memory_order_relaxed), std::memory_order_relaxed);\n block->nextPrivatizable.store((Block*)this, std::memory_order_relaxed);\n }\n }\n if( block ) {\n MALLOC_ASSERT( isSolidPtr(block->publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT );\n block->privatizePublicFreeList();\n block->adjustPositionInBin(this);\n }\n return block;\n}\n\nvoid Bin::addPublicFreeListBlock(Block* block)\n{\n MallocMutex::scoped_lock scoped_cs(mailLock);\n block->nextPrivatizable.store(mailbox.load(std::memory_order_relaxed), std::memory_order_relaxed);\n mailbox.store(block, std::memory_order_relaxed);\n}\n\n// Process publicly freed objects in all blocks and return empty blocks\n// to the backend in order to reduce overall footprint.\nbool Bin::cleanPublicFreeLists()\n{\n Block* block;\n if (!mailbox.load(std::memory_order_acquire))\n return false;\n else {\n // Grab all the blocks in the mailbox\n MallocMutex::scoped_lock scoped_cs(mailLock);\n block = mailbox.load(std::memory_order_relaxed);\n mailbox.store(nullptr, std::memory_order_relaxed);\n }\n bool released = false;\n while (block) {\n MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT );\n Block* tmp = block->nextPrivatizable.load(std::memory_order_relaxed);\n block->nextPrivatizable.store((Block*)this, std::memory_order_relaxed);\n block->privatizePublicFreeList();\n if (block->empty()) {\n processEmptyBlock(block, /*poolTheBlock=*/false);\n released = true;\n } else\n block->adjustPositionInBin(this);\n block = tmp;\n }\n return released;\n}\n\nbool Block::adjustFullness()\n{\n if (bumpPtr) {\n /* If we are still using a bump ptr for this block it is empty enough to use. */\n STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough);\n isFull = false;\n } else {\n const float threshold = (slabSize - sizeof(Block)) * (1 - emptyEnoughRatio);\n /* allocatedCount shows how many objects in the block are in use; however it still counts\n * blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */\n isFull = (allocatedCount*objectSize > threshold) ? true : false;\n#if COLLECT_STATISTICS\n if (isFull)\n STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty);\n else\n STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough);\n#endif\n }\n return isFull;\n}\n\n// This method resides in class Block, and not in class Bin, in order to avoid\n// calling getAllocationBin on a reasonably hot path in Block::freeOwnObject\nvoid Block::adjustPositionInBin(Bin* bin/*=nullptr*/)\n{\n // If the block were full, but became empty enough to use,\n // move it to the front of the list\n if (isFull && !adjustFullness()) {\n if (!bin)\n bin = tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize);\n bin->moveBlockToFront(this);\n }\n}\n\n/* Restore the bump pointer for an empty block that is planned to use */\nvoid Block::restoreBumpPtr()\n{\n MALLOC_ASSERT( allocatedCount == 0, ASSERT_TEXT );\n MALLOC_ASSERT( !isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT );\n STAT_increment(getThreadId(), getIndex(objectSize), freeRestoreBumpPtr);\n bumpPtr = (FreeObject *)((uintptr_t)this + slabSize - objectSize);\n freeList = nullptr;\n isFull = false;\n}\n\nvoid Block::freeOwnObject(void *object)\n{\n tlsPtr.load(std::memory_order_relaxed)->markUsed();\n allocatedCount--;\n MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT );\n#if COLLECT_STATISTICS\n // Note that getAllocationBin is not called on the hottest path with statistics off.\n if (tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize)->getActiveBlock() != this)\n STAT_increment(getThreadId(), getIndex(objectSize), freeToInactiveBlock);\n else\n STAT_increment(getThreadId(), getIndex(objectSize), freeToActiveBlock);\n#endif\n if (empty()) {\n // If the last object of a slab is freed, the slab cannot be marked full\n MALLOC_ASSERT(!isFull, ASSERT_TEXT);\n tlsPtr.load(std::memory_order_relaxed)->getAllocationBin(objectSize)->processEmptyBlock(this, /*poolTheBlock=*/true);\n } else { // hot path\n FreeObject *objectToFree = findObjectToFree(object);\n objectToFree->next = freeList;\n freeList = objectToFree;\n adjustPositionInBin();\n }\n}\n\nvoid Block::freePublicObject (FreeObject *objectToFree)\n{\n FreeObject* localPublicFreeList{};\n\n MALLOC_ITT_SYNC_RELEASING(&publicFreeList);\n#if FREELIST_NONBLOCKING\n // TBB_REVAMP_TODO: make it non atomic in non-blocking scenario\n localPublicFreeList = publicFreeList.load(std::memory_order_relaxed);\n do {\n objectToFree->next = localPublicFreeList;\n // no backoff necessary because trying to make change, not waiting for a change\n } while( !publicFreeList.compare_exchange_strong(localPublicFreeList, objectToFree) );\n#else\n STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList);\n {\n MallocMutex::scoped_lock scoped_cs(publicFreeListLock);\n localPublicFreeList = objectToFree->next = publicFreeList;\n publicFreeList = objectToFree;\n }\n#endif\n\n if( localPublicFreeList==nullptr ) {\n // if the block is abandoned, its nextPrivatizable pointer should be UNUSABLE\n // otherwise, it should point to the bin the block belongs to.\n // reading nextPrivatizable is thread-safe below, because:\n // 1) the executing thread atomically got publicFreeList==nullptr and changed it to non-nullptr;\n // 2) only owning thread can change it back to nullptr,\n // 3) but it can not be done until the block is put to the mailbox\n // So the executing thread is now the only one that can change nextPrivatizable\n Block* next = nextPrivatizable.load(std::memory_order_acquire);\n if( !isNotForUse(next) ) {\n MALLOC_ASSERT( next!=nullptr, ASSERT_TEXT );\n Bin* theBin = (Bin*) next;\n#if MALLOC_DEBUG && TBB_REVAMP_TODO\n // FIXME: The thread that returns the block is not the block's owner.\n // The below assertion compares 'theBin' against the caller's local bin, thus, it always fails.\n // Need to find a way to get the correct remote bin for comparison.\n { // check that nextPrivatizable points to the bin the block belongs to\n uint32_t index = getIndex( objectSize );\n TLSData* tls = getThreadMallocTLS();\n MALLOC_ASSERT( theBin==tls->bin+index, ASSERT_TEXT );\n }\n#endif // MALLOC_DEBUG\n theBin->addPublicFreeListBlock(this);\n }\n }\n STAT_increment(getThreadId(), ThreadCommonCounters, freeToOtherThread);\n STAT_increment(ownerTid.load(std::memory_order_relaxed), getIndex(objectSize), freeByOtherThread);\n}\n\n// Make objects freed by other threads available for use again\nvoid Block::privatizePublicFreeList( bool reset )\n{\n FreeObject *localPublicFreeList;\n // If reset is false, publicFreeList should not be zeroed but set to UNUSABLE\n // to properly synchronize with other threads freeing objects to this slab.\n const intptr_t endMarker = reset ? 0 : UNUSABLE;\n\n // Only the owner thread may reset the pointer to nullptr\n MALLOC_ASSERT( isOwnedByCurrentThread() || !reset, ASSERT_TEXT );\n#if FREELIST_NONBLOCKING\n localPublicFreeList = publicFreeList.exchange((FreeObject*)endMarker);\n#else\n STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList);\n {\n MallocMutex::scoped_lock scoped_cs(publicFreeListLock);\n localPublicFreeList = publicFreeList;\n publicFreeList = endMarker;\n }\n#endif\n MALLOC_ITT_SYNC_ACQUIRED(&publicFreeList);\n MALLOC_ASSERT( !(reset && isNotForUse(publicFreeList)), ASSERT_TEXT );\n\n // publicFreeList must have been UNUSABLE or valid, but not nullptr\n MALLOC_ASSERT( localPublicFreeList!=nullptr, ASSERT_TEXT );\n if( isSolidPtr(localPublicFreeList) ) {\n MALLOC_ASSERT( allocatedCount <= (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT );\n /* other threads did not change the counter freeing our blocks */\n allocatedCount--;\n FreeObject *temp = localPublicFreeList;\n while( isSolidPtr(temp->next) ){ // the list will end with either nullptr or UNUSABLE\n temp = temp->next;\n allocatedCount--;\n MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT );\n }\n /* merge with local freeList */\n temp->next = freeList;\n freeList = localPublicFreeList;\n STAT_increment(getThreadId(), getIndex(objectSize), allocPrivatized);\n }\n}\n\nvoid Block::privatizeOrphaned(TLSData *tls, unsigned index)\n{\n Bin* bin = tls->bin + index;\n STAT_increment(getThreadId(), index, allocBlockPublic);\n next = nullptr;\n previous = nullptr;\n MALLOC_ASSERT( publicFreeList.load(std::memory_order_relaxed) != nullptr, ASSERT_TEXT );\n /* There is not a race here since no other thread owns this block */\n markOwned(tls);\n // It is safe to change nextPrivatizable, as publicFreeList is not null\n MALLOC_ASSERT( isNotForUse(nextPrivatizable.load(std::memory_order_relaxed)), ASSERT_TEXT );\n nextPrivatizable.store((Block*)bin, std::memory_order_relaxed);\n // the next call is required to change publicFreeList to 0\n privatizePublicFreeList();\n if( empty() ) {\n restoreBumpPtr();\n } else {\n adjustFullness(); // check the block fullness and set isFull\n }\n MALLOC_ASSERT( !isNotForUse(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT );\n}\n\n\nbool Block::readyToShare()\n{\n FreeObject* oldVal = nullptr;\n#if FREELIST_NONBLOCKING\n publicFreeList.compare_exchange_strong(oldVal, (FreeObject*)UNUSABLE);\n#else\n STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList);\n {\n MallocMutex::scoped_lock scoped_cs(publicFreeListLock);\n if ( (oldVal=publicFreeList)==nullptr )\n publicFreeList = reinterpret_cast(UNUSABLE);\n }\n#endif\n return oldVal==nullptr;\n}\n\nvoid Block::shareOrphaned(intptr_t binTag, unsigned index)\n{\n MALLOC_ASSERT( binTag, ASSERT_TEXT );\n // unreferenced formal parameter warning\n tbb::detail::suppress_unused_warning(index);\n STAT_increment(getThreadId(), index, freeBlockPublic);\n markOrphaned();\n if ((intptr_t)nextPrivatizable.load(std::memory_order_relaxed) == binTag) {\n // First check passed: the block is not in mailbox yet.\n // Need to set publicFreeList to non-zero, so other threads\n // will not change nextPrivatizable and it can be zeroed.\n if ( !readyToShare() ) {\n // another thread freed an object; we need to wait until it finishes.\n // There is no need for exponential backoff, as the wait here is not for a lock;\n // but need to yield, so the thread we wait has a chance to run.\n // TODO: add a pause to also be friendly to hyperthreads\n int count = 256;\n while ((intptr_t)nextPrivatizable.load(std::memory_order_relaxed) == binTag) {\n if (--count==0) {\n do_yield();\n count = 256;\n }\n }\n }\n }\n MALLOC_ASSERT( publicFreeList.load(std::memory_order_relaxed) !=nullptr, ASSERT_TEXT );\n // now it is safe to change our data\n previous = nullptr;\n // it is caller responsibility to ensure that the list of blocks\n // formed by nextPrivatizable pointers is kept consistent if required.\n // if only called from thread shutdown code, it does not matter.\n nextPrivatizable.store((Block*)UNUSABLE, std::memory_order_relaxed);\n}\n\nvoid Block::cleanBlockHeader()\n{\n next = nullptr;\n previous = nullptr;\n freeList = nullptr;\n allocatedCount = 0;\n isFull = false;\n tlsPtr.store(nullptr, std::memory_order_relaxed);\n\n publicFreeList.store(nullptr, std::memory_order_relaxed);\n}\n\nvoid Block::initEmptyBlock(TLSData *tls, size_t size)\n{\n // Having getIndex and getObjectSize called next to each other\n // allows better compiler optimization as they basically share the code.\n unsigned int index = getIndex(size);\n unsigned int objSz = getObjectSize(size);\n\n cleanBlockHeader();\n MALLOC_ASSERT(objSz <= USHRT_MAX, \"objSz must not be less 2^16-1\");\n objectSize = objSz;\n markOwned(tls);\n // bump pointer should be prepared for first allocation - thus mode it down to objectSize\n bumpPtr = (FreeObject *)((uintptr_t)this + slabSize - objectSize);\n\n // each block should have the address where the head of the list of \"privatizable\" blocks is kept\n // the only exception is a block for boot strap which is initialized when TLS is yet nullptr\n nextPrivatizable.store( tls? (Block*)(tls->bin + index) : nullptr, std::memory_order_relaxed);\n TRACEF(( \"[ScalableMalloc trace] Empty block %p is initialized, owner is %ld, objectSize is %d, bumpPtr is %p\\n\",\n this, tlsPtr.load(std::memory_order_relaxed) ? getThreadId() : -1, objectSize, bumpPtr ));\n}\n\nBlock *OrphanedBlocks::get(TLSData *tls, unsigned int size)\n{\n // TODO: try to use index from getAllocationBin\n unsigned int index = getIndex(size);\n Block *block = bins[index].pop();\n if (block) {\n MALLOC_ITT_SYNC_ACQUIRED(bins+index);\n block->privatizeOrphaned(tls, index);\n }\n return block;\n}\n\nvoid OrphanedBlocks::put(intptr_t binTag, Block *block)\n{\n unsigned int index = getIndex(block->getSize());\n block->shareOrphaned(binTag, index);\n MALLOC_ITT_SYNC_RELEASING(bins+index);\n bins[index].push(block);\n}\n\nvoid OrphanedBlocks::reset()\n{\n for (uint32_t i=0; inext;\n block->privatizePublicFreeList( /*reset=*/false ); // do not set publicFreeList to nullptr\n if (block->empty()) {\n block->reset();\n // slab blocks in user's pools do not have valid backRefIdx\n if (!backend->inUserPool())\n removeBackRef(*(block->getBackRefIdx()));\n backend->putSlabBlock(block);\n released = true;\n } else {\n MALLOC_ITT_SYNC_RELEASING(bins+i);\n bins[i].push(block);\n }\n block = next;\n }\n }\n return released;\n}\n\nFreeBlockPool::ResOfGet FreeBlockPool::getBlock()\n{\n Block *b = head.exchange(nullptr);\n bool lastAccessMiss;\n\n if (b) {\n size--;\n Block *newHead = b->next;\n lastAccessMiss = false;\n head.store(newHead, std::memory_order_release);\n } else {\n lastAccessMiss = true;\n }\n return ResOfGet(b, lastAccessMiss);\n}\n\nvoid FreeBlockPool::returnBlock(Block *block)\n{\n MALLOC_ASSERT( size <= POOL_HIGH_MARK, ASSERT_TEXT );\n Block *localHead = head.exchange(nullptr);\n\n if (!localHead) {\n size = 0; // head was stolen by externalClean, correct size accordingly\n } else if (size == POOL_HIGH_MARK) {\n // release cold blocks and add hot one,\n // so keep POOL_LOW_MARK-1 blocks and add new block to head\n Block *headToFree = localHead, *helper;\n for (int i=0; inext;\n Block *last = headToFree;\n headToFree = headToFree->next;\n last->next = nullptr;\n size = POOL_LOW_MARK-1;\n for (Block *currBl = headToFree; currBl; currBl = helper) {\n helper = currBl->next;\n // slab blocks in user's pools do not have valid backRefIdx\n if (!backend->inUserPool())\n removeBackRef(currBl->backRefIdx);\n backend->putSlabBlock(currBl);\n }\n }\n size++;\n block->next = localHead;\n head.store(block, std::memory_order_release);\n}\n\nbool FreeBlockPool::externalCleanup()\n{\n Block *helper;\n bool released = false;\n\n for (Block *currBl=head.exchange(nullptr); currBl; currBl=helper) {\n helper = currBl->next;\n // slab blocks in user's pools do not have valid backRefIdx\n if (!backend->inUserPool())\n removeBackRef(currBl->backRefIdx);\n backend->putSlabBlock(currBl);\n released = true;\n }\n return released;\n}\n\n/* Prepare the block for returning to FreeBlockPool */\nvoid Block::reset()\n{\n // it is caller's responsibility to ensure no data is lost before calling this\n MALLOC_ASSERT( allocatedCount==0, ASSERT_TEXT );\n MALLOC_ASSERT( !isSolidPtr(publicFreeList.load(std::memory_order_relaxed)), ASSERT_TEXT );\n if (!isStartupAllocObject())\n STAT_increment(getThreadId(), getIndex(objectSize), freeBlockBack);\n\n cleanBlockHeader();\n\n nextPrivatizable.store(nullptr, std::memory_order_relaxed);\n\n objectSize = 0;\n // for an empty block, bump pointer should point right after the end of the block\n bumpPtr = (FreeObject *)((uintptr_t)this + slabSize);\n}\n\ninline void Bin::setActiveBlock (Block *block)\n{\n// MALLOC_ASSERT( bin, ASSERT_TEXT );\n MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT );\n // it is the caller responsibility to keep bin consistence (i.e. ensure this block is in the bin list)\n activeBlk = block;\n}\n\ninline Block* Bin::setPreviousBlockActive()\n{\n MALLOC_ASSERT( activeBlk, ASSERT_TEXT );\n Block* temp = activeBlk->previous;\n if( temp ) {\n MALLOC_ASSERT( !(temp->isFull), ASSERT_TEXT );\n activeBlk = temp;\n }\n return temp;\n}\n\ninline bool Block::isOwnedByCurrentThread() const {\n return tlsPtr.load(std::memory_order_relaxed) && ownerTid.isCurrentThreadId();\n}\n\nFreeObject *Block::findObjectToFree(const void *object) const\n{\n FreeObject *objectToFree;\n // Due to aligned allocations, a pointer passed to scalable_free\n // might differ from the address of internally allocated object.\n // Small objects however should always be fine.\n if (objectSize <= maxSegregatedObjectSize)\n objectToFree = (FreeObject*)object;\n // \"Fitting size\" allocations are suspicious if aligned higher than naturally\n else {\n if ( ! isAligned(object,2*fittingAlignment) )\n // TODO: the above check is questionable - it gives false negatives in ~50% cases,\n // so might even be slower in average than unconditional use of findAllocatedObject.\n // here it should be a \"real\" object\n objectToFree = (FreeObject*)object;\n else\n // here object can be an aligned address, so applying additional checks\n objectToFree = findAllocatedObject(object);\n MALLOC_ASSERT( isAligned(objectToFree,fittingAlignment), ASSERT_TEXT );\n }\n MALLOC_ASSERT( isProperlyPlaced(objectToFree), ASSERT_TEXT );\n\n return objectToFree;\n}\n\nvoid TLSData::release()\n{\n memPool->extMemPool.allLocalCaches.unregisterThread(this);\n externalCleanup(/*cleanOnlyUnused=*/false, /*cleanBins=*/false);\n\n for (unsigned index = 0; index < numBlockBins; index++) {\n Block *activeBlk = bin[index].getActiveBlock();\n if (!activeBlk)\n continue;\n Block *threadlessBlock = activeBlk->previous;\n bool syncOnMailbox = false;\n while (threadlessBlock) {\n Block *threadBlock = threadlessBlock->previous;\n if (threadlessBlock->empty()) {\n /* we destroy the thread, so not use its block pool */\n memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false);\n } else {\n memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock);\n syncOnMailbox = true;\n }\n threadlessBlock = threadBlock;\n }\n threadlessBlock = activeBlk;\n while (threadlessBlock) {\n Block *threadBlock = threadlessBlock->next;\n if (threadlessBlock->empty()) {\n /* we destroy the thread, so not use its block pool */\n memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false);\n } else {\n memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock);\n syncOnMailbox = true;\n }\n threadlessBlock = threadBlock;\n }\n bin[index].resetActiveBlock();\n\n if (syncOnMailbox) {\n // Although, we synchronized on nextPrivatizable inside a block, we still need to\n // synchronize on the bin lifetime because the thread releasing an object into the public\n // free list is touching the bin (mailbox and mailLock)\n MallocMutex::scoped_lock scoped_cs(bin[index].mailLock);\n }\n }\n}\n\n\n#if MALLOC_CHECK_RECURSION\n// TODO: Use dedicated heap for this\n\n/*\n * It's a special kind of allocation that can be used when malloc is\n * not available (either during startup or when malloc was already called and\n * we are, say, inside pthread_setspecific's call).\n * Block can contain objects of different sizes,\n * allocations are performed by moving bump pointer and increasing of object counter,\n * releasing is done via counter of objects allocated in the block\n * or moving bump pointer if releasing object is on a bound.\n * TODO: make bump pointer to grow to the same backward direction as all the others.\n */\n\nclass StartupBlock : public Block {\n size_t availableSize() const {\n return slabSize - ((uintptr_t)bumpPtr - (uintptr_t)this);\n }\n static StartupBlock *getBlock();\npublic:\n static FreeObject *allocate(size_t size);\n static size_t msize(void *ptr) { return *((size_t*)ptr - 1); }\n void free(void *ptr);\n};\n\nstatic MallocMutex startupMallocLock;\nstatic StartupBlock *firstStartupBlock;\n\nStartupBlock *StartupBlock::getBlock()\n{\n BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/false);\n if (backRefIdx.isInvalid()) return nullptr;\n\n StartupBlock *block = static_cast(\n defaultMemPool->extMemPool.backend.getSlabBlock(1));\n if (!block) return nullptr;\n\n block->cleanBlockHeader();\n setBackRef(backRefIdx, block);\n block->backRefIdx = backRefIdx;\n // use startupAllocObjSizeMark to mark objects from startup block marker\n block->objectSize = startupAllocObjSizeMark;\n block->bumpPtr = (FreeObject *)((uintptr_t)block + sizeof(StartupBlock));\n return block;\n}\n\nFreeObject *StartupBlock::allocate(size_t size)\n{\n FreeObject *result;\n StartupBlock *newBlock = nullptr;\n\n /* Objects must be aligned on their natural bounds,\n and objects bigger than word on word's bound. */\n size = alignUp(size, sizeof(size_t));\n // We need size of an object to implement msize.\n size_t reqSize = size + sizeof(size_t);\n {\n MallocMutex::scoped_lock scoped_cs(startupMallocLock);\n // Re-check whether we need a new block (conditions might have changed)\n if (!firstStartupBlock || firstStartupBlock->availableSize() < reqSize) {\n if (!newBlock) {\n newBlock = StartupBlock::getBlock();\n if (!newBlock) return nullptr;\n }\n newBlock->next = (Block*)firstStartupBlock;\n if (firstStartupBlock)\n firstStartupBlock->previous = (Block*)newBlock;\n firstStartupBlock = newBlock;\n }\n result = firstStartupBlock->bumpPtr;\n firstStartupBlock->allocatedCount++;\n firstStartupBlock->bumpPtr =\n (FreeObject *)((uintptr_t)firstStartupBlock->bumpPtr + reqSize);\n }\n\n // keep object size at the negative offset\n *((size_t*)result) = size;\n return (FreeObject*)((size_t*)result+1);\n}\n\nvoid StartupBlock::free(void *ptr)\n{\n Block* blockToRelease = nullptr;\n {\n MallocMutex::scoped_lock scoped_cs(startupMallocLock);\n\n MALLOC_ASSERT(firstStartupBlock, ASSERT_TEXT);\n MALLOC_ASSERT(startupAllocObjSizeMark==objectSize\n && allocatedCount>0, ASSERT_TEXT);\n MALLOC_ASSERT((uintptr_t)ptr>=(uintptr_t)this+sizeof(StartupBlock)\n && (uintptr_t)ptr+StartupBlock::msize(ptr)<=(uintptr_t)this+slabSize,\n ASSERT_TEXT);\n if (0 == --allocatedCount) {\n if (this == firstStartupBlock)\n firstStartupBlock = (StartupBlock*)firstStartupBlock->next;\n if (previous)\n previous->next = next;\n if (next)\n next->previous = previous;\n blockToRelease = this;\n } else if ((uintptr_t)ptr + StartupBlock::msize(ptr) == (uintptr_t)bumpPtr) {\n // last object in the block released\n FreeObject *newBump = (FreeObject*)((size_t*)ptr - 1);\n MALLOC_ASSERT((uintptr_t)newBump>(uintptr_t)this+sizeof(StartupBlock),\n ASSERT_TEXT);\n bumpPtr = newBump;\n }\n }\n if (blockToRelease) {\n blockToRelease->previous = blockToRelease->next = nullptr;\n defaultMemPool->returnEmptyBlock(blockToRelease, /*poolTheBlock=*/false);\n }\n}\n\n#endif /* MALLOC_CHECK_RECURSION */\n\n/********* End thread related code *************/\n\n/********* Library initialization *************/\n\n//! Value indicating the state of initialization.\n/* 0 = initialization not started.\n * 1 = initialization started but not finished.\n * 2 = initialization finished.\n * In theory, we only need values 0 and 2. But value 1 is nonetheless\n * useful for detecting errors in the double-check pattern.\n */\nstatic std::atomic mallocInitialized{0}; // implicitly initialized to 0\nstatic MallocMutex initMutex;\n\n/** The leading \"\\0\" is here so that applying \"strings\" to the binary\n delivers a clean result. */\nstatic char VersionString[] = \"\\0\" TBBMALLOC_VERSION_STRINGS;\n\n#if USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED\n\n/* Decrease race interval between dynamic library unloading and pthread key\n destructor. Protect only Pthreads with supported unloading. */\nclass ShutdownSync {\n/* flag is the number of threads in pthread key dtor body\n (i.e., between threadDtorStart() and threadDtorDone())\n or the signal to skip dtor, if flag < 0 */\n std::atomic flag;\n static const intptr_t skipDtor = INTPTR_MIN/2;\npublic:\n void init() { flag.store(0, std::memory_order_release); }\n/* Suppose that 2*abs(skipDtor) or more threads never call threadDtorStart()\n simultaneously, so flag never becomes negative because of that. */\n bool threadDtorStart() {\n if (flag.load(std::memory_order_acquire) < 0)\n return false;\n if (++flag <= 0) { // note that new value returned\n flag.fetch_sub(1); // flag is spoiled by us, restore it\n return false;\n }\n return true;\n }\n void threadDtorDone() {\n flag.fetch_sub(1);\n }\n void processExit() {\n if (flag.fetch_add(skipDtor) != 0) {\n SpinWaitUntilEq(flag, skipDtor);\n }\n }\n};\n\n#else\n\nclass ShutdownSync {\npublic:\n void init() { }\n bool threadDtorStart() { return true; }\n void threadDtorDone() { }\n void processExit() { }\n};\n\n#endif // USE_PTHREAD && __TBB_SOURCE_DIRECTLY_INCLUDED\n\nstatic ShutdownSync shutdownSync;\n\ninline bool isMallocInitialized() {\n // Load must have acquire fence; otherwise thread taking \"initialized\" path\n // might perform textually later loads *before* mallocInitialized becomes 2.\n return 2 == mallocInitialized.load(std::memory_order_acquire);\n}\n\n/* Caller is responsible for ensuring this routine is called exactly once. */\nextern \"C\" void MallocInitializeITT() {\n#if __TBB_USE_ITT_NOTIFY\n if (!usedBySrcIncluded)\n tbb::detail::r1::__TBB_load_ittnotify();\n#endif\n}\n\nvoid MemoryPool::initDefaultPool() {\n hugePages.init();\n}\n\n/*\n * Allocator initialization routine;\n * it is called lazily on the very first scalable_malloc call.\n */\nstatic bool initMemoryManager()\n{\n TRACEF(( \"[ScalableMalloc trace] sizeof(Block) is %d (expected 128); sizeof(uintptr_t) is %d\\n\",\n sizeof(Block), sizeof(uintptr_t) ));\n MALLOC_ASSERT( 2*blockHeaderAlignment == sizeof(Block), ASSERT_TEXT );\n MALLOC_ASSERT( sizeof(FreeObject) == sizeof(void*), ASSERT_TEXT );\n MALLOC_ASSERT( isAligned(defaultMemPool, sizeof(intptr_t)),\n \"Memory pool must be void*-aligned for atomic to work over aligned arguments.\");\n\n#if USE_WINTHREAD\n const size_t granularity = 64*1024; // granulatity of VirtualAlloc\n#else\n // POSIX.1-2001-compliant way to get page size\n const size_t granularity = sysconf(_SC_PAGESIZE);\n#endif\n if (!defaultMemPool) {\n // Do not rely on static constructors and do the assignment in case\n // of library static section not initialized at this call yet.\n defaultMemPool = (MemoryPool*)defaultMemPool_space;\n }\n bool initOk = defaultMemPool->\n extMemPool.init(0, nullptr, nullptr, granularity,\n /*keepAllMemory=*/false, /*fixedPool=*/false);\n// TODO: extMemPool.init() to not allocate memory\n if (!initOk || !initBackRefMain(&defaultMemPool->extMemPool.backend) || !ThreadId::init())\n return false;\n MemoryPool::initDefaultPool();\n // init() is required iff initMemoryManager() is called\n // after mallocProcessShutdownNotification()\n shutdownSync.init();\n#if COLLECT_STATISTICS\n initStatisticsCollection();\n#endif\n return true;\n}\n\nstatic bool GetBoolEnvironmentVariable(const char* name) {\n return tbb::detail::r1::GetBoolEnvironmentVariable(name);\n}\n\n//! Ensures that initMemoryManager() is called once and only once.\n/** Does not return until initMemoryManager() has been completed by a thread.\n There is no need to call this routine if mallocInitialized==2 . */\nstatic bool doInitialization()\n{\n MallocMutex::scoped_lock lock( initMutex );\n if (mallocInitialized.load(std::memory_order_relaxed)!=2) {\n MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==0, ASSERT_TEXT );\n mallocInitialized.store(1, std::memory_order_relaxed);\n RecursiveMallocCallProtector scoped;\n if (!initMemoryManager()) {\n mallocInitialized.store(0, std::memory_order_relaxed); // restore and out\n return false;\n }\n#ifdef MALLOC_EXTRA_INITIALIZATION\n MALLOC_EXTRA_INITIALIZATION;\n#endif\n#if MALLOC_CHECK_RECURSION\n RecursiveMallocCallProtector::detectNaiveOverload();\n#endif\n MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==1, ASSERT_TEXT );\n // Store must have release fence, otherwise mallocInitialized==2\n // might become remotely visible before side effects of\n // initMemoryManager() become remotely visible.\n mallocInitialized.store(2, std::memory_order_release);\n if( GetBoolEnvironmentVariable(\"TBB_VERSION\") ) {\n fputs(VersionString+1,stderr);\n hugePages.printStatus();\n }\n }\n /* It can't be 0 or I would have initialized it */\n MALLOC_ASSERT( mallocInitialized.load(std::memory_order_relaxed)==2, ASSERT_TEXT );\n return true;\n}\n\n/********* End library initialization *************/\n\n/********* The malloc show begins *************/\n\n\nFreeObject *Block::allocateFromFreeList()\n{\n FreeObject *result;\n\n if (!freeList) return nullptr;\n\n result = freeList;\n MALLOC_ASSERT( result, ASSERT_TEXT );\n\n freeList = result->next;\n MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT );\n allocatedCount++;\n STAT_increment(getThreadId(), getIndex(objectSize), allocFreeListUsed);\n\n return result;\n}\n\nFreeObject *Block::allocateFromBumpPtr()\n{\n FreeObject *result = bumpPtr;\n if (result) {\n bumpPtr = (FreeObject *) ((uintptr_t) bumpPtr - objectSize);\n if ( (uintptr_t)bumpPtr < (uintptr_t)this+sizeof(Block) ) {\n bumpPtr = nullptr;\n }\n MALLOC_ASSERT( allocatedCount < (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT );\n allocatedCount++;\n STAT_increment(getThreadId(), getIndex(objectSize), allocBumpPtrUsed);\n }\n return result;\n}\n\ninline FreeObject* Block::allocate()\n{\n MALLOC_ASSERT( isOwnedByCurrentThread(), ASSERT_TEXT );\n\n /* for better cache locality, first looking in the free list. */\n if ( FreeObject *result = allocateFromFreeList() ) {\n return result;\n }\n MALLOC_ASSERT( !freeList, ASSERT_TEXT );\n\n /* if free list is empty, try thread local bump pointer allocation. */\n if ( FreeObject *result = allocateFromBumpPtr() ) {\n return result;\n }\n MALLOC_ASSERT( !bumpPtr, ASSERT_TEXT );\n\n /* the block is considered full. */\n isFull = true;\n return nullptr;\n}\n\nsize_t Block::findObjectSize(void *object) const\n{\n size_t blSize = getSize();\n#if MALLOC_CHECK_RECURSION\n // Currently, there is no aligned allocations from startup blocks,\n // so we can return just StartupBlock::msize().\n // TODO: This must be extended if we add aligned allocation from startup blocks.\n if (!blSize)\n return StartupBlock::msize(object);\n#endif\n // object can be aligned, so real size can be less than block's\n size_t size =\n blSize - ((uintptr_t)object - (uintptr_t)findObjectToFree(object));\n MALLOC_ASSERT(size>0 && sizegetMemPool()->returnEmptyBlock(block, poolTheBlock);\n } else {\n /* all objects are free - let's restore the bump pointer */\n block->restoreBumpPtr();\n }\n}\n\ntemplate\nbool LocalLOCImpl::put(LargeMemoryBlock *object, ExtMemoryPool *extMemPool)\n{\n const size_t size = object->unalignedSize;\n // not spoil cache with too large object, that can cause its total cleanup\n if (size > MAX_TOTAL_SIZE)\n return false;\n LargeMemoryBlock *localHead = head.exchange(nullptr);\n\n object->prev = nullptr;\n object->next = localHead;\n if (localHead)\n localHead->prev = object;\n else {\n // those might not be cleaned during local cache stealing, correct them\n totalSize = 0;\n numOfBlocks = 0;\n tail = object;\n }\n localHead = object;\n totalSize += size;\n numOfBlocks++;\n // must meet both size and number of cached objects constrains\n if (totalSize > MAX_TOTAL_SIZE || numOfBlocks >= HIGH_MARK) {\n // scanning from tail until meet conditions\n while (totalSize > MAX_TOTAL_SIZE || numOfBlocks > LOW_MARK) {\n totalSize -= tail->unalignedSize;\n numOfBlocks--;\n tail = tail->prev;\n }\n LargeMemoryBlock *headToRelease = tail->next;\n tail->next = nullptr;\n\n extMemPool->freeLargeObjectList(headToRelease);\n }\n\n head.store(localHead, std::memory_order_release);\n return true;\n}\n\ntemplate\nLargeMemoryBlock *LocalLOCImpl::get(size_t size)\n{\n LargeMemoryBlock *localHead, *res = nullptr;\n\n if (size > MAX_TOTAL_SIZE)\n return nullptr;\n\n // TBB_REVAMP_TODO: review this line\n if (!head.load(std::memory_order_acquire) || (localHead = head.exchange(nullptr)) == nullptr) {\n // do not restore totalSize, numOfBlocks and tail at this point,\n // as they are used only in put(), where they must be restored\n return nullptr;\n }\n\n for (LargeMemoryBlock *curr = localHead; curr; curr=curr->next) {\n if (curr->unalignedSize == size) {\n res = curr;\n if (curr->next)\n curr->next->prev = curr->prev;\n else\n tail = curr->prev;\n if (curr != localHead)\n curr->prev->next = curr->next;\n else\n localHead = curr->next;\n totalSize -= size;\n numOfBlocks--;\n break;\n }\n }\n\n head.store(localHead, std::memory_order_release);\n return res;\n}\n\ntemplate\nbool LocalLOCImpl::externalCleanup(ExtMemoryPool *extMemPool)\n{\n if (LargeMemoryBlock *localHead = head.exchange(nullptr)) {\n extMemPool->freeLargeObjectList(localHead);\n return true;\n }\n return false;\n}\n\nvoid *MemoryPool::getFromLLOCache(TLSData* tls, size_t size, size_t alignment)\n{\n LargeMemoryBlock *lmb = nullptr;\n\n size_t headersSize = sizeof(LargeMemoryBlock)+sizeof(LargeObjectHdr);\n size_t allocationSize = LargeObjectCache::alignToBin(size+headersSize+alignment);\n if (allocationSize < size) // allocationSize is wrapped around after alignToBin\n return nullptr;\n MALLOC_ASSERT(allocationSize >= alignment, \"Overflow must be checked before.\");\n\n if (tls) {\n tls->markUsed();\n lmb = tls->lloc.get(allocationSize);\n }\n if (!lmb)\n lmb = extMemPool.mallocLargeObject(this, allocationSize);\n\n if (lmb) {\n // doing shuffle we suppose that alignment offset guarantees\n // that different cache lines are in use\n MALLOC_ASSERT(alignment >= estimatedCacheLineSize, ASSERT_TEXT);\n\n void *alignedArea = (void*)alignUp((uintptr_t)lmb+headersSize, alignment);\n uintptr_t alignedRight =\n alignDown((uintptr_t)lmb+lmb->unalignedSize - size, alignment);\n // Has some room to shuffle object between cache lines?\n // Note that alignedRight and alignedArea are aligned at alignment.\n unsigned ptrDelta = alignedRight - (uintptr_t)alignedArea;\n if (ptrDelta && tls) { // !tls is cold path\n // for the hot path of alignment==estimatedCacheLineSize,\n // allow compilers to use shift for division\n // (since estimatedCacheLineSize is a power-of-2 constant)\n unsigned numOfPossibleOffsets = alignment == estimatedCacheLineSize?\n ptrDelta / estimatedCacheLineSize :\n ptrDelta / alignment;\n unsigned myCacheIdx = ++tls->currCacheIdx;\n unsigned offset = myCacheIdx % numOfPossibleOffsets;\n\n // Move object to a cache line with an offset that is different from\n // previous allocation. This supposedly allows us to use cache\n // associativity more efficiently.\n alignedArea = (void*)((uintptr_t)alignedArea + offset*alignment);\n }\n MALLOC_ASSERT((uintptr_t)lmb+lmb->unalignedSize >=\n (uintptr_t)alignedArea+size, \"Object doesn't fit the block.\");\n LargeObjectHdr *header = (LargeObjectHdr*)alignedArea-1;\n header->memoryBlock = lmb;\n header->backRefIdx = lmb->backRefIdx;\n setBackRef(header->backRefIdx, header);\n\n lmb->objectSize = size;\n\n MALLOC_ASSERT( isLargeObject(alignedArea), ASSERT_TEXT );\n MALLOC_ASSERT( isAligned(alignedArea, alignment), ASSERT_TEXT );\n\n return alignedArea;\n }\n return nullptr;\n}\n\nvoid MemoryPool::putToLLOCache(TLSData *tls, void *object)\n{\n LargeObjectHdr *header = (LargeObjectHdr*)object - 1;\n // overwrite backRefIdx to simplify double free detection\n header->backRefIdx = BackRefIdx();\n\n if (tls) {\n tls->markUsed();\n if (tls->lloc.put(header->memoryBlock, &extMemPool))\n return;\n }\n extMemPool.freeLargeObject(header->memoryBlock);\n}\n\n/*\n * All aligned allocations fall into one of the following categories:\n * 1. if both request size and alignment are <= maxSegregatedObjectSize,\n * we just align the size up, and request this amount, because for every size\n * aligned to some power of 2, the allocated object is at least that aligned.\n * 2. for sizegetTLS(/*create=*/true);\n // take into account only alignment that are higher then natural\n result =\n memPool->getFromLLOCache(tls, size, largeObjectAlignment>alignment?\n largeObjectAlignment: alignment);\n }\n\n MALLOC_ASSERT( isAligned(result, alignment), ASSERT_TEXT );\n return result;\n}\n\nstatic void *reallocAligned(MemoryPool *memPool, void *ptr,\n size_t newSize, size_t alignment = 0)\n{\n void *result;\n size_t copySize;\n\n if (isLargeObject(ptr)) {\n LargeMemoryBlock* lmb = ((LargeObjectHdr *)ptr - 1)->memoryBlock;\n copySize = lmb->unalignedSize-((uintptr_t)ptr-(uintptr_t)lmb);\n\n // Apply different strategies if size decreases\n if (newSize <= copySize && (0 == alignment || isAligned(ptr, alignment))) {\n\n // For huge objects (that do not fit in backend cache), keep the same space unless\n // the new size is at least twice smaller\n bool isMemoryBlockHuge = copySize > memPool->extMemPool.backend.getMaxBinnedSize();\n size_t threshold = isMemoryBlockHuge ? copySize / 2 : 0;\n if (newSize > threshold) {\n lmb->objectSize = newSize;\n return ptr;\n }\n // TODO: For large objects suitable for the backend cache,\n // split out the excessive part and put it to the backend.\n }\n // Reallocate for real\n copySize = lmb->objectSize;\n#if BACKEND_HAS_MREMAP\n if (void *r = memPool->extMemPool.remap(ptr, copySize, newSize,\n alignment < largeObjectAlignment ? largeObjectAlignment : alignment))\n return r;\n#endif\n result = alignment ? allocateAligned(memPool, newSize, alignment) :\n internalPoolMalloc(memPool, newSize);\n\n } else {\n Block* block = (Block *)alignDown(ptr, slabSize);\n copySize = block->findObjectSize(ptr);\n\n // TODO: Move object to another bin if size decreases and the current bin is \"empty enough\".\n // Currently, in case of size decreasing, old pointer is returned\n if (newSize <= copySize && (0==alignment || isAligned(ptr, alignment))) {\n return ptr;\n } else {\n result = alignment ? allocateAligned(memPool, newSize, alignment) :\n internalPoolMalloc(memPool, newSize);\n }\n }\n if (result) {\n memcpy(result, ptr, copySize < newSize ? copySize : newSize);\n internalPoolFree(memPool, ptr, 0);\n }\n return result;\n}\n\n#if MALLOC_DEBUG\n/* A predicate checks if an object is properly placed inside its block */\ninline bool Block::isProperlyPlaced(const void *object) const\n{\n return 0 == ((uintptr_t)this + slabSize - (uintptr_t)object) % objectSize;\n}\n#endif\n\n/* Finds the real object inside the block */\nFreeObject *Block::findAllocatedObject(const void *address) const\n{\n // calculate offset from the end of the block space\n uint16_t offset = (uintptr_t)this + slabSize - (uintptr_t)address;\n MALLOC_ASSERT( offset<=slabSize-sizeof(Block), ASSERT_TEXT );\n // find offset difference from a multiple of allocation size\n offset %= objectSize;\n // and move the address down to where the real object starts.\n return (FreeObject*)((uintptr_t)address - (offset? objectSize-offset: 0));\n}\n\n/*\n * Bad dereference caused by a foreign pointer is possible only here, not earlier in call chain.\n * Separate function isolates SEH code, as it has bad influence on compiler optimization.\n */\nstatic inline BackRefIdx safer_dereference (const BackRefIdx *ptr)\n{\n BackRefIdx id;\n#if _MSC_VER\n __try {\n#endif\n id = dereference(ptr);\n#if _MSC_VER\n } __except( GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION?\n EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH ) {\n id = BackRefIdx();\n }\n#endif\n return id;\n}\n\ntemplate\nbool isLargeObject(void *object)\n{\n if (!isAligned(object, largeObjectAlignment))\n return false;\n LargeObjectHdr *header = (LargeObjectHdr*)object - 1;\n BackRefIdx idx = (memOrigin == unknownMem) ?\n safer_dereference(&header->backRefIdx) : dereference(&header->backRefIdx);\n\n return idx.isLargeObject()\n // in valid LargeObjectHdr memoryBlock is not nullptr\n && header->memoryBlock\n // in valid LargeObjectHdr memoryBlock points somewhere before header\n // TODO: more strict check\n && (uintptr_t)header->memoryBlock < (uintptr_t)header\n && getBackRef(idx) == header;\n}\n\nstatic inline bool isSmallObject (void *ptr)\n{\n Block* expectedBlock = (Block*)alignDown(ptr, slabSize);\n const BackRefIdx* idx = expectedBlock->getBackRefIdx();\n\n bool isSmall = expectedBlock == getBackRef(safer_dereference(idx));\n if (isSmall)\n expectedBlock->checkFreePrecond(ptr);\n return isSmall;\n}\n\n/**** Check if an object was allocated by scalable_malloc ****/\nstatic inline bool isRecognized (void* ptr)\n{\n return defaultMemPool->extMemPool.backend.ptrCanBeValid(ptr) &&\n (isLargeObject(ptr) || isSmallObject(ptr));\n}\n\nstatic inline void freeSmallObject(void *object)\n{\n /* mask low bits to get the block */\n Block *block = (Block *)alignDown(object, slabSize);\n block->checkFreePrecond(object);\n\n#if MALLOC_CHECK_RECURSION\n if (block->isStartupAllocObject()) {\n ((StartupBlock *)block)->free(object);\n return;\n }\n#endif\n if (block->isOwnedByCurrentThread()) {\n block->freeOwnObject(object);\n } else { /* Slower path to add to the shared list, the allocatedCount is updated by the owner thread in malloc. */\n FreeObject *objectToFree = block->findObjectToFree(object);\n block->freePublicObject(objectToFree);\n }\n}\n\nstatic void *internalPoolMalloc(MemoryPool* memPool, size_t size)\n{\n Bin* bin;\n Block * mallocBlock;\n\n if (!memPool) return nullptr;\n\n if (!size) size = sizeof(size_t);\n\n TLSData *tls = memPool->getTLS(/*create=*/true);\n\n /* Allocate a large object */\n if (size >= minLargeObjectSize)\n return memPool->getFromLLOCache(tls, size, largeObjectAlignment);\n\n if (!tls) return nullptr;\n\n tls->markUsed();\n /*\n * Get an element in thread-local array corresponding to the given size;\n * It keeps ptr to the active block for allocations of this size\n */\n bin = tls->getAllocationBin(size);\n if ( !bin ) return nullptr;\n\n /* Get a block to try to allocate in. */\n for( mallocBlock = bin->getActiveBlock(); mallocBlock;\n mallocBlock = bin->setPreviousBlockActive() ) // the previous block should be empty enough\n {\n if( FreeObject *result = mallocBlock->allocate() )\n return result;\n }\n\n /*\n * else privatize publicly freed objects in some block and allocate from it\n */\n mallocBlock = bin->getPrivatizedFreeListBlock();\n if (mallocBlock) {\n MALLOC_ASSERT( mallocBlock->freeListNonNull(), ASSERT_TEXT );\n if ( FreeObject *result = mallocBlock->allocateFromFreeList() )\n return result;\n /* Else something strange happened, need to retry from the beginning; */\n TRACEF(( \"[ScalableMalloc trace] Something is wrong: no objects in public free list; reentering.\\n\" ));\n return internalPoolMalloc(memPool, size);\n }\n\n /*\n * no suitable own blocks, try to get a partial block that some other thread has discarded.\n */\n mallocBlock = memPool->extMemPool.orphanedBlocks.get(tls, size);\n while (mallocBlock) {\n bin->pushTLSBin(mallocBlock);\n bin->setActiveBlock(mallocBlock); // TODO: move under the below condition?\n if( FreeObject *result = mallocBlock->allocate() )\n return result;\n mallocBlock = memPool->extMemPool.orphanedBlocks.get(tls, size);\n }\n\n /*\n * else try to get a new empty block\n */\n mallocBlock = memPool->getEmptyBlock(size);\n if (mallocBlock) {\n bin->pushTLSBin(mallocBlock);\n bin->setActiveBlock(mallocBlock);\n if( FreeObject *result = mallocBlock->allocate() )\n return result;\n /* Else something strange happened, need to retry from the beginning; */\n TRACEF(( \"[ScalableMalloc trace] Something is wrong: no objects in empty block; reentering.\\n\" ));\n return internalPoolMalloc(memPool, size);\n }\n /*\n * else nothing works so return nullptr\n */\n TRACEF(( \"[ScalableMalloc trace] No memory found, returning nullptr.\\n\" ));\n return nullptr;\n}\n\n// When size==0 (i.e. unknown), detect here whether the object is large.\n// For size is known and < minLargeObjectSize, we still need to check\n// if the actual object is large, because large objects might be used\n// for aligned small allocations.\nstatic bool internalPoolFree(MemoryPool *memPool, void *object, size_t size)\n{\n if (!memPool || !object) return false;\n\n // The library is initialized at allocation call, so releasing while\n // not initialized means foreign object is releasing.\n MALLOC_ASSERT(isMallocInitialized(), ASSERT_TEXT);\n MALLOC_ASSERT(memPool->extMemPool.userPool() || isRecognized(object),\n \"Invalid pointer during object releasing is detected.\");\n\n if (size >= minLargeObjectSize || isLargeObject(object))\n memPool->putToLLOCache(memPool->getTLS(/*create=*/false), object);\n else\n freeSmallObject(object);\n return true;\n}\n\nstatic void *internalMalloc(size_t size)\n{\n if (!size) size = sizeof(size_t);\n\n#if MALLOC_CHECK_RECURSION\n if (RecursiveMallocCallProtector::sameThreadActive())\n return sizegetFromLLOCache(nullptr, size, slabSize);\n#endif\n\n if (!isMallocInitialized())\n if (!doInitialization())\n return nullptr;\n return internalPoolMalloc(defaultMemPool, size);\n}\n\nstatic void internalFree(void *object)\n{\n internalPoolFree(defaultMemPool, object, 0);\n}\n\nstatic size_t internalMsize(void* ptr)\n{\n MALLOC_ASSERT(ptr, \"Invalid pointer passed to internalMsize\");\n if (isLargeObject(ptr)) {\n // TODO: return the maximum memory size, that can be written to this object\n LargeMemoryBlock* lmb = ((LargeObjectHdr*)ptr - 1)->memoryBlock;\n return lmb->objectSize;\n } else {\n Block *block = (Block*)alignDown(ptr, slabSize);\n return block->findObjectSize(ptr);\n }\n}\n\n} // namespace internal\n\nusing namespace rml::internal;\n\n// legacy entry point saved for compatibility with binaries complied\n// with pre-6003 versions of TBB\nTBBMALLOC_EXPORT rml::MemoryPool *pool_create(intptr_t pool_id, const MemPoolPolicy *policy)\n{\n rml::MemoryPool *pool;\n MemPoolPolicy pol(policy->pAlloc, policy->pFree, policy->granularity);\n\n pool_create_v1(pool_id, &pol, &pool);\n return pool;\n}\n\nrml::MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy,\n rml::MemoryPool **pool)\n{\n if ( !policy->pAlloc || policy->versionfixedPool || policy->pFree)) {\n *pool = nullptr;\n return INVALID_POLICY;\n }\n if ( policy->version>MemPoolPolicy::TBBMALLOC_POOL_VERSION // future versions are not supported\n // new flags can be added in place of reserved, but default\n // behaviour must be supported by this version\n || policy->reserved ) {\n *pool = nullptr;\n return UNSUPPORTED_POLICY;\n }\n if (!isMallocInitialized())\n if (!doInitialization()) {\n *pool = nullptr;\n return NO_MEMORY;\n }\n rml::internal::MemoryPool *memPool =\n (rml::internal::MemoryPool*)internalMalloc((sizeof(rml::internal::MemoryPool)));\n if (!memPool) {\n *pool = nullptr;\n return NO_MEMORY;\n }\n memset(static_cast(memPool), 0, sizeof(rml::internal::MemoryPool));\n if (!memPool->init(pool_id, policy)) {\n internalFree(memPool);\n *pool = nullptr;\n return NO_MEMORY;\n }\n\n *pool = (rml::MemoryPool*)memPool;\n return POOL_OK;\n}\n\nbool pool_destroy(rml::MemoryPool* memPool)\n{\n if (!memPool) return false;\n bool ret = ((rml::internal::MemoryPool*)memPool)->destroy();\n internalFree(memPool);\n\n return ret;\n}\n\nbool pool_reset(rml::MemoryPool* memPool)\n{\n if (!memPool) return false;\n\n return ((rml::internal::MemoryPool*)memPool)->reset();\n}\n\nvoid *pool_malloc(rml::MemoryPool* mPool, size_t size)\n{\n return internalPoolMalloc((rml::internal::MemoryPool*)mPool, size);\n}\n\nvoid *pool_realloc(rml::MemoryPool* mPool, void *object, size_t size)\n{\n if (!object)\n return internalPoolMalloc((rml::internal::MemoryPool*)mPool, size);\n if (!size) {\n internalPoolFree((rml::internal::MemoryPool*)mPool, object, 0);\n return nullptr;\n }\n return reallocAligned((rml::internal::MemoryPool*)mPool, object, size, 0);\n}\n\nvoid *pool_aligned_malloc(rml::MemoryPool* mPool, size_t size, size_t alignment)\n{\n if (!isPowerOfTwo(alignment) || 0==size)\n return nullptr;\n\n return allocateAligned((rml::internal::MemoryPool*)mPool, size, alignment);\n}\n\nvoid *pool_aligned_realloc(rml::MemoryPool* memPool, void *ptr, size_t size, size_t alignment)\n{\n if (!isPowerOfTwo(alignment))\n return nullptr;\n rml::internal::MemoryPool *mPool = (rml::internal::MemoryPool*)memPool;\n void *tmp;\n\n if (!ptr)\n tmp = allocateAligned(mPool, size, alignment);\n else if (!size) {\n internalPoolFree(mPool, ptr, 0);\n return nullptr;\n } else\n tmp = reallocAligned(mPool, ptr, size, alignment);\n\n return tmp;\n}\n\nbool pool_free(rml::MemoryPool *mPool, void *object)\n{\n return internalPoolFree((rml::internal::MemoryPool*)mPool, object, 0);\n}\n\nrml::MemoryPool *pool_identify(void *object)\n{\n rml::internal::MemoryPool *pool;\n if (isLargeObject(object)) {\n LargeObjectHdr *header = (LargeObjectHdr*)object - 1;\n pool = header->memoryBlock->pool;\n } else {\n Block *block = (Block*)alignDown(object, slabSize);\n pool = block->getMemPool();\n }\n // do not return defaultMemPool, as it can't be used in pool_free() etc\n __TBB_ASSERT_RELEASE(pool!=defaultMemPool,\n \"rml::pool_identify() can't be used for scalable_malloc() etc results.\");\n return (rml::MemoryPool*)pool;\n}\n\nsize_t pool_msize(rml::MemoryPool *mPool, void* object)\n{\n if (object) {\n // No assert for object recognition, cause objects allocated from non-default\n // memory pool do not participate in range checking and do not have valid backreferences for\n // small objects. Instead, check that an object belong to the certain memory pool.\n MALLOC_ASSERT_EX(mPool == pool_identify(object), \"Object does not belong to the specified pool\");\n return internalMsize(object);\n }\n errno = EINVAL;\n // Unlike _msize, return 0 in case of parameter error.\n // Returning size_t(-1) looks more like the way to troubles.\n return 0;\n}\n\n} // namespace rml\n\nusing namespace rml::internal;\n\n#if MALLOC_TRACE\nstatic unsigned int threadGoingDownCount = 0;\n#endif\n\n/*\n * When a thread is shutting down this routine should be called to remove all the thread ids\n * from the malloc blocks and replace them with a nullptr thread id.\n *\n * For pthreads, the function is set as a callback in pthread_key_create for TLS bin.\n * It will be automatically called at thread exit with the key value as the argument,\n * unless that value is nullptr.\n * For Windows, it is called from DllMain( DLL_THREAD_DETACH ).\n *\n * However neither of the above is called for the main process thread, so the routine\n * also needs to be called during the process shutdown.\n *\n*/\n// TODO: Consider making this function part of class MemoryPool.\nvoid doThreadShutdownNotification(TLSData* tls, bool main_thread)\n{\n TRACEF(( \"[ScalableMalloc trace] Thread id %d blocks return start %d\\n\",\n getThreadId(), threadGoingDownCount++ ));\n\n#if USE_PTHREAD\n if (tls) {\n if (!shutdownSync.threadDtorStart()) return;\n tls->getMemPool()->onThreadShutdown(tls);\n shutdownSync.threadDtorDone();\n } else\n#endif\n {\n suppress_unused_warning(tls); // not used on Windows\n // The default pool is safe to use at this point:\n // on Linux, only the main thread can go here before destroying defaultMemPool;\n // on Windows, shutdown is synchronized via loader lock and isMallocInitialized().\n // See also __TBB_mallocProcessShutdownNotification()\n defaultMemPool->onThreadShutdown(defaultMemPool->getTLS(/*create=*/false));\n // Take lock to walk through other pools; but waiting might be dangerous at this point\n // (e.g. on Windows the main thread might deadlock)\n bool locked = false;\n MallocMutex::scoped_lock lock(MemoryPool::memPoolListLock, /*wait=*/!main_thread, &locked);\n if (locked) { // the list is safe to process\n for (MemoryPool *memPool = defaultMemPool->next; memPool; memPool = memPool->next)\n memPool->onThreadShutdown(memPool->getTLS(/*create=*/false));\n }\n }\n\n TRACEF(( \"[ScalableMalloc trace] Thread id %d blocks return end\\n\", getThreadId() ));\n}\n\n#if USE_PTHREAD\nvoid mallocThreadShutdownNotification(void* arg)\n{\n // The routine is called for each pool (as TLS dtor) on each thread, except for the main thread\n if (!isMallocInitialized()) return;\n doThreadShutdownNotification((TLSData*)arg, false);\n}\n#else\nextern \"C\" void __TBB_mallocThreadShutdownNotification()\n{\n // The routine is called once per thread on Windows\n if (!isMallocInitialized()) return;\n doThreadShutdownNotification(nullptr, false);\n}\n#endif\n\nextern \"C\" void __TBB_mallocProcessShutdownNotification(bool windows_process_dying)\n{\n if (!isMallocInitialized()) return;\n\n // Don't clean allocator internals if the entire process is exiting\n if (!windows_process_dying) {\n doThreadShutdownNotification(nullptr, /*main_thread=*/true);\n }\n#if __TBB_MALLOC_LOCACHE_STAT\n printf(\"cache hit ratio %f, size hit %f\\n\",\n 1.*cacheHits/mallocCalls, 1.*memHitKB/memAllocKB);\n defaultMemPool->extMemPool.loc.reportStat(stdout);\n#endif\n\n shutdownSync.processExit();\n#if __TBB_SOURCE_DIRECTLY_INCLUDED\n/* Pthread keys must be deleted as soon as possible to not call key dtor\n on thread termination when then the tbbmalloc code can be already unloaded.\n*/\n defaultMemPool->destroy();\n destroyBackRefMain(&defaultMemPool->extMemPool.backend);\n ThreadId::destroy(); // Delete key for thread id\n hugePages.reset();\n // new total malloc initialization is possible after this point\n mallocInitialized.store(0, std::memory_order_release);\n#endif // __TBB_SOURCE_DIRECTLY_INCLUDED\n\n#if COLLECT_STATISTICS\n unsigned nThreads = ThreadId::getMaxThreadId();\n for( int i=1; i<=nThreads && iextMemPool.backend.ptrCanBeValid(object)) {\n if (isLargeObject(object)) {\n // must check 1st for large object, because small object check touches 4 pages on left,\n // and it can be inaccessible\n TLSData *tls = defaultMemPool->getTLS(/*create=*/false);\n\n defaultMemPool->putToLLOCache(tls, object);\n return;\n } else if (isSmallObject(object)) {\n freeSmallObject(object);\n return;\n }\n }\n if (original_free)\n original_free(object);\n}\n\n/********* End the free code *************/\n\n/********* Code for scalable_realloc ***********/\n\n/*\n * From K&R\n * \"realloc changes the size of the object pointed to by p to size. The contents will\n * be unchanged up to the minimum of the old and the new sizes. If the new size is larger,\n * the new space is uninitialized. realloc returns a pointer to the new space, or\n * nullptr if the request cannot be satisfied, in which case *p is unchanged.\"\n *\n */\nextern \"C\" void* scalable_realloc(void* ptr, size_t size)\n{\n void *tmp;\n\n if (!ptr)\n tmp = internalMalloc(size);\n else if (!size) {\n internalFree(ptr);\n return nullptr;\n } else\n tmp = reallocAligned(defaultMemPool, ptr, size, 0);\n\n if (!tmp) errno = ENOMEM;\n return tmp;\n}\n\n/*\n * A variant that provides additional memory safety, by checking whether the given address\n * was obtained with this allocator, and if not redirecting to the provided alternative call.\n */\nextern \"C\" TBBMALLOC_EXPORT void* __TBB_malloc_safer_realloc(void* ptr, size_t sz, void* original_realloc)\n{\n void *tmp; // TODO: fix warnings about uninitialized use of tmp\n\n if (!ptr) {\n tmp = internalMalloc(sz);\n } else if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(ptr)) {\n if (!sz) {\n internalFree(ptr);\n return nullptr;\n } else {\n tmp = reallocAligned(defaultMemPool, ptr, sz, 0);\n }\n }\n#if USE_WINTHREAD\n else if (original_realloc && sz) {\n orig_ptrs *original_ptrs = static_cast(original_realloc);\n if ( original_ptrs->msize ){\n size_t oldSize = original_ptrs->msize(ptr);\n tmp = internalMalloc(sz);\n if (tmp) {\n memcpy(tmp, ptr, szfree ){\n original_ptrs->free( ptr );\n }\n }\n } else\n tmp = nullptr;\n }\n#else\n else if (original_realloc) {\n typedef void* (*realloc_ptr_t)(void*,size_t);\n realloc_ptr_t original_realloc_ptr;\n (void *&)original_realloc_ptr = original_realloc;\n tmp = original_realloc_ptr(ptr,sz);\n }\n#endif\n else tmp = nullptr;\n\n if (!tmp) errno = ENOMEM;\n return tmp;\n}\n\n/********* End code for scalable_realloc ***********/\n\n/********* Code for scalable_calloc ***********/\n\n/*\n * From K&R\n * calloc returns a pointer to space for an array of nobj objects,\n * each of size size, or nullptr if the request cannot be satisfied.\n * The space is initialized to zero bytes.\n *\n */\n\nextern \"C\" void * scalable_calloc(size_t nobj, size_t size)\n{\n // it's square root of maximal size_t value\n const size_t mult_not_overflow = size_t(1) << (sizeof(size_t)*CHAR_BIT/2);\n const size_t arraySize = nobj * size;\n\n // check for overflow during multiplication:\n if (nobj>=mult_not_overflow || size>=mult_not_overflow) // 1) heuristic check\n if (nobj && arraySize / nobj != size) { // 2) exact check\n errno = ENOMEM;\n return nullptr;\n }\n void* result = internalMalloc(arraySize);\n if (result)\n memset(result, 0, arraySize);\n else\n errno = ENOMEM;\n return result;\n}\n\n/********* End code for scalable_calloc ***********/\n\n/********* Code for aligned allocation API **********/\n\nextern \"C\" int scalable_posix_memalign(void **memptr, size_t alignment, size_t size)\n{\n if ( !isPowerOfTwoAtLeast(alignment, sizeof(void*)) )\n return EINVAL;\n void *result = allocateAligned(defaultMemPool, size, alignment);\n if (!result)\n return ENOMEM;\n *memptr = result;\n return 0;\n}\n\nextern \"C\" void * scalable_aligned_malloc(size_t size, size_t alignment)\n{\n if (!isPowerOfTwo(alignment) || 0==size) {\n errno = EINVAL;\n return nullptr;\n }\n void *tmp = allocateAligned(defaultMemPool, size, alignment);\n if (!tmp) errno = ENOMEM;\n return tmp;\n}\n\nextern \"C\" void * scalable_aligned_realloc(void *ptr, size_t size, size_t alignment)\n{\n if (!isPowerOfTwo(alignment)) {\n errno = EINVAL;\n return nullptr;\n }\n void *tmp;\n\n if (!ptr)\n tmp = allocateAligned(defaultMemPool, size, alignment);\n else if (!size) {\n scalable_free(ptr);\n return nullptr;\n } else\n tmp = reallocAligned(defaultMemPool, ptr, size, alignment);\n\n if (!tmp) errno = ENOMEM;\n return tmp;\n}\n\nextern \"C\" TBBMALLOC_EXPORT void * __TBB_malloc_safer_aligned_realloc(void *ptr, size_t size, size_t alignment, void* orig_function)\n{\n /* corner cases left out of reallocAligned to not deal with errno there */\n if (!isPowerOfTwo(alignment)) {\n errno = EINVAL;\n return nullptr;\n }\n void *tmp = nullptr;\n\n if (!ptr) {\n tmp = allocateAligned(defaultMemPool, size, alignment);\n } else if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(ptr)) {\n if (!size) {\n internalFree(ptr);\n return nullptr;\n } else {\n tmp = reallocAligned(defaultMemPool, ptr, size, alignment);\n }\n }\n#if USE_WINTHREAD\n else {\n orig_aligned_ptrs *original_ptrs = static_cast(orig_function);\n if (size) {\n // Without orig_msize, we can't do anything with this.\n // Just keeping old pointer.\n if ( original_ptrs->aligned_msize ){\n // set alignment and offset to have possibly correct oldSize\n size_t oldSize = original_ptrs->aligned_msize(ptr, sizeof(void*), 0);\n tmp = allocateAligned(defaultMemPool, size, alignment);\n if (tmp) {\n memcpy(tmp, ptr, sizealigned_free ){\n original_ptrs->aligned_free( ptr );\n }\n }\n }\n } else {\n if ( original_ptrs->aligned_free ){\n original_ptrs->aligned_free( ptr );\n }\n return nullptr;\n }\n }\n#else\n // As original_realloc can't align result, and there is no way to find\n // size of reallocating object, we are giving up.\n suppress_unused_warning(orig_function);\n#endif\n if (!tmp) errno = ENOMEM;\n return tmp;\n}\n\nextern \"C\" void scalable_aligned_free(void *ptr)\n{\n internalFree(ptr);\n}\n\n/********* end code for aligned allocation API **********/\n\n/********* Code for scalable_msize ***********/\n\n/*\n * Returns the size of a memory block allocated in the heap.\n */\nextern \"C\" size_t scalable_msize(void* ptr)\n{\n if (ptr) {\n MALLOC_ASSERT(isRecognized(ptr), \"Invalid pointer in scalable_msize detected.\");\n return internalMsize(ptr);\n }\n errno = EINVAL;\n // Unlike _msize, return 0 in case of parameter error.\n // Returning size_t(-1) looks more like the way to troubles.\n return 0;\n}\n\n/*\n * A variant that provides additional memory safety, by checking whether the given address\n * was obtained with this allocator, and if not redirecting to the provided alternative call.\n */\nextern \"C\" TBBMALLOC_EXPORT size_t __TBB_malloc_safer_msize(void *object, size_t (*original_msize)(void*))\n{\n if (object) {\n // Check if the memory was allocated by scalable_malloc\n if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(object))\n return internalMsize(object);\n else if (original_msize)\n return original_msize(object);\n }\n // object is nullptr or unknown, or foreign and no original_msize\n#if USE_WINTHREAD\n errno = EINVAL; // errno expected to be set only on this platform\n#endif\n return 0;\n}\n\n/*\n * The same as above but for _aligned_msize case\n */\nextern \"C\" TBBMALLOC_EXPORT size_t __TBB_malloc_safer_aligned_msize(void *object, size_t alignment, size_t offset, size_t (*orig_aligned_msize)(void*,size_t,size_t))\n{\n if (object) {\n // Check if the memory was allocated by scalable_malloc\n if (mallocInitialized.load(std::memory_order_acquire) && isRecognized(object))\n return internalMsize(object);\n else if (orig_aligned_msize)\n return orig_aligned_msize(object,alignment,offset);\n }\n // object is nullptr or unknown\n errno = EINVAL;\n return 0;\n}\n\n/********* End code for scalable_msize ***********/\n\nextern \"C\" int scalable_allocation_mode(int param, intptr_t value)\n{\n if (param == TBBMALLOC_SET_SOFT_HEAP_LIMIT) {\n defaultMemPool->extMemPool.backend.setRecommendedMaxSize((size_t)value);\n return TBBMALLOC_OK;\n } else if (param == USE_HUGE_PAGES) {\n#if __unix__\n switch (value) {\n case 0:\n case 1:\n hugePages.setMode(value);\n return TBBMALLOC_OK;\n default:\n return TBBMALLOC_INVALID_PARAM;\n }\n#else\n return TBBMALLOC_NO_EFFECT;\n#endif\n#if __TBB_SOURCE_DIRECTLY_INCLUDED\n } else if (param == TBBMALLOC_INTERNAL_SOURCE_INCLUDED) {\n switch (value) {\n case 0: // used by dynamic library\n case 1: // used by static library or directly included sources\n usedBySrcIncluded = value;\n return TBBMALLOC_OK;\n default:\n return TBBMALLOC_INVALID_PARAM;\n }\n#endif\n } else if (param == TBBMALLOC_SET_HUGE_SIZE_THRESHOLD) {\n defaultMemPool->extMemPool.loc.setHugeSizeThreshold((size_t)value);\n return TBBMALLOC_OK;\n }\n return TBBMALLOC_INVALID_PARAM;\n}\n\nextern \"C\" int scalable_allocation_command(int cmd, void *param)\n{\n if (param)\n return TBBMALLOC_INVALID_PARAM;\n\n bool released = false;\n switch(cmd) {\n case TBBMALLOC_CLEAN_THREAD_BUFFERS:\n if (TLSData *tls = defaultMemPool->getTLS(/*create=*/false))\n released = tls->externalCleanup(/*cleanOnlyUnused*/false, /*cleanBins=*/true);\n break;\n case TBBMALLOC_CLEAN_ALL_BUFFERS:\n released = defaultMemPool->extMemPool.hardCachesCleanup(true);\n break;\n default:\n return TBBMALLOC_INVALID_PARAM;\n }\n return released ? TBBMALLOC_OK : TBBMALLOC_NO_EFFECT;\n}\n" }, { "path": "src/tbb/src/tbbmalloc/large_objects.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"tbbmalloc_internal.h\"\n#include \"../src/tbb/environment.h\"\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // Suppress warning: unary minus operator applied to unsigned type, result still unsigned\n // TBB_REVAMP_TODO: review this warning\n // #pragma warning(push)\n // #pragma warning(disable:4146)\n#endif\n\n/******************************* Allocation of large objects *********************************************/\n\nnamespace rml {\nnamespace internal {\n\n/* ---------------------------- Large Object cache init section ---------------------------------------- */\n\nvoid LargeObjectCache::init(ExtMemoryPool *memPool)\n{\n extMemPool = memPool;\n // scalable_allocation_mode can be called before allocator initialization, respect this manual request\n if (hugeSizeThreshold == 0) {\n // Huge size threshold initialization if environment variable was set\n long requestedThreshold = tbb::detail::r1::GetIntegralEnvironmentVariable(\"TBB_MALLOC_SET_HUGE_SIZE_THRESHOLD\");\n // Read valid env or initialize by default with max possible values\n if (requestedThreshold != -1) {\n setHugeSizeThreshold(requestedThreshold);\n } else {\n setHugeSizeThreshold(maxHugeSize);\n }\n }\n}\n\n/* ----------------------------- Huge size threshold settings ----------------------------------------- */\n\nvoid LargeObjectCache::setHugeSizeThreshold(size_t value)\n{\n // Valid in the huge cache range: [MaxLargeSize, MaxHugeSize].\n if (value <= maxHugeSize) {\n hugeSizeThreshold = value >= maxLargeSize ? alignToBin(value) : maxLargeSize;\n\n // Calculate local indexes for the global threshold size (for fast search inside a regular cleanup)\n largeCache.hugeSizeThresholdIdx = LargeCacheType::numBins;\n hugeCache.hugeSizeThresholdIdx = HugeCacheType::sizeToIdx(hugeSizeThreshold);\n }\n}\n\nbool LargeObjectCache::sizeInCacheRange(size_t size)\n{\n return size < maxHugeSize && (size <= defaultMaxHugeSize || size >= hugeSizeThreshold);\n}\n\n/* ----------------------------------------------------------------------------------------------------- */\n\n/* The functor called by the aggregator for the operation list */\ntemplate\nclass CacheBinFunctor {\n typename LargeObjectCacheImpl::CacheBin *const bin;\n ExtMemoryPool *const extMemPool;\n typename LargeObjectCacheImpl::BinBitMask *const bitMask;\n const int idx;\n\n LargeMemoryBlock *toRelease;\n bool needCleanup;\n uintptr_t currTime;\n\n /* Do preprocessing under the operation list. */\n /* All the OP_PUT_LIST operations are merged in the one operation.\n All OP_GET operations are merged with the OP_PUT_LIST operations but\n it demands the update of the moving average value in the bin.\n Only the last OP_CLEAN_TO_THRESHOLD operation has sense.\n The OP_CLEAN_ALL operation also should be performed only once.\n Moreover it cancels the OP_CLEAN_TO_THRESHOLD operation. */\n class OperationPreprocessor {\n // TODO: remove the dependency on CacheBin.\n typename LargeObjectCacheImpl::CacheBin *const bin;\n\n /* Contains the relative time in the operation list.\n It counts in the reverse order since the aggregator also\n provides operations in the reverse order. */\n uintptr_t lclTime;\n\n /* opGet contains only OP_GET operations which cannot be merge with OP_PUT operations\n opClean contains all OP_CLEAN_TO_THRESHOLD and OP_CLEAN_ALL operations. */\n CacheBinOperation *opGet, *opClean;\n /* The time of the last OP_CLEAN_TO_THRESHOLD operations */\n uintptr_t cleanTime;\n\n /* lastGetOpTime - the time of the last OP_GET operation.\n lastGet - the same meaning as CacheBin::lastGet */\n uintptr_t lastGetOpTime, lastGet;\n\n /* The total sum of all usedSize changes requested with CBOP_UPDATE_USED_SIZE operations. */\n size_t updateUsedSize;\n\n /* The list of blocks for the OP_PUT_LIST operation. */\n LargeMemoryBlock *head, *tail;\n int putListNum;\n\n /* if the OP_CLEAN_ALL is requested. */\n bool isCleanAll;\n\n inline void commitOperation(CacheBinOperation *op) const;\n inline void addOpToOpList(CacheBinOperation *op, CacheBinOperation **opList) const;\n bool getFromPutList(CacheBinOperation* opGet, uintptr_t currTime);\n void addToPutList( LargeMemoryBlock *head, LargeMemoryBlock *tail, int num );\n\n public:\n OperationPreprocessor(typename LargeObjectCacheImpl::CacheBin *bin) :\n bin(bin), lclTime(0), opGet(nullptr), opClean(nullptr), cleanTime(0),\n lastGetOpTime(0), lastGet(0), updateUsedSize(0), head(nullptr), tail(nullptr), putListNum(0), isCleanAll(false) {}\n void operator()(CacheBinOperation* opList);\n uintptr_t getTimeRange() const { return -lclTime; }\n\n friend class CacheBinFunctor;\n };\n\npublic:\n CacheBinFunctor(typename LargeObjectCacheImpl::CacheBin *bin, ExtMemoryPool *extMemPool,\n typename LargeObjectCacheImpl::BinBitMask *bitMask, int idx) :\n bin(bin), extMemPool(extMemPool), bitMask(bitMask), idx(idx), toRelease(nullptr), needCleanup(false), currTime(0) {}\n void operator()(CacheBinOperation* opList);\n\n bool isCleanupNeeded() const { return needCleanup; }\n LargeMemoryBlock *getToRelease() const { return toRelease; }\n uintptr_t getCurrTime() const { return currTime; }\n};\n\n/* ---------------- Cache Bin Aggregator Operation Helpers ---------------- */\n\n// The list of structures which describe the operation data\nstruct OpGet {\n static const CacheBinOperationType type = CBOP_GET;\n LargeMemoryBlock **res;\n size_t size;\n uintptr_t currTime;\n};\n\nstruct OpPutList {\n static const CacheBinOperationType type = CBOP_PUT_LIST;\n LargeMemoryBlock *head;\n};\n\nstruct OpCleanToThreshold {\n static const CacheBinOperationType type = CBOP_CLEAN_TO_THRESHOLD;\n LargeMemoryBlock **res;\n uintptr_t currTime;\n};\n\nstruct OpCleanAll {\n static const CacheBinOperationType type = CBOP_CLEAN_ALL;\n LargeMemoryBlock **res;\n};\n\nstruct OpUpdateUsedSize {\n static const CacheBinOperationType type = CBOP_UPDATE_USED_SIZE;\n size_t size;\n};\n\nunion CacheBinOperationData {\nprivate:\n OpGet opGet;\n OpPutList opPutList;\n OpCleanToThreshold opCleanToThreshold;\n OpCleanAll opCleanAll;\n OpUpdateUsedSize opUpdateUsedSize;\n};\n\n// Forward declarations\ntemplate OpTypeData& opCast(CacheBinOperation &op);\n\n// Describes the aggregator operation\nstruct CacheBinOperation : public MallocAggregatedOperation::type {\n CacheBinOperationType type;\n\n template \n CacheBinOperation(OpTypeData &d, CacheBinOperationStatus st = CBST_WAIT) {\n opCast(*this) = d;\n type = OpTypeData::type;\n MallocAggregatedOperation::type::status = st;\n }\nprivate:\n CacheBinOperationData data;\n\n template \n friend OpTypeData& opCast(CacheBinOperation &op);\n};\n\n// The opCast function can be the member of CacheBinOperation but it will have\n// small stylistic ambiguity: it will look like a getter (with a cast) for the\n// CacheBinOperation::data data member but it should return a reference to\n// simplify the code from a lot of getter/setter calls. So the global cast in\n// the style of static_cast (or reinterpret_cast) seems to be more readable and\n// have more explicit semantic.\ntemplate \nOpTypeData& opCast(CacheBinOperation &op) {\n return *reinterpret_cast(&op.data);\n}\n\n/* ------------------------------------------------------------------------ */\n\n#if __TBB_MALLOC_LOCACHE_STAT\n//intptr_t mallocCalls, cacheHits;\nstd::atomic mallocCalls, cacheHits;\n//intptr_t memAllocKB, memHitKB;\nstd::atomic memAllocKB, memHitKB;\n#endif\n\n#if MALLOC_DEBUG\ninline bool lessThanWithOverflow(intptr_t a, intptr_t b)\n{\n return (a < b && (b - a < static_cast(UINTPTR_MAX/2))) ||\n (a > b && (a - b > static_cast(UINTPTR_MAX/2)));\n}\n#endif\n\n/* ----------------------------------- Operation processing methods ------------------------------------ */\n\ntemplate void CacheBinFunctor::\n OperationPreprocessor::commitOperation(CacheBinOperation *op) const\n{\n // FencedStore( (intptr_t&)(op->status), CBST_DONE );\n op->status.store(CBST_DONE, std::memory_order_release);\n}\n\ntemplate void CacheBinFunctor::\n OperationPreprocessor::addOpToOpList(CacheBinOperation *op, CacheBinOperation **opList) const\n{\n op->next = *opList;\n *opList = op;\n}\n\ntemplate bool CacheBinFunctor::\n OperationPreprocessor::getFromPutList(CacheBinOperation *opGet, uintptr_t currTime)\n{\n if ( head ) {\n uintptr_t age = head->age;\n LargeMemoryBlock *next = head->next;\n *opCast(*opGet).res = head;\n commitOperation( opGet );\n head = next;\n putListNum--;\n MALLOC_ASSERT( putListNum>=0, ASSERT_TEXT );\n\n // use moving average with current hit interval\n bin->updateMeanHitRange( currTime - age );\n return true;\n }\n return false;\n}\n\ntemplate void CacheBinFunctor::\n OperationPreprocessor::addToPutList(LargeMemoryBlock *h, LargeMemoryBlock *t, int num)\n{\n if ( head ) {\n MALLOC_ASSERT( tail, ASSERT_TEXT );\n tail->next = h;\n h->prev = tail;\n tail = t;\n putListNum += num;\n } else {\n head = h;\n tail = t;\n putListNum = num;\n }\n}\n\ntemplate void CacheBinFunctor::\n OperationPreprocessor::operator()(CacheBinOperation* opList)\n{\n for ( CacheBinOperation *op = opList, *opNext; op; op = opNext ) {\n opNext = op->next;\n switch ( op->type ) {\n case CBOP_GET:\n {\n lclTime--;\n if ( !lastGetOpTime ) {\n lastGetOpTime = lclTime;\n lastGet = 0;\n } else if ( !lastGet ) lastGet = lclTime;\n\n if ( !getFromPutList(op,lclTime) ) {\n opCast(*op).currTime = lclTime;\n addOpToOpList( op, &opGet );\n }\n }\n break;\n\n case CBOP_PUT_LIST:\n {\n LargeMemoryBlock *head = opCast(*op).head;\n LargeMemoryBlock *curr = head, *prev = nullptr;\n\n int num = 0;\n do {\n // we do not kept prev pointers during assigning blocks to bins, set them now\n curr->prev = prev;\n\n // Save the local times to the memory blocks. Local times are necessary\n // for the getFromPutList function which updates the hit range value in\n // CacheBin when OP_GET and OP_PUT_LIST operations are merged successfully.\n // The age will be updated to the correct global time after preprocessing\n // when global cache time is updated.\n curr->age = --lclTime;\n\n prev = curr;\n num += 1;\n\n STAT_increment(getThreadId(), ThreadCommonCounters, cacheLargeObj);\n } while ((curr = curr->next) != nullptr);\n\n LargeMemoryBlock *tail = prev;\n addToPutList(head, tail, num);\n\n while ( opGet ) {\n CacheBinOperation *next = opGet->next;\n if ( !getFromPutList(opGet, opCast(*opGet).currTime) )\n break;\n opGet = next;\n }\n }\n break;\n\n case CBOP_UPDATE_USED_SIZE:\n updateUsedSize += opCast(*op).size;\n commitOperation( op );\n break;\n\n case CBOP_CLEAN_ALL:\n isCleanAll = true;\n addOpToOpList( op, &opClean );\n break;\n\n case CBOP_CLEAN_TO_THRESHOLD:\n {\n uintptr_t currTime = opCast(*op).currTime;\n // We don't worry about currTime overflow since it is a rare\n // occurrence and doesn't affect correctness\n cleanTime = cleanTime < currTime ? currTime : cleanTime;\n addOpToOpList( op, &opClean );\n }\n break;\n\n default:\n MALLOC_ASSERT( false, \"Unknown operation.\" );\n }\n }\n MALLOC_ASSERT( !( opGet && head ), \"Not all put/get pairs are processed!\" );\n}\n\ntemplate void CacheBinFunctor::operator()(CacheBinOperation* opList)\n{\n MALLOC_ASSERT( opList, \"Empty operation list is passed into operation handler.\" );\n\n OperationPreprocessor prep(bin);\n prep(opList);\n\n if ( uintptr_t timeRange = prep.getTimeRange() ) {\n uintptr_t startTime = extMemPool->loc.getCurrTimeRange(timeRange);\n // endTime is used as the current (base) time since the local time is negative.\n uintptr_t endTime = startTime + timeRange;\n\n if ( prep.lastGetOpTime && prep.lastGet ) bin->setLastGet(prep.lastGet+endTime);\n\n if ( CacheBinOperation *opGet = prep.opGet ) {\n bool isEmpty = false;\n do {\n#if __TBB_MALLOC_WHITEBOX_TEST\n tbbmalloc_whitebox::locGetProcessed++;\n#endif\n const OpGet &opGetData = opCast(*opGet);\n if ( !isEmpty ) {\n if ( LargeMemoryBlock *res = bin->get() ) {\n uintptr_t getTime = opGetData.currTime + endTime;\n // use moving average with current hit interval\n bin->updateMeanHitRange( getTime - res->age);\n bin->updateCachedSize( -opGetData.size );\n *opGetData.res = res;\n } else {\n isEmpty = true;\n uintptr_t lastGetOpTime = prep.lastGetOpTime+endTime;\n bin->forgetOutdatedState(lastGetOpTime);\n bin->updateAgeThreshold(lastGetOpTime);\n }\n }\n\n CacheBinOperation *opNext = opGet->next;\n bin->updateUsedSize( opGetData.size, bitMask, idx );\n prep.commitOperation( opGet );\n opGet = opNext;\n } while ( opGet );\n if ( prep.lastGetOpTime )\n bin->setLastGet( prep.lastGetOpTime + endTime );\n } else if ( LargeMemoryBlock *curr = prep.head ) {\n curr->prev = nullptr;\n while ( curr ) {\n // Update local times to global times\n curr->age += endTime;\n curr=curr->next;\n }\n#if __TBB_MALLOC_WHITEBOX_TEST\n tbbmalloc_whitebox::locPutProcessed+=prep.putListNum;\n#endif\n toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum, extMemPool->loc.hugeSizeThreshold);\n }\n needCleanup = extMemPool->loc.isCleanupNeededOnRange(timeRange, startTime);\n currTime = endTime - 1;\n }\n\n if ( CacheBinOperation *opClean = prep.opClean ) {\n if ( prep.isCleanAll )\n *opCast(*opClean).res = bin->cleanAll(bitMask, idx);\n else\n *opCast(*opClean).res = bin->cleanToThreshold(prep.cleanTime, bitMask, idx);\n\n CacheBinOperation *opNext = opClean->next;\n prep.commitOperation( opClean );\n\n while ((opClean = opNext) != nullptr) {\n opNext = opClean->next;\n prep.commitOperation(opClean);\n }\n }\n\n if ( size_t size = prep.updateUsedSize )\n bin->updateUsedSize(size, bitMask, idx);\n}\n/* ----------------------------------------------------------------------------------------------------- */\n/* --------------------------- Methods for creating and executing operations --------------------------- */\ntemplate void LargeObjectCacheImpl::\n CacheBin::ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime)\n{\n CacheBinFunctor func( this, extMemPool, bitMask, idx );\n aggregator.execute( op, func, longLifeTime );\n\n if ( LargeMemoryBlock *toRelease = func.getToRelease()) {\n extMemPool->backend.returnLargeObject(toRelease);\n }\n\n if ( func.isCleanupNeeded() ) {\n extMemPool->loc.doCleanup( func.getCurrTime(), /*doThreshDecr=*/false);\n }\n}\n\ntemplate LargeMemoryBlock *LargeObjectCacheImpl::\n CacheBin::get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx)\n{\n LargeMemoryBlock *lmb=nullptr;\n OpGet data = {&lmb, size, static_cast(0)};\n CacheBinOperation op(data);\n ExecuteOperation( &op, extMemPool, bitMask, idx );\n return lmb;\n}\n\ntemplate void LargeObjectCacheImpl::\n CacheBin::putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx)\n{\n MALLOC_ASSERT(sizeof(LargeMemoryBlock)+sizeof(CacheBinOperation)<=head->unalignedSize, \"CacheBinOperation is too large to be placed in LargeMemoryBlock!\");\n\n OpPutList data = {head};\n CacheBinOperation *op = new (head+1) CacheBinOperation(data, CBST_NOWAIT);\n ExecuteOperation( op, extMemPool, bitMask, idx, false );\n}\n\ntemplate bool LargeObjectCacheImpl::\n CacheBin::cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx)\n{\n LargeMemoryBlock *toRelease = nullptr;\n\n /* oldest may be more recent then age, that's why cast to signed type\n was used. age overflow is also processed correctly. */\n if (last.load(std::memory_order_relaxed) &&\n (intptr_t)(currTime - oldest.load(std::memory_order_relaxed)) > ageThreshold.load(std::memory_order_relaxed)) {\n OpCleanToThreshold data = {&toRelease, currTime};\n CacheBinOperation op(data);\n ExecuteOperation( &op, extMemPool, bitMask, idx );\n }\n bool released = toRelease;\n\n Backend *backend = &extMemPool->backend;\n while ( toRelease ) {\n LargeMemoryBlock *helper = toRelease->next;\n backend->returnLargeObject(toRelease);\n toRelease = helper;\n }\n return released;\n}\n\ntemplate bool LargeObjectCacheImpl::\n CacheBin::releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx)\n{\n LargeMemoryBlock *toRelease = nullptr;\n\n if (last.load(std::memory_order_relaxed)) {\n OpCleanAll data = {&toRelease};\n CacheBinOperation op(data);\n ExecuteOperation(&op, extMemPool, bitMask, idx);\n }\n bool released = toRelease;\n\n Backend *backend = &extMemPool->backend;\n while ( toRelease ) {\n LargeMemoryBlock *helper = toRelease->next;\n MALLOC_ASSERT(!helper || lessThanWithOverflow(helper->age, toRelease->age),\n ASSERT_TEXT);\n backend->returnLargeObject(toRelease);\n toRelease = helper;\n }\n return released;\n}\n\ntemplate void LargeObjectCacheImpl::\n CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx)\n{\n OpUpdateUsedSize data = {size};\n CacheBinOperation op(data);\n ExecuteOperation( &op, extMemPool, bitMask, idx );\n}\n\n/* ------------------------------ Unsafe methods used with the aggregator ------------------------------ */\n\ntemplate LargeMemoryBlock *LargeObjectCacheImpl::\n CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num, size_t hugeSizeThreshold)\n{\n size_t size = head->unalignedSize;\n usedSize.store(usedSize.load(std::memory_order_relaxed) - num * size, std::memory_order_relaxed);\n MALLOC_ASSERT( !last.load(std::memory_order_relaxed) ||\n (last.load(std::memory_order_relaxed)->age != 0 && last.load(std::memory_order_relaxed)->age != -1U), ASSERT_TEXT );\n MALLOC_ASSERT( (tail==head && num==1) || (tail!=head && num>1), ASSERT_TEXT );\n MALLOC_ASSERT( tail, ASSERT_TEXT );\n LargeMemoryBlock *toRelease = nullptr;\n if (size < hugeSizeThreshold && !lastCleanedAge) {\n // 1st object of such size was released.\n // Not cache it, and remember when this occurs\n // to take into account during cache miss.\n lastCleanedAge = tail->age;\n toRelease = tail;\n tail = tail->prev;\n if (tail)\n tail->next = nullptr;\n else\n head = nullptr;\n num--;\n }\n if (num) {\n // add [head;tail] list to cache\n tail->next = first;\n if (first)\n first->prev = tail;\n first = head;\n if (!last.load(std::memory_order_relaxed)) {\n MALLOC_ASSERT(0 == oldest.load(std::memory_order_relaxed), ASSERT_TEXT);\n oldest.store(tail->age, std::memory_order_relaxed);\n last.store(tail, std::memory_order_relaxed);\n }\n\n cachedSize.store(cachedSize.load(std::memory_order_relaxed) + num * size, std::memory_order_relaxed);\n }\n\n // No used object, and nothing in the bin, mark the bin as empty\n if (!usedSize.load(std::memory_order_relaxed) && !first)\n bitMask->set(idx, false);\n\n return toRelease;\n}\n\ntemplate LargeMemoryBlock *LargeObjectCacheImpl::\n CacheBin::get()\n{\n LargeMemoryBlock *result=first;\n if (result) {\n first = result->next;\n if (first)\n first->prev = nullptr;\n else {\n last.store(nullptr, std::memory_order_relaxed);\n oldest.store(0, std::memory_order_relaxed);\n }\n }\n\n return result;\n}\n\ntemplate void LargeObjectCacheImpl::\n CacheBin::forgetOutdatedState(uintptr_t currTime)\n{\n // If the time since the last get is LongWaitFactor times more than ageThreshold\n // for the bin, treat the bin as rarely-used and forget everything we know\n // about it.\n // If LongWaitFactor is too small, we forget too early and\n // so prevents good caching, while if too high, caching blocks\n // with unrelated usage pattern occurs.\n const uintptr_t sinceLastGet = currTime - lastGet;\n bool doCleanup = false;\n\n intptr_t threshold = ageThreshold.load(std::memory_order_relaxed);\n if (threshold)\n doCleanup = sinceLastGet > static_cast(Props::LongWaitFactor * threshold);\n else if (lastCleanedAge)\n doCleanup = sinceLastGet > static_cast(Props::LongWaitFactor * (lastCleanedAge - lastGet));\n\n if (doCleanup) {\n lastCleanedAge = 0;\n ageThreshold.store(0, std::memory_order_relaxed);\n }\n\n}\n\ntemplate LargeMemoryBlock *LargeObjectCacheImpl::\n CacheBin::cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx)\n{\n /* oldest may be more recent then age, that's why cast to signed type\n was used. age overflow is also processed correctly. */\n if ( !last.load(std::memory_order_relaxed) ||\n (intptr_t)(currTime - last.load(std::memory_order_relaxed)->age) < ageThreshold.load(std::memory_order_relaxed) )\n return nullptr;\n\n#if MALLOC_DEBUG\n uintptr_t nextAge = 0;\n#endif\n do {\n#if MALLOC_DEBUG\n // check that list ordered\n MALLOC_ASSERT(!nextAge || lessThanWithOverflow(nextAge, last.load(std::memory_order_relaxed)->age),\n ASSERT_TEXT);\n nextAge = last.load(std::memory_order_relaxed)->age;\n#endif\n cachedSize.store(cachedSize.load(std::memory_order_relaxed) - last.load(std::memory_order_relaxed)->unalignedSize, std::memory_order_relaxed);\n last.store(last.load(std::memory_order_relaxed)->prev, std::memory_order_relaxed);\n } while (last.load(std::memory_order_relaxed) &&\n (intptr_t)(currTime - last.load(std::memory_order_relaxed)->age) > ageThreshold.load(std::memory_order_relaxed));\n\n LargeMemoryBlock *toRelease = nullptr;\n if (last.load(std::memory_order_relaxed)) {\n toRelease = last.load(std::memory_order_relaxed)->next;\n oldest.store(last.load(std::memory_order_relaxed)->age, std::memory_order_relaxed);\n last.load(std::memory_order_relaxed)->next = nullptr;\n } else {\n toRelease = first;\n first = nullptr;\n oldest.store(0, std::memory_order_relaxed);\n if (!usedSize.load(std::memory_order_relaxed))\n bitMask->set(idx, false);\n }\n MALLOC_ASSERT( toRelease, ASSERT_TEXT );\n lastCleanedAge = toRelease->age;\n\n return toRelease;\n}\n\ntemplate LargeMemoryBlock *LargeObjectCacheImpl::\n CacheBin::cleanAll(BinBitMask *bitMask, int idx)\n{\n if (!last.load(std::memory_order_relaxed)) return nullptr;\n\n LargeMemoryBlock *toRelease = first;\n last.store(nullptr, std::memory_order_relaxed);\n first = nullptr;\n oldest.store(0, std::memory_order_relaxed);\n cachedSize.store(0, std::memory_order_relaxed);\n if (!usedSize.load(std::memory_order_relaxed))\n bitMask->set(idx, false);\n\n return toRelease;\n}\n\n/* ----------------------------------------------------------------------------------------------------- */\n\n#if __TBB_MALLOC_BACKEND_STAT\ntemplate size_t LargeObjectCacheImpl::\n CacheBin::reportStat(int num, FILE *f)\n{\n#if __TBB_MALLOC_LOCACHE_STAT\n if (first)\n printf(\"%d(%lu): total %lu KB thr %ld lastCln %lu oldest %lu\\n\",\n num, num*Props::CacheStep+Props::MinSize,\n cachedSize.load(std::memory_order_relaxed)/1024, ageThresholdageThreshold.load(std::memory_order_relaxed), lastCleanedAge, oldest.load(std::memory_order_relaxed));\n#else\n suppress_unused_warning(num);\n suppress_unused_warning(f);\n#endif\n return cachedSize.load(std::memory_order_relaxed);\n}\n#endif\n\n// Release objects from cache blocks that are older than ageThreshold\ntemplate\nbool LargeObjectCacheImpl::regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currTime, bool doThreshDecr)\n{\n bool released = false;\n BinsSummary binsSummary;\n\n // Threshold settings is below this cache or starts from zero index\n if (hugeSizeThresholdIdx == 0) return false;\n\n // Starting searching for bin that is less than huge size threshold (can be cleaned-up)\n int startSearchIdx = hugeSizeThresholdIdx - 1;\n\n for (int i = bitMask.getMaxTrue(startSearchIdx); i >= 0; i = bitMask.getMaxTrue(i-1)) {\n bin[i].updateBinsSummary(&binsSummary);\n if (!doThreshDecr && tooLargeLOC.load(std::memory_order_relaxed) > 2 && binsSummary.isLOCTooLarge()) {\n // if LOC is too large for quite long time, decrease the threshold\n // based on bin hit statistics.\n // For this, redo cleanup from the beginning.\n // Note: on this iteration total usedSz can be not too large\n // in comparison to total cachedSz, as we calculated it only\n // partially. We are ok with it.\n i = bitMask.getMaxTrue(startSearchIdx)+1;\n doThreshDecr = true;\n binsSummary.reset();\n continue;\n }\n if (doThreshDecr)\n bin[i].decreaseThreshold();\n\n if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) {\n released = true;\n }\n }\n // We want to find if LOC was too large for some time continuously,\n // so OK with races between incrementing and zeroing, but incrementing\n // must be atomic.\n if (binsSummary.isLOCTooLarge()) {\n tooLargeLOC++;\n } else {\n tooLargeLOC.store(0, std::memory_order_relaxed);\n }\n return released;\n}\n\ntemplate\nbool LargeObjectCacheImpl::cleanAll(ExtMemoryPool *extMemPool)\n{\n bool released = false;\n for (int i = numBins-1; i >= 0; i--) {\n released |= bin[i].releaseAllToBackend(extMemPool, &bitMask, i);\n }\n return released;\n}\n\ntemplate\nvoid LargeObjectCacheImpl::reset() {\n tooLargeLOC.store(0, std::memory_order_relaxed);\n for (int i = numBins-1; i >= 0; i--)\n bin[i].init();\n bitMask.reset();\n}\n\n#if __TBB_MALLOC_WHITEBOX_TEST\ntemplate\nsize_t LargeObjectCacheImpl::getLOCSize() const\n{\n size_t size = 0;\n for (int i = numBins-1; i >= 0; i--)\n size += bin[i].getSize();\n return size;\n}\n\nsize_t LargeObjectCache::getLOCSize() const\n{\n return largeCache.getLOCSize() + hugeCache.getLOCSize();\n}\n\ntemplate\nsize_t LargeObjectCacheImpl::getUsedSize() const\n{\n size_t size = 0;\n for (int i = numBins-1; i >= 0; i--)\n size += bin[i].getUsedSize();\n return size;\n}\n\nsize_t LargeObjectCache::getUsedSize() const\n{\n return largeCache.getUsedSize() + hugeCache.getUsedSize();\n}\n#endif // __TBB_MALLOC_WHITEBOX_TEST\n\ninline bool LargeObjectCache::isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime)\n{\n return range >= cacheCleanupFreq\n || currTime+range < currTime-1 // overflow, 0 is power of 2, do cleanup\n // (prev;prev+range] contains n*cacheCleanupFreq\n || alignUp(currTime, cacheCleanupFreq)allLocalCaches.markUnused();\n\n bool large_cache_cleaned = largeCache.regularCleanup(extMemPool, currTime, doThreshDecr);\n bool huge_cache_cleaned = hugeCache.regularCleanup(extMemPool, currTime, doThreshDecr);\n return large_cache_cleaned || huge_cache_cleaned;\n}\n\nbool LargeObjectCache::decreasingCleanup()\n{\n return doCleanup(cacheCurrTime.load(std::memory_order_acquire), /*doThreshDecr=*/true);\n}\n\nbool LargeObjectCache::regularCleanup()\n{\n return doCleanup(cacheCurrTime.load(std::memory_order_acquire), /*doThreshDecr=*/false);\n}\n\nbool LargeObjectCache::cleanAll()\n{\n bool large_cache_cleaned = largeCache.cleanAll(extMemPool);\n bool huge_cache_cleaned = hugeCache.cleanAll(extMemPool);\n return large_cache_cleaned || huge_cache_cleaned;\n}\n\nvoid LargeObjectCache::reset()\n{\n largeCache.reset();\n hugeCache.reset();\n}\n\ntemplate\nLargeMemoryBlock *LargeObjectCacheImpl::get(ExtMemoryPool *extMemoryPool, size_t size)\n{\n int idx = Props::sizeToIdx(size);\n\n LargeMemoryBlock *lmb = bin[idx].get(extMemoryPool, size, &bitMask, idx);\n\n if (lmb) {\n MALLOC_ITT_SYNC_ACQUIRED(bin+idx);\n STAT_increment(getThreadId(), ThreadCommonCounters, allocCachedLargeObj);\n }\n return lmb;\n}\n\ntemplate\nvoid LargeObjectCacheImpl::updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size)\n{\n int idx = Props::sizeToIdx(size);\n MALLOC_ASSERT(idx < static_cast(numBins), ASSERT_TEXT);\n bin[idx].updateUsedSize(extMemPool, op==decrease? -size : size, &bitMask, idx);\n}\n\n#if __TBB_MALLOC_LOCACHE_STAT\ntemplate\nvoid LargeObjectCacheImpl::reportStat(FILE *f)\n{\n size_t cachedSize = 0;\n for (int i=0; i\nvoid LargeObjectCacheImpl::putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *toCache)\n{\n int toBinIdx = Props::sizeToIdx(toCache->unalignedSize);\n\n MALLOC_ITT_SYNC_RELEASING(bin+toBinIdx);\n bin[toBinIdx].putList(extMemPool, toCache, &bitMask, toBinIdx);\n}\n\nvoid LargeObjectCache::updateCacheState(DecreaseOrIncrease op, size_t size)\n{\n if (size < maxLargeSize)\n largeCache.updateCacheState(extMemPool, op, size);\n else if (size < maxHugeSize)\n hugeCache.updateCacheState(extMemPool, op, size);\n}\n\n\nuintptr_t LargeObjectCache::getCurrTimeRange(uintptr_t range)\n{\n return (cacheCurrTime.fetch_add(range) + 1);\n}\n\nvoid LargeObjectCache::registerRealloc(size_t oldSize, size_t newSize)\n{\n updateCacheState(decrease, oldSize);\n updateCacheState(increase, alignToBin(newSize));\n}\n\nsize_t LargeObjectCache::alignToBin(size_t size) {\n return size < maxLargeSize ? LargeCacheType::alignToBin(size) : HugeCacheType::alignToBin(size);\n}\n\n// Used for internal purpose\nint LargeObjectCache::sizeToIdx(size_t size)\n{\n MALLOC_ASSERT(size <= maxHugeSize, ASSERT_TEXT);\n return size < maxLargeSize ?\n LargeCacheType::sizeToIdx(size) :\n LargeCacheType::numBins + HugeCacheType::sizeToIdx(size);\n}\n\nvoid LargeObjectCache::putList(LargeMemoryBlock *list)\n{\n LargeMemoryBlock *toProcess, *n;\n\n for (LargeMemoryBlock *curr = list; curr; curr = toProcess) {\n LargeMemoryBlock *tail = curr;\n toProcess = curr->next;\n if (!sizeInCacheRange(curr->unalignedSize)) {\n extMemPool->backend.returnLargeObject(curr);\n continue;\n }\n int currIdx = sizeToIdx(curr->unalignedSize);\n\n // Find all blocks fitting to same bin. Not use more efficient sorting\n // algorithm because list is short (commonly,\n // LocalLOC's HIGH_MARK-LOW_MARK, i.e. 24 items).\n for (LargeMemoryBlock *b = toProcess; b; b = n) {\n n = b->next;\n if (sizeToIdx(b->unalignedSize) == currIdx) {\n tail->next = b;\n tail = b;\n if (toProcess == b)\n toProcess = toProcess->next;\n else {\n b->prev->next = b->next;\n if (b->next)\n b->next->prev = b->prev;\n }\n }\n }\n tail->next = nullptr;\n if (curr->unalignedSize < maxLargeSize)\n largeCache.putList(extMemPool, curr);\n else\n hugeCache.putList(extMemPool, curr);\n }\n}\n\nvoid LargeObjectCache::put(LargeMemoryBlock *largeBlock)\n{\n size_t blockSize = largeBlock->unalignedSize;\n if (sizeInCacheRange(blockSize)) {\n largeBlock->next = nullptr;\n if (blockSize < maxLargeSize)\n largeCache.putList(extMemPool, largeBlock);\n else\n hugeCache.putList(extMemPool, largeBlock);\n } else {\n extMemPool->backend.returnLargeObject(largeBlock);\n }\n}\n\nLargeMemoryBlock *LargeObjectCache::get(size_t size)\n{\n MALLOC_ASSERT( size >= minLargeSize, ASSERT_TEXT );\n if (sizeInCacheRange(size)) {\n return size < maxLargeSize ?\n largeCache.get(extMemPool, size) : hugeCache.get(extMemPool, size);\n }\n return nullptr;\n}\n\nLargeMemoryBlock *ExtMemoryPool::mallocLargeObject(MemoryPool *pool, size_t allocationSize)\n{\n#if __TBB_MALLOC_LOCACHE_STAT\n mallocCalls++;\n memAllocKB.fetch_add(allocationSize/1024);\n#endif\n LargeMemoryBlock* lmb = loc.get(allocationSize);\n if (!lmb) {\n BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/true);\n if (backRefIdx.isInvalid())\n return nullptr;\n\n // unalignedSize is set in getLargeBlock\n lmb = backend.getLargeBlock(allocationSize);\n if (!lmb) {\n removeBackRef(backRefIdx);\n loc.updateCacheState(decrease, allocationSize);\n return nullptr;\n }\n lmb->backRefIdx = backRefIdx;\n lmb->pool = pool;\n STAT_increment(getThreadId(), ThreadCommonCounters, allocNewLargeObj);\n } else {\n#if __TBB_MALLOC_LOCACHE_STAT\n cacheHits++;\n memHitKB.fetch_add(allocationSize/1024);\n#endif\n }\n return lmb;\n}\n\nvoid ExtMemoryPool::freeLargeObject(LargeMemoryBlock *mBlock)\n{\n loc.put(mBlock);\n}\n\nvoid ExtMemoryPool::freeLargeObjectList(LargeMemoryBlock *head)\n{\n loc.putList(head);\n}\n\nbool ExtMemoryPool::softCachesCleanup()\n{\n bool ret = false;\n if (!softCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel)) {\n ret = loc.regularCleanup();\n softCachesCleanupInProgress.store(0, std::memory_order_release);\n }\n return ret;\n}\n\nbool ExtMemoryPool::hardCachesCleanup(bool wait)\n{\n if (hardCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel)) {\n if (!wait)\n return false;\n\n AtomicBackoff backoff;\n while (hardCachesCleanupInProgress.exchange(1, std::memory_order_acq_rel))\n backoff.pause();\n }\n\n // thread-local caches must be cleaned before LOC,\n // because object from thread-local cache can be released to LOC\n bool ret = releaseAllLocalCaches();\n ret |= orphanedBlocks.cleanup(&backend);\n ret |= loc.cleanAll();\n ret |= backend.clean();\n\n hardCachesCleanupInProgress.store(0, std::memory_order_release);\n return ret;\n}\n\n#if BACKEND_HAS_MREMAP\nvoid *ExtMemoryPool::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment)\n{\n const size_t oldUnalignedSize = ((LargeObjectHdr*)ptr - 1)->memoryBlock->unalignedSize;\n void *o = backend.remap(ptr, oldSize, newSize, alignment);\n if (o) {\n LargeMemoryBlock *lmb = ((LargeObjectHdr*)o - 1)->memoryBlock;\n loc.registerRealloc(oldUnalignedSize, lmb->unalignedSize);\n }\n return o;\n}\n#endif /* BACKEND_HAS_MREMAP */\n\n/*********** End allocation of large objects **********/\n\n} // namespace internal\n} // namespace rml\n\n#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)\n // #pragma warning(pop)\n#endif\n" }, { "path": "src/tbb/src/tbbmalloc/large_objects.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbbmalloc_internal_H\n #error tbbmalloc_internal.h must be included at this point\n#endif\n\n#ifndef __TBB_large_objects_H\n#define __TBB_large_objects_H\n\n//! The list of possible Cache Bin Aggregator operations.\n/* Declared here to avoid Solaris Studio* 12.2 \"multiple definitions\" error */\nenum CacheBinOperationType {\n CBOP_INVALID = 0,\n CBOP_GET,\n CBOP_PUT_LIST,\n CBOP_CLEAN_TO_THRESHOLD,\n CBOP_CLEAN_ALL,\n CBOP_UPDATE_USED_SIZE\n};\n\n// The Cache Bin Aggregator operation status list.\n// CBST_NOWAIT can be specified for non-blocking operations.\nenum CacheBinOperationStatus {\n CBST_WAIT = 0,\n CBST_NOWAIT,\n CBST_DONE\n};\n\n/*\n * Bins that grow with arithmetic step\n */\ntemplate\nstruct LargeBinStructureProps {\npublic:\n static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE;\n static const size_t CacheStep = 8 * 1024;\n static const unsigned NumBins = (MaxSize - MinSize) / CacheStep;\n\n static size_t alignToBin(size_t size) {\n return alignUp(size, CacheStep);\n }\n\n static int sizeToIdx(size_t size) {\n MALLOC_ASSERT(MinSize <= size && size < MaxSize, ASSERT_TEXT);\n MALLOC_ASSERT(size % CacheStep == 0, ASSERT_TEXT);\n return (size - MinSize) / CacheStep;\n }\n};\n\n/*\n * Bins that grow with special geometric progression.\n */\ntemplate\nstruct HugeBinStructureProps {\n\nprivate:\n // Sizes grow with the following formula: Size = MinSize * (2 ^ (Index / StepFactor))\n // There are StepFactor bins (8 be default) between each power of 2 bin\n static const int MaxSizeExp = Log2::value;\n static const int MinSizeExp = Log2::value;\n static const int StepFactor = 8;\n static const int StepFactorExp = Log2::value;\n\npublic:\n static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE;\n static const unsigned NumBins = (MaxSizeExp - MinSizeExp) * StepFactor;\n\n static size_t alignToBin(size_t size) {\n MALLOC_ASSERT(size >= StepFactor, \"Size must not be less than the StepFactor\");\n\n int sizeExp = (int)BitScanRev(size);\n MALLOC_ASSERT(sizeExp >= 0, \"BitScanRev() cannot return -1, as size >= stepfactor > 0\");\n MALLOC_ASSERT(sizeExp >= StepFactorExp, \"sizeExp >= StepFactorExp, because size >= stepFactor\");\n int minorStepExp = sizeExp - StepFactorExp;\n\n return alignUp(size, 1ULL << minorStepExp);\n }\n\n // Sizes between the power of 2 values are approximated to StepFactor.\n static int sizeToIdx(size_t size) {\n MALLOC_ASSERT(MinSize <= size && size <= MaxSize, ASSERT_TEXT);\n\n int sizeExp = (int)BitScanRev(size); // same as __TBB_Log2\n MALLOC_ASSERT(sizeExp >= 0, \"BitScanRev() cannot return -1, as size >= stepfactor > 0\");\n MALLOC_ASSERT(sizeExp >= StepFactorExp, \"sizeExp >= StepFactorExp, because size >= stepFactor\");\n int minorStepExp = sizeExp - StepFactorExp;\n\n size_t majorStepSize = 1ULL << sizeExp;\n int minorIdx = (size - majorStepSize) >> minorStepExp;\n MALLOC_ASSERT(size == majorStepSize + ((size_t)minorIdx << minorStepExp),\n \"Size is not aligned on the bin\");\n return StepFactor * (sizeExp - MinSizeExp) + minorIdx;\n }\n};\n\n/*\n * Cache properties accessor\n *\n * TooLargeFactor -- when cache size treated \"too large\" in comparison to user data size\n * OnMissFactor -- If cache miss occurred and cache was cleaned,\n * set ageThreshold to OnMissFactor * the difference\n * between current time and last time cache was cleaned.\n * LongWaitFactor -- to detect rarely-used bins and forget about their usage history\n */\ntemplate\nstruct LargeObjectCacheProps : public StructureProps {\n static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, LongWaitFactor = LONG_WAIT;\n};\n\ntemplate\nclass LargeObjectCacheImpl {\nprivate:\n\n // Current sizes of used and cached objects. It's calculated while we are\n // traversing bins, and used for isLOCTooLarge() check at the same time.\n class BinsSummary {\n size_t usedSz;\n size_t cachedSz;\n public:\n BinsSummary() : usedSz(0), cachedSz(0) {}\n // \"too large\" criteria\n bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor * usedSz; }\n void update(size_t usedSize, size_t cachedSize) {\n usedSz += usedSize;\n cachedSz += cachedSize;\n }\n void reset() { usedSz = cachedSz = 0; }\n };\n\npublic:\n // The number of bins to cache large/huge objects.\n static const uint32_t numBins = Props::NumBins;\n\n typedef BitMaskMax BinBitMask;\n\n // 2-linked list of same-size cached blocks ordered by age (oldest on top)\n // TODO: are we really want the list to be 2-linked? This allows us\n // reduce memory consumption and do less operations under lock.\n // TODO: try to switch to 32-bit logical time to save space in CacheBin\n // and move bins to different cache lines.\n class CacheBin {\n private:\n LargeMemoryBlock* first;\n std::atomic last;\n /* age of an oldest block in the list; equal to last->age, if last defined,\n used for quick checking it without acquiring the lock. */\n std::atomic oldest;\n /* currAge when something was excluded out of list because of the age,\n not because of cache hit */\n uintptr_t lastCleanedAge;\n /* Current threshold value for the blocks of a particular size.\n Set on cache miss. */\n std::atomic ageThreshold;\n\n /* total size of all objects corresponding to the bin and allocated by user */\n std::atomic usedSize;\n /* total size of all objects cached in the bin */\n std::atomic cachedSize;\n /* mean time of presence of block in the bin before successful reuse */\n std::atomic meanHitRange;\n /* time of last get called for the bin */\n uintptr_t lastGet;\n\n typename MallocAggregator::type aggregator;\n\n void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true);\n\n /* should be placed in zero-initialized memory, ctor not needed. */\n CacheBin();\n\n public:\n void init() {\n memset(static_cast(this), 0, sizeof(CacheBin));\n }\n\n /* ---------- Cache accessors ---------- */\n void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx);\n LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx);\n\n /* ---------- Cleanup functions -------- */\n bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx);\n bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx);\n /* ------------------------------------- */\n\n void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx);\n void decreaseThreshold() {\n intptr_t threshold = ageThreshold.load(std::memory_order_relaxed);\n if (threshold)\n ageThreshold.store((threshold + meanHitRange.load(std::memory_order_relaxed)) / 2, std::memory_order_relaxed);\n }\n void updateBinsSummary(BinsSummary *binsSummary) const {\n binsSummary->update(usedSize.load(std::memory_order_relaxed), cachedSize.load(std::memory_order_relaxed));\n }\n size_t getSize() const { return cachedSize.load(std::memory_order_relaxed); }\n size_t getUsedSize() const { return usedSize.load(std::memory_order_relaxed); }\n size_t reportStat(int num, FILE *f);\n\n /* --------- Unsafe methods used with the aggregator ------- */\n void forgetOutdatedState(uintptr_t currTime);\n LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask,\n int idx, int num, size_t hugeObjectThreshold);\n LargeMemoryBlock *get();\n LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx);\n LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx);\n void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) {\n if (!usedSize.load(std::memory_order_relaxed)) bitMask->set(idx, true);\n usedSize.store(usedSize.load(std::memory_order_relaxed) + size, std::memory_order_relaxed);\n if (!usedSize.load(std::memory_order_relaxed) && !first) bitMask->set(idx, false);\n }\n void updateMeanHitRange( intptr_t hitRange ) {\n hitRange = hitRange >= 0 ? hitRange : 0;\n intptr_t mean = meanHitRange.load(std::memory_order_relaxed);\n mean = mean ? (mean + hitRange) / 2 : hitRange;\n meanHitRange.store(mean, std::memory_order_relaxed);\n }\n void updateAgeThreshold( uintptr_t currTime ) {\n if (lastCleanedAge)\n ageThreshold.store(Props::OnMissFactor * (currTime - lastCleanedAge), std::memory_order_relaxed);\n }\n void updateCachedSize(size_t size) {\n cachedSize.store(cachedSize.load(std::memory_order_relaxed) + size, std::memory_order_relaxed);\n }\n void setLastGet( uintptr_t newLastGet ) {\n lastGet = newLastGet;\n }\n /* -------------------------------------------------------- */\n };\n\n // Huge bins index for fast regular cleanup searching in case of\n // the \"huge size threshold\" setting defined\n intptr_t hugeSizeThresholdIdx;\n\nprivate:\n // How many times LOC was \"too large\"\n std::atomic tooLargeLOC;\n // for fast finding of used bins and bins with non-zero usedSize;\n // indexed from the end, as we need largest 1st\n BinBitMask bitMask;\n // bins with lists of recently freed large blocks cached for reuse\n CacheBin bin[numBins];\n\npublic:\n /* ------------ CacheBin structure dependent stuff ------------ */\n static size_t alignToBin(size_t size) {\n return Props::alignToBin(size);\n }\n static int sizeToIdx(size_t size) {\n return Props::sizeToIdx(size);\n }\n\n /* --------- Main cache functions (put, get object) ------------ */\n void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock);\n LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size);\n\n /* ------------------------ Cleanup ---------------------------- */\n bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr);\n bool cleanAll(ExtMemoryPool *extMemPool);\n\n /* -------------------------- Other ---------------------------- */\n void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size);\n\n void reset();\n void reportStat(FILE *f);\n#if __TBB_MALLOC_WHITEBOX_TEST\n size_t getLOCSize() const;\n size_t getUsedSize() const;\n#endif\n};\n\nclass LargeObjectCache {\nprivate:\n // Large bins [minLargeSize, maxLargeSize)\n // Huge bins [maxLargeSize, maxHugeSize)\n static const size_t minLargeSize = 8 * 1024,\n maxLargeSize = 8 * 1024 * 1024,\n // Cache memory up to 1TB (or 2GB for 32-bit arch), but sieve objects from the special threshold\n maxHugeSize = tbb::detail::select_size_t_constant<2147483648U, 1099511627776ULL>::value;\n\npublic:\n // Upper bound threshold for caching size. After that size all objects sieve through cache\n // By default - 64MB, previous value was 129MB (needed by some Intel(R) Math Kernel Library (Intel(R) MKL) benchmarks)\n static const size_t defaultMaxHugeSize = 64UL * 1024UL * 1024UL;\n // After that size large object interpreted as huge and does not participate in regular cleanup.\n // Can be changed during the program execution.\n size_t hugeSizeThreshold;\n\nprivate:\n // Large objects cache properties\n typedef LargeBinStructureProps LargeBSProps;\n typedef LargeObjectCacheProps LargeCacheTypeProps;\n\n // Huge objects cache properties\n typedef HugeBinStructureProps HugeBSProps;\n typedef LargeObjectCacheProps HugeCacheTypeProps;\n\n // Cache implementation type with properties\n typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType;\n typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType;\n\n // Beginning of largeCache is more actively used and smaller than hugeCache,\n // so put hugeCache first to prevent false sharing\n // with LargeObjectCache's predecessor\n HugeCacheType hugeCache;\n LargeCacheType largeCache;\n\n /* logical time, incremented on each put/get operation\n To prevent starvation between pools, keep separately for each pool.\n Overflow is OK, as we only want difference between\n its current value and some recent.\n\n Both malloc and free should increment logical time, as in\n a different case multiple cached blocks would have same age,\n and accuracy of predictors suffers.\n */\n std::atomic cacheCurrTime;\n\n // Memory pool that owns this LargeObjectCache.\n // strict 1:1 relation, never changed\n ExtMemoryPool *extMemPool;\n\n // Returns artificial bin index,\n // it's used only during sorting and never saved\n static int sizeToIdx(size_t size);\n\n // Our friends\n friend class Backend;\n\npublic:\n void init(ExtMemoryPool *memPool);\n\n // Item accessors\n void put(LargeMemoryBlock *largeBlock);\n void putList(LargeMemoryBlock *head);\n LargeMemoryBlock *get(size_t size);\n\n void updateCacheState(DecreaseOrIncrease op, size_t size);\n bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime);\n\n // Cleanup operations\n bool doCleanup(uintptr_t currTime, bool doThreshDecr);\n bool decreasingCleanup();\n bool regularCleanup();\n bool cleanAll();\n void reset();\n\n void reportStat(FILE *f);\n#if __TBB_MALLOC_WHITEBOX_TEST\n size_t getLOCSize() const;\n size_t getUsedSize() const;\n#endif\n\n // Cache deals with exact-fit sizes, so need to align each size\n // to the specific bin when put object to cache\n static size_t alignToBin(size_t size);\n\n void setHugeSizeThreshold(size_t value);\n\n // Check if we should cache or sieve this size\n bool sizeInCacheRange(size_t size);\n\n uintptr_t getCurrTimeRange(uintptr_t range);\n void registerRealloc(size_t oldSize, size_t newSize);\n};\n\n#endif // __TBB_large_objects_H\n\n" }, { "path": "src/tbb/src/tbbmalloc/shared_utils.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_shared_utils_H\n#define __TBB_shared_utils_H\n\n// Include files containing declarations of intptr_t and uintptr_t\n#include // size_t\n#if _MSC_VER\ntypedef unsigned __int16 uint16_t;\ntypedef unsigned __int32 uint32_t;\ntypedef unsigned __int64 uint64_t;\n #if !UINTPTR_MAX\n #define UINTPTR_MAX SIZE_MAX\n #endif\n#else // _MSC_VER\n#include \n#endif\n\n/*\n * Functions to align an integer down or up to the given power of two,\n * and test for such an alignment, and for power of two.\n */\ntemplate\nstatic inline T alignDown(T arg, uintptr_t alignment) {\n return T( (uintptr_t)arg & ~(alignment-1));\n}\ntemplate\nstatic inline T alignUp (T arg, uintptr_t alignment) {\n return T(((uintptr_t)arg+(alignment-1)) & ~(alignment-1));\n // /*is this better?*/ return (((uintptr_t)arg-1) | (alignment-1)) + 1;\n}\ntemplate // works for not power-of-2 alignments\nstatic inline T alignUpGeneric(T arg, uintptr_t alignment) {\n if (size_t rem = arg % alignment) {\n arg += alignment - rem;\n }\n return arg;\n}\n\n/*\n * Compile time Log2 calculation\n */\ntemplate \nstruct Log2 { static const int value = 1 + Log2<(NUM >> 1)>::value; };\ntemplate <>\nstruct Log2<1> { static const int value = 0; };\n\n#if defined(min)\n#undef min\n#endif\n\ntemplate\nT min ( const T& val1, const T& val2 ) {\n return val1 < val2 ? val1 : val2;\n}\n\n/*\n * Functions to parse files information (system files for example)\n */\n\n#include \n\n#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER)\n // Suppress overzealous compiler warnings that default ctor and assignment\n // operator cannot be generated and object 'class' can never be instantiated.\n // #pragma warning(push)\n // #pragma warning(disable:4510 4512 4610)\n#endif\n\n#if __SUNPRO_CC\n // Suppress overzealous compiler warnings that a class with a reference member\n // lacks a user-defined constructor, which can lead to errors\n #pragma error_messages (off, refmemnoconstr)\n#endif\n\n// TODO: add a constructor to remove warnings suppression\nstruct parseFileItem {\n const char* format;\n long long& value;\n};\n\n#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER)\n // #pragma warning(pop)\n#endif\n\n#if __SUNPRO_CC\n #pragma error_messages (on, refmemnoconstr)\n#endif\n\ntemplate \nvoid parseFile(const char* file, const parseFileItem (&items)[N]) {\n // Tries to find all items in each line\n int found[N] = { 0 };\n // If all items found, stop forward file reading\n int numFound = 0;\n // Line storage\n char buf[BUF_LINE_SIZE];\n\n if (FILE *f = fopen(file, \"r\")) {\n while (numFound < N && fgets(buf, BUF_LINE_SIZE, f)) {\n for (int i = 0; i < N; ++i) {\n if (!found[i] && 1 == sscanf(buf, items[i].format, &items[i].value)) {\n ++numFound;\n found[i] = 1;\n }\n }\n }\n fclose(f);\n }\n}\n\nnamespace rml {\nnamespace internal {\n\n/*\n * Best estimate of cache line size, for the purpose of avoiding false sharing.\n * Too high causes memory overhead, too low causes false-sharing overhead.\n * Because, e.g., 32-bit code might run on a 64-bit system with a larger cache line size,\n * it would probably be better to probe at runtime where possible and/or allow for an environment variable override,\n * but currently this is still used for compile-time layout of class Block, so the change is not entirely trivial.\n */\n#if __powerpc64__ || __ppc64__ || __bgp__\nconst uint32_t estimatedCacheLineSize = 128;\n#else\nconst uint32_t estimatedCacheLineSize = 64;\n#endif\n\n} // namespace internal\n} // namespace rml\n\n#endif /* __TBB_shared_utils_H */\n\n" }, { "path": "src/tbb/src/tbbmalloc/tbbmalloc.cpp", "content": "/*\n Copyright (c) 2005-2023 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"TypeDefinitions.h\" // Customize.h and proxy.h get included\n#include \"tbbmalloc_internal_api.h\"\n\n#include \"../tbb/assert_impl.h\" // Out-of-line TBB assertion handling routines are instantiated here.\n#include \"oneapi/tbb/version.h\"\n#include \"oneapi/tbb/scalable_allocator.h\"\n\n#undef UNICODE\n\n#if USE_PTHREAD\n#include // dlopen\n#elif USE_WINTHREAD\n#include \n#endif\n\nnamespace rml {\nnamespace internal {\n\n#if TBB_USE_DEBUG\n#define DEBUG_SUFFIX \"_debug\"\n#else\n#define DEBUG_SUFFIX\n#endif /* TBB_USE_DEBUG */\n\n// MALLOCLIB_NAME is the name of the oneTBB memory allocator library.\n#if _WIN32||_WIN64\n#define MALLOCLIB_NAME \"tbbmalloc\" DEBUG_SUFFIX \".dll\"\n#elif __APPLE__\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX \".2.dylib\"\n#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX \".so\"\n#elif __unix__\n#define MALLOCLIB_NAME \"libtbbmalloc\" DEBUG_SUFFIX __TBB_STRING(.so.2)\n#else\n#error Unknown OS\n#endif\n\nvoid init_tbbmalloc() {\n#if __TBB_USE_ITT_NOTIFY\n MallocInitializeITT();\n#endif\n\n/* Preventing TBB allocator library from unloading to prevent\n resource leak, as memory is not released on the library unload.\n*/\n#if USE_WINTHREAD && !__TBB_SOURCE_DIRECTLY_INCLUDED && !__TBB_WIN8UI_SUPPORT\n // Prevent Windows from displaying message boxes if it fails to load library\n UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS);\n HMODULE lib;\n BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS\n |GET_MODULE_HANDLE_EX_FLAG_PIN,\n (LPCTSTR)&scalable_malloc, &lib);\n MALLOC_ASSERT(lib && ret, \"Allocator can't find itself.\");\n tbb::detail::suppress_unused_warning(ret);\n SetErrorMode (prev_mode);\n#endif /* USE_PTHREAD && !__TBB_SOURCE_DIRECTLY_INCLUDED */\n}\n\n#if !__TBB_SOURCE_DIRECTLY_INCLUDED\n#if USE_WINTHREAD\nextern \"C\" BOOL WINAPI DllMain( HINSTANCE /*hInst*/, DWORD callReason, LPVOID lpvReserved)\n{\n if (callReason==DLL_THREAD_DETACH)\n {\n __TBB_mallocThreadShutdownNotification();\n }\n else if (callReason==DLL_PROCESS_DETACH)\n {\n __TBB_mallocProcessShutdownNotification(lpvReserved != nullptr);\n }\n return TRUE;\n}\n#else /* !USE_WINTHREAD */\nstruct RegisterProcessShutdownNotification {\n// Work around non-reentrancy in dlopen() on Android\n RegisterProcessShutdownNotification() {\n // prevents unloading, POSIX case\n\n // We need better support for the library pinning\n // when dlopen can't find TBBmalloc library.\n // for example: void *ret = dlopen(MALLOCLIB_NAME, RTLD_NOW);\n // MALLOC_ASSERT(ret, \"Allocator can't load itself.\");\n dlopen(MALLOCLIB_NAME, RTLD_NOW);\n }\n\n RegisterProcessShutdownNotification(RegisterProcessShutdownNotification&) = delete;\n RegisterProcessShutdownNotification& operator=(const RegisterProcessShutdownNotification&) = delete;\n\n ~RegisterProcessShutdownNotification() {\n __TBB_mallocProcessShutdownNotification(false);\n }\n};\n\nstatic RegisterProcessShutdownNotification reg;\n#endif /* !USE_WINTHREAD */\n#endif /* !__TBB_SOURCE_DIRECTLY_INCLUDED */\n\n} } // namespaces\n\n" }, { "path": "src/tbb/src/tbbmalloc/tbbmalloc.rc", "content": "// Copyright (c) 2005-2024 Intel Corporation\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Includes\n//\n#include \n#include \"../../include/oneapi/tbb/version.h\"\n\n/////////////////////////////////////////////////////////////////////////////\n// Neutral resources\n\n#ifdef _WIN32\nLANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL\n#pragma code_page(1252)\n#endif //_WIN32\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Version\n//\n#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH\n#define TBB_VERSION TBB_VERSION_STRING\n\nVS_VERSION_INFO VERSIONINFO\n FILEVERSION TBB_VERNUMBERS\n PRODUCTVERSION TBB_VERNUMBERS\n FILEFLAGSMASK 0x17L\n#ifdef _DEBUG\n FILEFLAGS 0x1L\n#else\n FILEFLAGS 0x0L\n#endif\n FILEOS 0x40004L\n FILETYPE 0x2L\n FILESUBTYPE 0x0L\nBEGIN\n BLOCK \"StringFileInfo\"\n BEGIN\n BLOCK \"000004b0\"\n BEGIN\n VALUE \"CompanyName\", \"Intel Corporation\\0\"\n VALUE \"FileDescription\", \"oneAPI Threading Building Blocks (oneTBB) library\\0\"\n VALUE \"FileVersion\", TBB_VERSION \"\\0\"\n VALUE \"LegalCopyright\", \"Copyright 2005-2024 Intel Corporation. All Rights Reserved.\\0\"\n VALUE \"LegalTrademarks\", \"\\0\"\n#ifndef TBB_USE_DEBUG\n VALUE \"OriginalFilename\", \"tbbmalloc.dll\\0\"\n#else\n VALUE \"OriginalFilename\", \"tbbmalloc_debug.dll\\0\"\n#endif\n VALUE \"ProductName\", \"oneAPI Threading Building Blocks (oneTBB)\\0\"\n VALUE \"ProductVersion\", TBB_VERSION \"\\0\"\n VALUE \"PrivateBuild\", \"\\0\"\n VALUE \"SpecialBuild\", \"\\0\"\n END\n END\n BLOCK \"VarFileInfo\"\n BEGIN\n VALUE \"Translation\", 0x0, 1200\n END\nEND\n" }, { "path": "src/tbb/src/tbbmalloc/tbbmalloc_internal.h", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbbmalloc_internal_H\n#define __TBB_tbbmalloc_internal_H\n\n#include \"TypeDefinitions.h\" /* Also includes customization layer Customize.h */\n\n#if USE_PTHREAD\n // Some pthreads documentation says that must be first header.\n #include \n typedef pthread_key_t tls_key_t;\n#elif USE_WINTHREAD\n #include \n typedef DWORD tls_key_t;\n#else\n #error Must define USE_PTHREAD or USE_WINTHREAD\n#endif\n\n#include \n\n// TODO: *BSD also has it\n#define BACKEND_HAS_MREMAP __linux__\n#define CHECK_ALLOCATION_RANGE MALLOC_DEBUG || MALLOC_ZONE_OVERLOAD_ENABLED || MALLOC_UNIXLIKE_OVERLOAD_ENABLED\n\n#include \"oneapi/tbb/detail/_config.h\" // for __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN\n#include \"oneapi/tbb/detail/_template_helpers.h\"\n#if __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN\n #define _EXCEPTION_PTR_H /* prevents exception_ptr.h inclusion */\n #define _GLIBCXX_NESTED_EXCEPTION_H /* prevents nested_exception.h inclusion */\n#endif\n\n#include \n#include \n#include // for CHAR_BIT\n#include // for memset\n#if MALLOC_CHECK_RECURSION\n#include /* for placement new */\n#endif\n#include \"oneapi/tbb/scalable_allocator.h\"\n#include \"tbbmalloc_internal_api.h\"\n\n/********* Various compile-time options **************/\n\n#if !__TBB_DEFINE_MIC && __TBB_MIC_NATIVE\n #error Intel(R) Many Integrated Core Compiler does not define __MIC__ anymore.\n#endif\n\n#define MALLOC_TRACE 0\n\n#if MALLOC_TRACE\n#define TRACEF(x) printf x\n#else\n#define TRACEF(x) ((void)0)\n#endif /* MALLOC_TRACE */\n\n#define ASSERT_TEXT nullptr\n\n#define COLLECT_STATISTICS ( MALLOC_DEBUG && MALLOCENV_COLLECT_STATISTICS )\n#ifndef USE_INTERNAL_TID\n#define USE_INTERNAL_TID COLLECT_STATISTICS || MALLOC_TRACE\n#endif\n\n#include \"Statistics.h\"\n\n// call yield for whitebox testing, skip in real library\n#ifndef WhiteboxTestingYield\n#define WhiteboxTestingYield() ((void)0)\n#endif\n\n\n/********* End compile-time options **************/\n\nnamespace rml {\n\nnamespace internal {\n\n#if __TBB_MALLOC_LOCACHE_STAT\nextern intptr_t mallocCalls, cacheHits;\nextern intptr_t memAllocKB, memHitKB;\n#endif\n\n//! Utility template function to prevent \"unused\" warnings by various compilers.\ntemplate\nvoid suppress_unused_warning( const T& ) {}\n\n/********** Various global default constants ********/\n\n/*\n * Default huge page size\n */\n#if defined __loongarch64\nstatic const size_t HUGE_PAGE_SIZE = 32 * 1024 * 1024;\n#else\nstatic const size_t HUGE_PAGE_SIZE = 2 * 1024 * 1024;\n#endif\n\n/********** End of global default constants *********/\n\n/********** Various numeric parameters controlling allocations ********/\n\n/*\n * slabSize - the size of a block for allocation of small objects,\n * it must be larger than maxSegregatedObjectSize.\n */\nconst uintptr_t slabSize = 16*1024;\n\n/*\n * Large blocks cache cleanup frequency.\n * It should be power of 2 for the fast checking.\n */\nconst unsigned cacheCleanupFreq = 256;\n\n/*\n * Alignment of large (>= minLargeObjectSize) objects.\n */\nconst size_t largeObjectAlignment = estimatedCacheLineSize;\n\n/*\n * This number of bins in the TLS that leads to blocks that we can allocate in.\n */\nconst uint32_t numBlockBinLimit = 31;\n\n/********** End of numeric parameters controlling allocations *********/\n\nclass BlockI;\nclass Block;\nstruct LargeMemoryBlock;\nstruct ExtMemoryPool;\nstruct MemRegion;\nclass FreeBlock;\nclass TLSData;\nclass Backend;\nclass MemoryPool;\nstruct CacheBinOperation;\nextern const uint32_t minLargeObjectSize;\n\nenum DecreaseOrIncrease {\n decrease, increase\n};\n\nclass TLSKey {\n tls_key_t TLS_pointer_key;\npublic:\n bool init();\n bool destroy();\n TLSData* getThreadMallocTLS() const;\n void setThreadMallocTLS( TLSData * newvalue );\n TLSData* createTLS(MemoryPool *memPool, Backend *backend);\n};\n\ntemplate\ninline void AtomicUpdate(std::atomic& location, Arg newVal, const Compare &cmp)\n{\n static_assert(sizeof(Arg) == sizeof(intptr_t), \"Type of argument must match AtomicCompareExchange type.\");\n Arg old = location.load(std::memory_order_acquire);\n for (; cmp(old, newVal); ) {\n if (location.compare_exchange_strong(old, newVal))\n break;\n // TODO: do we need backoff after unsuccessful CAS?\n //old = val;\n }\n}\n\n// TODO: make BitMaskBasic more general\n// TODO: check that BitMaskBasic is not used for synchronization\n// (currently, it fits BitMaskMin well, but not as suitable for BitMaskMax)\ntemplate\nclass BitMaskBasic {\n static const unsigned SZ = (NUM-1)/(CHAR_BIT*sizeof(uintptr_t))+1;\n static const unsigned WORD_LEN = CHAR_BIT*sizeof(uintptr_t);\n\n std::atomic mask[SZ];\n\nprotected:\n void set(size_t idx, bool val) {\n MALLOC_ASSERT(idx\nclass BitMaskMin : public BitMaskBasic {\npublic:\n void set(size_t idx, bool val) { BitMaskBasic::set(idx, val); }\n int getMinTrue(unsigned startIdx) const {\n return BitMaskBasic::getMinTrue(startIdx);\n }\n};\n\ntemplate\nclass BitMaskMax : public BitMaskBasic {\npublic:\n void set(size_t idx, bool val) {\n MALLOC_ASSERT(NUM >= idx + 1, ASSERT_TEXT);\n\n BitMaskBasic::set(NUM - 1 - idx, val);\n }\n int getMaxTrue(unsigned startIdx) const {\n MALLOC_ASSERT(NUM >= startIdx + 1, ASSERT_TEXT);\n\n int p = BitMaskBasic::getMinTrue(NUM-startIdx-1);\n return -1==p? -1 : (int)NUM - 1 - p;\n }\n};\n\n\n// The part of thread-specific data that can be modified by other threads.\n// Such modifications must be protected by AllLocalCaches::listLock.\nstruct TLSRemote {\n TLSRemote *next,\n *prev;\n};\n\n// The list of all thread-local data; supporting cleanup of thread caches\nclass AllLocalCaches {\n TLSRemote *head;\n MallocMutex listLock; // protects operations in the list\npublic:\n void registerThread(TLSRemote *tls);\n void unregisterThread(TLSRemote *tls);\n bool cleanup(bool cleanOnlyUnused);\n void markUnused();\n void reset() { head = nullptr; }\n};\n\nclass LifoList {\npublic:\n inline LifoList();\n inline void push(Block *block);\n inline Block *pop();\n inline Block *grab();\n\nprivate:\n std::atomic top;\n MallocMutex lock;\n};\n\n/*\n * When a block that is not completely free is returned for reuse by other threads\n * this is where the block goes.\n *\n * LifoList assumes zero initialization; so below its constructors are omitted,\n * to avoid linking with C++ libraries on Linux.\n */\n\nclass OrphanedBlocks {\n LifoList bins[numBlockBinLimit];\npublic:\n Block *get(TLSData *tls, unsigned int size);\n void put(intptr_t binTag, Block *block);\n void reset();\n bool cleanup(Backend* backend);\n};\n\n/* Large objects entities */\n#include \"large_objects.h\"\n\n// select index size for BackRefMain based on word size: default is uint32_t,\n// uint16_t for 32-bit platforms\ntemplate\nstruct MainIndexSelect {\n typedef uint32_t main_type;\n};\n\ntemplate<>\nstruct MainIndexSelect {\n typedef uint16_t main_type;\n};\n\nclass BackRefIdx { // composite index to backreference array\npublic:\n typedef MainIndexSelect<4 < sizeof(uintptr_t)>::main_type main_t;\nprivate:\n static const main_t invalid = ~main_t(0);\n main_t main; // index in BackRefMain\n uint16_t largeObj:1; // is this object \"large\"?\n uint16_t offset :15; // offset from beginning of BackRefBlock\npublic:\n BackRefIdx() : main(invalid), largeObj(0), offset(0) {}\n bool isInvalid() const { return main == invalid; }\n bool isLargeObject() const { return largeObj; }\n main_t getMain() const { return main; }\n uint16_t getOffset() const { return offset; }\n\n#if __TBB_USE_THREAD_SANITIZER\n friend\n __attribute__((no_sanitize(\"thread\")))\n BackRefIdx dereference(const BackRefIdx* ptr) {\n BackRefIdx idx;\n idx.main = ptr->main;\n idx.largeObj = ptr->largeObj;\n idx.offset = ptr->offset;\n return idx;\n }\n#else\n friend\n BackRefIdx dereference(const BackRefIdx* ptr) {\n return *ptr;\n }\n#endif\n\n // only newBackRef can modify BackRefIdx\n static BackRefIdx newBackRef(bool largeObj);\n};\n\n// Block header is used during block coalescing\n// and must be preserved in used blocks.\nclass BlockI {\n#if __clang__ && !__INTEL_COMPILER\n // #pragma clang diagnostic push\n // #pragma clang diagnostic ignored \"-Wunused-private-field\"\n#endif\n intptr_t blockState[2];\n#if __clang__ && !__INTEL_COMPILER\n // #pragma clang diagnostic pop // \"-Wunused-private-field\"\n#endif\n};\n\nstruct LargeMemoryBlock : public BlockI {\n MemoryPool *pool; // owner pool\n LargeMemoryBlock *next, // ptrs in list of cached blocks\n *prev,\n // 2-linked list of pool's large objects\n // Used to destroy backrefs on pool destroy (backrefs are global)\n // and for object releasing during pool reset.\n *gPrev,\n *gNext;\n uintptr_t age; // age of block while in cache\n size_t objectSize; // the size requested by a client\n size_t unalignedSize; // the size requested from backend\n BackRefIdx backRefIdx; // cached here, used copy is in LargeObjectHdr\n};\n\n// Classes and methods for backend.cpp\n#include \"backend.h\"\n\n// An TBB allocator mode that can be controlled by user\n// via API/environment variable. Must be placed in zero-initialized memory.\n// External synchronization assumed.\n// TODO: TBB_VERSION support\nclass AllocControlledMode {\n intptr_t val;\n bool setDone;\n\npublic:\n intptr_t get() const {\n MALLOC_ASSERT(setDone, ASSERT_TEXT);\n return val;\n }\n\n // Note: set() can be called before init()\n void set(intptr_t newVal) {\n val = newVal;\n setDone = true;\n }\n\n bool ready() const {\n return setDone;\n }\n\n // envName - environment variable to get controlled mode\n void initReadEnv(const char *envName, intptr_t defaultVal) {\n if (!setDone) {\n // unreferenced formal parameter warning\n tbb::detail::suppress_unused_warning(envName);\n#if !__TBB_WIN8UI_SUPPORT\n // TODO: use strtol to get the actual value of the envirable\n const char *envVal = getenv(envName);\n if (envVal && !strcmp(envVal, \"1\"))\n val = 1;\n else\n#endif\n val = defaultVal;\n setDone = true;\n }\n }\n};\n\n// Page type to be used inside MapMemory.\n// Regular (4KB aligned), Huge and Transparent Huge Pages (2MB aligned).\nenum PageType {\n REGULAR = 0,\n PREALLOCATED_HUGE_PAGE,\n TRANSPARENT_HUGE_PAGE\n};\n\n// init() and printStatus() is called only under global initialization lock.\n// Race is possible between registerAllocation() and registerReleasing(),\n// harm is that up to single huge page releasing is missed (because failure\n// to get huge page is registered only 1st time), that is negligible.\n// setMode is also can be called concurrently.\n// Object must reside in zero-initialized memory\n// TODO: can we check for huge page presence during every 10th mmap() call\n// in case huge page is released by another process?\nclass HugePagesStatus {\nprivate:\n AllocControlledMode requestedMode; // changed only by user\n // to keep enabled and requestedMode consistent\n MallocMutex setModeLock;\n size_t pageSize;\n std::atomic needActualStatusPrint;\n\n static void doPrintStatus(bool state, const char *stateName) {\n // Under macOS* fprintf/snprintf acquires an internal lock, so when\n // 1st allocation is done under the lock, we got a deadlock.\n // Do not use fprintf etc during initialization.\n fputs(\"TBBmalloc: huge pages\\t\", stderr);\n if (!state)\n fputs(\"not \", stderr);\n fputs(stateName, stderr);\n fputs(\"\\n\", stderr);\n }\n\n void parseSystemMemInfo() {\n bool hpAvailable = false;\n bool thpAvailable = false;\n long long hugePageSize = -1;\n\n#if __unix__\n // Check huge pages existence\n long long meminfoHugePagesTotal = 0;\n\n parseFileItem meminfoItems[] = {\n // Parse system huge page size\n { \"Hugepagesize: %lld kB\", hugePageSize },\n // Check if there are preallocated huge pages on the system\n // https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt\n { \"HugePages_Total: %lld\", meminfoHugePagesTotal } };\n\n parseFile(\"/proc/meminfo\", meminfoItems);\n\n // Double check another system information regarding preallocated\n // huge pages if there are no information in /proc/meminfo\n long long vmHugePagesTotal = 0;\n\n parseFileItem vmItem[] = { { \"%lld\", vmHugePagesTotal } };\n\n // We parse a counter number, it can't be huge\n parseFile(\"/proc/sys/vm/nr_hugepages\", vmItem);\n\n if (hugePageSize > -1 && (meminfoHugePagesTotal > 0 || vmHugePagesTotal > 0)) {\n MALLOC_ASSERT(hugePageSize != 0, \"Huge Page size can't be zero if we found preallocated.\");\n\n // Any non zero value clearly states that there are preallocated\n // huge pages on the system\n hpAvailable = true;\n }\n\n // Check if there is transparent huge pages support on the system\n long long thpPresent = 'n';\n parseFileItem thpItem[] = { { \"[alwa%cs] madvise never\\n\", thpPresent } };\n parseFile(\"/sys/kernel/mm/transparent_hugepage/enabled\", thpItem);\n\n if (hugePageSize > -1 && thpPresent == 'y') {\n MALLOC_ASSERT(hugePageSize != 0, \"Huge Page size can't be zero if we found thp existence.\");\n thpAvailable = true;\n }\n#endif\n MALLOC_ASSERT(!pageSize, \"Huge page size can't be set twice. Double initialization.\");\n\n // Initialize object variables\n if (hugePageSize > -1) {\n pageSize = hugePageSize * 1024; // was read in KB from meminfo\n } else {\n pageSize = 0;\n }\n isHPAvailable = hpAvailable;\n isTHPAvailable = thpAvailable;\n }\n\npublic:\n\n // System information\n bool isHPAvailable;\n bool isTHPAvailable;\n\n // User defined value\n bool isEnabled;\n\n void init() {\n parseSystemMemInfo();\n MallocMutex::scoped_lock lock(setModeLock);\n requestedMode.initReadEnv(\"TBB_MALLOC_USE_HUGE_PAGES\", 0);\n isEnabled = (isHPAvailable || isTHPAvailable) && requestedMode.get();\n }\n\n // Could be set from user code at any place.\n // If we didn't call init() at this place, isEnabled will be false\n void setMode(intptr_t newVal) {\n MallocMutex::scoped_lock lock(setModeLock);\n requestedMode.set(newVal);\n isEnabled = (isHPAvailable || isTHPAvailable) && newVal;\n }\n\n void reset() {\n needActualStatusPrint.store(0, std::memory_order_relaxed);\n pageSize = 0;\n isEnabled = isHPAvailable = isTHPAvailable = false;\n }\n\n // If memory mapping size is a multiple of huge page size, some OS kernels\n // can use huge pages transparently. Use this when huge pages are requested.\n size_t getGranularity() const {\n if (requestedMode.ready())\n return requestedMode.get() ? pageSize : 0;\n else\n return HUGE_PAGE_SIZE; // the mode is not yet known; assume typical 2MB huge pages\n }\n\n void printStatus() {\n doPrintStatus(requestedMode.get(), \"requested\");\n if (requestedMode.get()) { // report actual status iff requested\n if (pageSize)\n needActualStatusPrint.store(1, std::memory_order_release);\n else\n doPrintStatus(/*state=*/false, \"available\");\n }\n }\n};\n\nclass AllLargeBlocksList {\n MallocMutex largeObjLock;\n LargeMemoryBlock *loHead;\npublic:\n void add(LargeMemoryBlock *lmb);\n void remove(LargeMemoryBlock *lmb);\n template void releaseAll(Backend *backend);\n};\n\nstruct ExtMemoryPool {\n Backend backend;\n LargeObjectCache loc;\n AllLocalCaches allLocalCaches;\n OrphanedBlocks orphanedBlocks;\n\n intptr_t poolId;\n // To find all large objects. Used during user pool destruction,\n // to release all backreferences in large blocks (slab blocks do not have them).\n AllLargeBlocksList lmbList;\n // Callbacks to be used instead of MapMemory/UnmapMemory.\n rawAllocType rawAlloc;\n rawFreeType rawFree;\n size_t granularity;\n bool keepAllMemory,\n delayRegsReleasing,\n // TODO: implements fixedPool with calling rawFree on destruction\n fixedPool;\n TLSKey tlsPointerKey; // per-pool TLS key\n\n std::atomic softCachesCleanupInProgress;\n std::atomic hardCachesCleanupInProgress;\n\n bool init(intptr_t poolId, rawAllocType rawAlloc, rawFreeType rawFree,\n size_t granularity, bool keepAllMemory, bool fixedPool);\n bool initTLS();\n\n // i.e., not system default pool for scalable_malloc/scalable_free\n bool userPool() const { return rawAlloc; }\n\n // true if something has been released\n bool softCachesCleanup();\n bool releaseAllLocalCaches();\n bool hardCachesCleanup(bool wait);\n void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment);\n bool reset() {\n loc.reset();\n allLocalCaches.reset();\n orphanedBlocks.reset();\n bool ret = tlsPointerKey.destroy();\n backend.reset();\n return ret;\n }\n bool destroy() {\n MALLOC_ASSERT(isPoolValid(),\n \"Possible double pool_destroy or heap corruption\");\n if (!userPool()) {\n loc.reset();\n allLocalCaches.reset();\n }\n // pthread_key_dtors must be disabled before memory unmapping\n // TODO: race-free solution\n bool ret = tlsPointerKey.destroy();\n if (rawFree || !userPool())\n ret &= backend.destroy();\n // pool is not valid after this point\n granularity = 0;\n return ret;\n }\n void delayRegionsReleasing(bool mode) { delayRegsReleasing = mode; }\n inline bool regionsAreReleaseable() const;\n\n LargeMemoryBlock *mallocLargeObject(MemoryPool *pool, size_t allocationSize);\n void freeLargeObject(LargeMemoryBlock *lmb);\n void freeLargeObjectList(LargeMemoryBlock *head);\n#if MALLOC_DEBUG\n // use granulatity as marker for pool validity\n bool isPoolValid() const { return granularity; }\n#endif\n};\n\ninline bool Backend::inUserPool() const { return extMemPool->userPool(); }\n\nstruct LargeObjectHdr {\n LargeMemoryBlock *memoryBlock;\n /* Backreference points to LargeObjectHdr.\n Duplicated in LargeMemoryBlock to reuse in subsequent allocations. */\n BackRefIdx backRefIdx;\n};\n\nstruct FreeObject {\n FreeObject *next;\n};\n\n\n/******* A helper class to support overriding malloc with scalable_malloc *******/\n#if MALLOC_CHECK_RECURSION\n\nclass RecursiveMallocCallProtector {\n // pointer to an automatic data of holding thread\n static std::atomic autoObjPtr;\n static MallocMutex rmc_mutex;\n static std::atomic owner_thread;\n/* Under FreeBSD 8.0 1st call to any pthread function including pthread_self\n leads to pthread initialization, that causes malloc calls. As 1st usage of\n RecursiveMallocCallProtector can be before pthread initialized, pthread calls\n can't be used in 1st instance of RecursiveMallocCallProtector.\n RecursiveMallocCallProtector is used 1st time in checkInitialization(),\n so there is a guarantee that on 2nd usage pthread is initialized.\n No such situation observed with other supported OSes.\n */\n#if __FreeBSD__\n static bool canUsePthread;\n#else\n static const bool canUsePthread = true;\n#endif\n/*\n The variable modified in checkInitialization,\n so can be read without memory barriers.\n */\n static bool mallocRecursionDetected;\n\n MallocMutex::scoped_lock* lock_acquired;\n char scoped_lock_space[sizeof(MallocMutex::scoped_lock)+1];\n \npublic:\n RecursiveMallocCallProtector() : lock_acquired(nullptr) {\n lock_acquired = new (scoped_lock_space) MallocMutex::scoped_lock( rmc_mutex );\n if (canUsePthread)\n owner_thread.store(pthread_self(), std::memory_order_relaxed);\n autoObjPtr.store(&scoped_lock_space, std::memory_order_relaxed);\n }\n\n RecursiveMallocCallProtector(RecursiveMallocCallProtector&) = delete;\n RecursiveMallocCallProtector& operator=(RecursiveMallocCallProtector) = delete;\n\n ~RecursiveMallocCallProtector() {\n if (lock_acquired) {\n autoObjPtr.store(nullptr, std::memory_order_relaxed);\n lock_acquired->~scoped_lock();\n }\n }\n static bool sameThreadActive() {\n if (!autoObjPtr.load(std::memory_order_relaxed)) // fast path\n return false;\n // Some thread has an active recursive call protector; check if the current one.\n // Exact pthread_self based test\n if (canUsePthread) {\n if (pthread_equal( owner_thread.load(std::memory_order_relaxed), pthread_self() )) {\n mallocRecursionDetected = true;\n return true;\n } else\n return false;\n }\n // inexact stack size based test\n const uintptr_t threadStackSz = 2*1024*1024;\n int dummy;\n\n uintptr_t xi = (uintptr_t)autoObjPtr.load(std::memory_order_relaxed), yi = (uintptr_t)&dummy;\n uintptr_t diffPtr = xi > yi ? xi - yi : yi - xi;\n\n return diffPtr < threadStackSz;\n }\n\n/* The function is called on 1st scalable_malloc call to check if malloc calls\n scalable_malloc (nested call must set mallocRecursionDetected). */\n static void detectNaiveOverload() {\n if (!malloc_proxy) {\n#if __FreeBSD__\n/* If !canUsePthread, we can't call pthread_self() before, but now pthread\n is already on, so can do it. */\n if (!canUsePthread) {\n canUsePthread = true;\n owner_thread.store(pthread_self(), std::memory_order_relaxed);\n }\n#endif\n free(malloc(1));\n }\n }\n};\n\n#else\n\nclass RecursiveMallocCallProtector {\npublic:\n RecursiveMallocCallProtector() {}\n ~RecursiveMallocCallProtector() {}\n};\n\n#endif /* MALLOC_CHECK_RECURSION */\n\nunsigned int getThreadId();\n\nbool initBackRefMain(Backend *backend);\nvoid destroyBackRefMain(Backend *backend);\nvoid removeBackRef(BackRefIdx backRefIdx);\nvoid setBackRef(BackRefIdx backRefIdx, void *newPtr);\nvoid *getBackRef(BackRefIdx backRefIdx);\n\n} // namespace internal\n} // namespace rml\n\n#endif // __TBB_tbbmalloc_internal_H\n" }, { "path": "src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_tbbmalloc_internal_api_H\n#define __TBB_tbbmalloc_internal_api_H\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif /* __cplusplus */\n\ntypedef enum {\n /* Tune usage of source included allocator. Selected value is large enough\n to not intercept with constants from AllocationModeParam. */\n TBBMALLOC_INTERNAL_SOURCE_INCLUDED = 65536\n} AllocationModeInternalParam;\n\nvoid MallocInitializeITT();\nvoid __TBB_mallocProcessShutdownNotification(bool);\n#if _WIN32||_WIN64\nvoid __TBB_mallocThreadShutdownNotification();\n#endif\n\n#ifdef __cplusplus\n} /* extern \"C\" */\n#endif /* __cplusplus */\n\n#endif /* __TBB_tbbmalloc_internal_api_H */\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/CMakeLists.txt", "content": "# Copyright (c) 2020-2024 Intel Corporation\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif (NOT BUILD_SHARED_LIBS)\n return()\nendif()\n\nadd_library(tbbmalloc_proxy \n function_replacement.cpp\n proxy.cpp)\n\nif (WIN32)\n target_sources(tbbmalloc_proxy PRIVATE tbbmalloc_proxy.rc)\nendif()\n\nadd_library(TBB::tbbmalloc_proxy ALIAS tbbmalloc_proxy)\n\ntarget_compile_definitions(tbbmalloc_proxy\n PUBLIC\n $<$:TBB_USE_DEBUG>\n PRIVATE\n __TBBMALLOCPROXY_BUILD)\n\ntarget_include_directories(tbbmalloc_proxy\n PUBLIC\n $\n $)\n\nif (NOT APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL \"GNU\")\n # gcc 5.0 and later have -Wno-sized-deallocation options\n set(TBB_WARNING_SUPPRESS ${TBB_WARNING_SUPPRESS}\n $<$>:-Wno-sized-deallocation>)\nendif()\n\ntarget_compile_options(tbbmalloc_proxy\n PRIVATE\n ${TBB_CXX_STD_FLAG} # TODO: consider making it PUBLIC.\n ${TBB_MMD_FLAG}\n ${TBB_DSE_FLAG}\n ${TBB_WARNING_LEVEL}\n ${TBB_WARNING_SUPPRESS}\n ${TBB_LIB_COMPILE_FLAGS}\n ${TBB_COMMON_COMPILE_FLAGS}\n)\n\nif (UNIX AND NOT APPLE)\n # Avoid use of target_link_libraries here as it changes /DEF option to \\DEF on Windows.\n set_target_properties(tbbmalloc_proxy PROPERTIES\n LINK_FLAGS \"${TBB_LINK_DEF_FILE_FLAG}\\\"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-proxy.def\\\"\"\n LINK_DEPENDS \"${CMAKE_CURRENT_SOURCE_DIR}/def/${TBB_DEF_FILE_PREFIX}-proxy.def\"\n DEFINE_SYMBOL \"\")\nendif()\n\n# Prefer using target_link_options instead of target_link_libraries to specify link options because\n# target_link_libraries may incorrectly handle some options (on Windows, for example).\nif (COMMAND target_link_options)\n target_link_options(tbbmalloc_proxy\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nelse()\n target_link_libraries(tbbmalloc_proxy\n PRIVATE\n ${TBB_LIB_LINK_FLAGS}\n ${TBB_COMMON_LINK_FLAGS}\n )\nendif()\n\ntarget_link_libraries(tbbmalloc_proxy\n PRIVATE\n TBB::tbbmalloc\n Threads::Threads\n ${TBB_LIB_LINK_LIBS}\n ${TBB_COMMON_LINK_LIBS}\n)\n\nif(TBB_BUILD_APPLE_FRAMEWORKS)\n set_target_properties(tbbmalloc_proxy PROPERTIES \n FRAMEWORK TRUE\n FRAMEWORK_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER com.intel.tbbmalloc-proxy\n MACOSX_FRAMEWORK_IDENTIFIER com.intel.tbbmalloc-proxy\n MACOSX_FRAMEWORK_BUNDLE_VERSION ${TBBMALLOC_BINARY_VERSION}.${TBB_BINARY_MINOR_VERSION}\n MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${TBBMALLOC_BINARY_VERSION})\nendif()\n\ntbb_install_target(tbbmalloc_proxy)\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/def/lin32-proxy.def", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\ncalloc;\nfree;\nmalloc;\nrealloc;\nposix_memalign;\nmemalign;\naligned_alloc;\nvalloc;\npvalloc;\nmallinfo;\nmallopt;\nmalloc_usable_size;\n__libc_malloc;\n__libc_realloc;\n__libc_calloc;\n__libc_free;\n__libc_memalign;\n__libc_pvalloc;\n__libc_valloc;\n__TBB_malloc_proxy;\n_ZdaPv; /* next ones are new/delete */\n_ZdaPvRKSt9nothrow_t;\n_ZdlPv;\n_ZdlPvRKSt9nothrow_t;\n_Znaj;\n_ZnajRKSt9nothrow_t;\n_Znwj;\n_ZnwjRKSt9nothrow_t;\n\nlocal:\n\n/* TBB symbols */\n*3rml8internal*;\n*3tbb*;\n*__TBB*;\n\n};\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/def/lin64-proxy.def", "content": "/*\n Copyright (c) 2005-2021 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n{\nglobal:\ncalloc;\nfree;\nmalloc;\nrealloc;\nposix_memalign;\nmemalign;\naligned_alloc;\nvalloc;\npvalloc;\nmallinfo;\nmallopt;\nmalloc_usable_size;\n__libc_malloc;\n__libc_realloc;\n__libc_calloc;\n__libc_free;\n__libc_memalign;\n__libc_pvalloc;\n__libc_valloc;\n__TBB_malloc_proxy;\n_ZdaPv; /* next ones are new/delete */\n_ZdaPvRKSt9nothrow_t;\n_ZdlPv;\n_ZdlPvRKSt9nothrow_t;\n_Znam;\n_ZnamRKSt9nothrow_t;\n_Znwm;\n_ZnwmRKSt9nothrow_t;\n\nlocal:\n\n/* TBB symbols */\n*3rml8internal*;\n*3tbb*;\n*__TBB*;\n\n};\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/function_replacement.cpp", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/detail/_assert.h\"\n#include \"../tbb/assert_impl.h\"\n\n#if !__TBB_WIN8UI_SUPPORT && defined(_WIN32)\n\n#ifndef _CRT_SECURE_NO_DEPRECATE\n#define _CRT_SECURE_NO_DEPRECATE 1\n#endif\n\n// no standard-conforming implementation of snprintf prior to VS 2015\n#if !defined(_MSC_VER) || _MSC_VER>=1900\n#define LOG_PRINT(s, n, format, ...) snprintf(s, n, format, __VA_ARGS__)\n#else\n#define LOG_PRINT(s, n, format, ...) _snprintf_s(s, n, _TRUNCATE, format, __VA_ARGS__)\n#endif\n\n#include \n#include \n#include \n#include \n\n#include \"function_replacement.h\"\n\n// The information about a standard memory allocation function for the replacement log\nstruct FunctionInfo {\n const char* funcName;\n const char* dllName;\n};\n\n// Namespace that processes and manages the output of records to the Log journal\n// that will be provided to user by TBB_malloc_replacement_log()\nnamespace Log {\n // Value of RECORDS_COUNT is set due to the fact that we maximally\n // scan 8 modules, and in every module we can swap 6 opcodes. (rounded to 8)\n static const unsigned RECORDS_COUNT = 8 * 8;\n static const unsigned RECORD_LENGTH = MAX_PATH;\n\n // Need to add 1 to count of records, because last record must be always nullptr\n static char *records[RECORDS_COUNT + 1];\n static bool replacement_status = true;\n\n // Internal counter that contains number of next string for record\n static unsigned record_number = 0;\n\n // Function that writes info about (not)found opcodes to the Log journal\n // functionInfo - information about a standard memory allocation function for the replacement log\n // opcodeString - string, that contain byte code of this function\n // status - information about function replacement status\n static void record(FunctionInfo functionInfo, const char * opcodeString, bool status) {\n __TBB_ASSERT(functionInfo.dllName, \"Empty DLL name value\");\n __TBB_ASSERT(functionInfo.funcName, \"Empty function name value\");\n __TBB_ASSERT(opcodeString, \"Empty opcode\");\n __TBB_ASSERT(record_number <= RECORDS_COUNT, \"Incorrect record number\");\n\n //If some replacement failed -> set status to false\n replacement_status &= status;\n\n // If we reach the end of the log, write this message to the last line\n if (record_number == RECORDS_COUNT) {\n // %s - workaround to fix empty variable argument parsing behavior in GCC\n LOG_PRINT(records[RECORDS_COUNT - 1], RECORD_LENGTH, \"%s\", \"Log was truncated.\");\n return;\n }\n\n char* entry = (char*)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, RECORD_LENGTH);\n __TBB_ASSERT(entry, \"Invalid memory was returned\");\n\n LOG_PRINT(entry, RECORD_LENGTH, \"%s: %s (%s), byte pattern: <%s>\",\n status ? \"Success\" : \"Fail\", functionInfo.funcName, functionInfo.dllName, opcodeString);\n\n records[record_number++] = entry;\n }\n};\n\ninline UINT_PTR Ptr2Addrint(LPVOID ptr)\n{\n Int2Ptr i2p;\n i2p.lpv = ptr;\n return i2p.uip;\n}\n\ninline LPVOID Addrint2Ptr(UINT_PTR ptr)\n{\n Int2Ptr i2p;\n i2p.uip = ptr;\n return i2p.lpv;\n}\n\n// Is the distance between addr1 and addr2 smaller than dist\ninline bool IsInDistance(UINT_PTR addr1, UINT_PTR addr2, __int64 dist)\n{\n __int64 diff = addr1>addr2 ? addr1-addr2 : addr2-addr1;\n return diff= m_allocSize)\n {\n // Found a free region, try to allocate a page in this region\n void *newPage = VirtualAlloc(newAddr, m_allocSize, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);\n if (!newPage)\n break;\n\n // Add the new page to the pages database\n MemoryBuffer *pBuff = new (m_lastBuffer) MemoryBuffer(newPage, m_allocSize);\n ++m_lastBuffer;\n return pBuff;\n }\n }\n\n // Failed to find a buffer in the distance\n return 0;\n }\n\npublic:\n MemoryProvider()\n {\n SYSTEM_INFO sysInfo;\n GetSystemInfo(&sysInfo);\n m_allocSize = sysInfo.dwAllocationGranularity;\n m_lastBuffer = &m_pages[0];\n }\n\n // We can't free the pages in the destructor because the trampolines\n // are using these memory locations and a replaced function might be called\n // after the destructor was called.\n ~MemoryProvider()\n {\n }\n\n // Return a memory location in distance less than 2^31 from input address\n UINT_PTR GetLocation(UINT_PTR addr)\n {\n MemoryBuffer *pBuff = m_pages;\n for (; pBuffm_next, addr, MAX_DISTANCE); ++pBuff)\n {\n if (pBuff->m_next < pBuff->m_base + pBuff->m_size)\n {\n UINT_PTR loc = pBuff->m_next;\n pBuff->m_next += MAX_PROBE_SIZE;\n return loc;\n }\n }\n\n pBuff = CreateBuffer(addr);\n if(!pBuff)\n return 0;\n\n UINT_PTR loc = pBuff->m_next;\n pBuff->m_next += MAX_PROBE_SIZE;\n return loc;\n }\n\nprivate:\n MemoryBuffer m_pages[MAX_NUM_BUFFERS];\n MemoryBuffer *m_lastBuffer;\n DWORD m_allocSize;\n};\n\nstatic MemoryProvider memProvider;\n\n// Compare opcodes from dictionary (str1) and opcodes from code (str2)\n// str1 might contain '*' to mask addresses\n// RETURN: 0 if opcodes did not match, 1 on success\nsize_t compareStrings( const char *str1, const char *str2 )\n{\n for (size_t i=0; str1[i]!=0; i++){\n if( str1[i]!='*' && str1[i]!='#' && str1[i]!=str2[i] ) return 0;\n }\n return 1;\n}\n\n// Check function prologue with known prologues from the dictionary\n// opcodes - dictionary\n// inpAddr - pointer to function prologue\n// Dictionary contains opcodes for several full asm instructions\n// + one opcode byte for the next asm instruction for safe address processing\n// RETURN: 1 + the index of the matched pattern, or 0 if no match found.\nstatic UINT CheckOpcodes( const char ** opcodes, void *inpAddr, bool abortOnError, const FunctionInfo* functionInfo = nullptr)\n{\n static size_t opcodesStringsCount = 0;\n static size_t maxOpcodesLength = 0;\n static size_t opcodes_pointer = (size_t)opcodes;\n char opcodeString[2*MAX_PATTERN_SIZE+1];\n size_t i;\n size_t result = 0;\n\n // Get the values for static variables\n // max length and number of patterns\n if( !opcodesStringsCount || opcodes_pointer != (size_t)opcodes ){\n while( *(opcodes + opcodesStringsCount)!= nullptr ){\n if( (i=strlen(*(opcodes + opcodesStringsCount))) > maxOpcodesLength )\n maxOpcodesLength = i;\n opcodesStringsCount++;\n }\n opcodes_pointer = (size_t)opcodes;\n __TBB_ASSERT( maxOpcodesLength/2 <= MAX_PATTERN_SIZE, \"Pattern exceeded the limit of 28 opcodes/56 symbols\" );\n }\n\n // Translate prologue opcodes to string format to compare\n for( i=0; i= SIZE_OF_RELJUMP, \"Incorrect bytecode pattern?\" );\n UINT_PTR trampAddr = memProvider.GetLocation(srcAddr);\n if (!trampAddr)\n return 0;\n *storedAddr = Addrint2Ptr(trampAddr);\n // Set 'executable' flag for original instructions in the new place\n DWORD pageFlags = PAGE_EXECUTE_READWRITE;\n if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0;\n // Copy original instructions to the new place\n memcpy(*storedAddr, codePtr, bytesToMove);\n offset = srcAddr - trampAddr;\n offset32 = (UINT)(offset & 0xFFFFFFFF);\n CorrectOffset( trampAddr, pattern, offset32 );\n // Set jump to the code after replacement\n offset32 -= SIZE_OF_RELJUMP;\n *(UCHAR*)(trampAddr+bytesToMove) = 0xE9;\n memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32));\n }\n\n // The following will work correctly even if srcAddr>tgtAddr, as long as\n // address difference is less than 2^31, which is guaranteed by IsInDistance.\n offset = tgtAddr - srcAddr - SIZE_OF_RELJUMP;\n offset32 = (UINT)(offset & 0xFFFFFFFF);\n // Insert the jump to the new code\n *codePtr = 0xE9;\n memcpy(codePtr+1, &offset32, sizeof(offset32));\n\n // Fill the rest with NOPs to correctly see disassembler of old code in debugger.\n for( unsigned i=SIZE_OF_RELJUMP; i= SIZE_OF_INDJUMP, \"Incorrect bytecode pattern?\" );\n UINT_PTR trampAddr = memProvider.GetLocation(srcAddr);\n if (!trampAddr)\n return 0;\n *storedAddr = Addrint2Ptr(trampAddr);\n // Set 'executable' flag for original instructions in the new place\n DWORD pageFlags = PAGE_EXECUTE_READWRITE;\n if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0;\n // Copy original instructions to the new place\n memcpy(*storedAddr, codePtr, bytesToMove);\n offset = srcAddr - trampAddr;\n offset32 = (UINT)(offset & 0xFFFFFFFF);\n CorrectOffset( trampAddr, pattern, offset32 );\n // Set jump to the code after replacement. It is within the distance of relative jump!\n offset32 -= SIZE_OF_RELJUMP;\n *(UCHAR*)(trampAddr+bytesToMove) = 0xE9;\n memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32));\n }\n\n // Fill the buffer\n offset = location - srcAddr - SIZE_OF_INDJUMP;\n offset32 = (UINT)(offset & 0xFFFFFFFF);\n *(codePtr) = 0xFF;\n *(codePtr+1) = 0x25;\n memcpy(codePtr+2, &offset32, sizeof(offset32));\n\n // Fill the rest with NOPs to correctly see disassembler of old code in debugger.\n for( unsigned i=SIZE_OF_INDJUMP; i 0, \"abortOnError ignored in CheckOpcodes?\" );\n pattern = opcodes[opcodeIdx-1]; // -1 compensates for +1 in CheckOpcodes\n }\n }\n\n probeSize = InsertTrampoline32(inpAddr, targetAddr, pattern, origFunc);\n if (!probeSize)\n probeSize = InsertTrampoline64(inpAddr, targetAddr, pattern, origFunc);\n\n // Restore original protection\n VirtualProtect(inpAddr, MAX_PROBE_SIZE, origProt, &origProt);\n\n if (!probeSize)\n return FALSE;\n\n FlushInstructionCache(GetCurrentProcess(), inpAddr, probeSize);\n FlushInstructionCache(GetCurrentProcess(), origFunc, probeSize);\n\n return TRUE;\n}\n\n// Routine to replace the functions\n// TODO: replace opcodesNumber with opcodes and opcodes number to check if we replace right code.\nFRR_TYPE ReplaceFunctionA(const char *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc)\n{\n // Cache the results of the last search for the module\n // Assume that there was no DLL unload between\n static char cachedName[MAX_PATH+1];\n static HMODULE cachedHM = 0;\n\n if (!dllName || !*dllName)\n return FRR_NODLL;\n\n if (!cachedHM || strncmp(dllName, cachedName, MAX_PATH) != 0)\n {\n // Find the module handle for the input dll\n HMODULE hModule = GetModuleHandleA(dllName);\n if (hModule == 0)\n {\n // Couldn't find the module with the input name\n cachedHM = 0;\n return FRR_NODLL;\n }\n\n cachedHM = hModule;\n strncpy(cachedName, dllName, MAX_PATH);\n }\n\n FARPROC inpFunc = GetProcAddress(cachedHM, funcName);\n if (inpFunc == 0)\n {\n // Function was not found\n return FRR_NOFUNC;\n }\n\n if (!InsertTrampoline((void*)inpFunc, (void*)newFunc, opcodes, (void**)origFunc)){\n // Failed to insert the trampoline to the target address\n return FRR_FAILED;\n }\n\n return FRR_OK;\n}\n\nFRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc)\n{\n // Cache the results of the last search for the module\n // Assume that there was no DLL unload between\n static wchar_t cachedName[MAX_PATH+1];\n static HMODULE cachedHM = 0;\n\n if (!dllName || !*dllName)\n return FRR_NODLL;\n\n if (!cachedHM || wcsncmp(dllName, cachedName, MAX_PATH) != 0)\n {\n // Find the module handle for the input dll\n HMODULE hModule = GetModuleHandleW(dllName);\n if (hModule == 0)\n {\n // Couldn't find the module with the input name\n cachedHM = 0;\n return FRR_NODLL;\n }\n\n cachedHM = hModule;\n wcsncpy(cachedName, dllName, MAX_PATH);\n }\n\n FARPROC inpFunc = GetProcAddress(cachedHM, funcName);\n if (inpFunc == 0)\n {\n // Function was not found\n return FRR_NOFUNC;\n }\n\n if (!InsertTrampoline((void*)inpFunc, (void*)newFunc, opcodes, (void**)origFunc)){\n // Failed to insert the trampoline to the target address\n return FRR_FAILED;\n }\n\n return FRR_OK;\n}\n\nbool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module)\n{\n FARPROC inpFunc = GetProcAddress(module, funcName);\n FunctionInfo functionInfo = { funcName, dllName };\n\n if (!inpFunc) {\n Log::record(functionInfo, \"unknown\", /*status*/ false);\n return false;\n }\n\n return CheckOpcodes( opcodes, (void*)inpFunc, /*abortOnError=*/false, &functionInfo) != 0;\n}\n\n// Public Windows API\nextern \"C\" __declspec(dllexport) int TBB_malloc_replacement_log(char *** function_replacement_log_ptr)\n{\n if (function_replacement_log_ptr != nullptr) {\n *function_replacement_log_ptr = Log::records;\n }\n\n // If we have no logs -> return false status\n return Log::replacement_status && Log::records[0] != nullptr ? 0 : -1;\n}\n\n#endif /* !__TBB_WIN8UI_SUPPORT && defined(_WIN32) */\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/function_replacement.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#ifndef __TBB_function_replacement_H\n#define __TBB_function_replacement_H\n\n#include //for ptrdiff_t\ntypedef enum {\n FRR_OK, /* Succeeded in replacing the function */\n FRR_NODLL, /* The requested DLL was not found */\n FRR_NOFUNC, /* The requested function was not found */\n FRR_FAILED, /* The function replacement request failed */\n} FRR_TYPE;\n\ntypedef enum {\n FRR_FAIL, /* Required function */\n FRR_IGNORE, /* optional function */\n} FRR_ON_ERROR;\n\ntypedef void (*FUNCPTR)();\n\n#ifndef UNICODE\n#define ReplaceFunction ReplaceFunctionA\n#else\n#define ReplaceFunction ReplaceFunctionW\n#endif //UNICODE\n\nFRR_TYPE ReplaceFunctionA(const char *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=nullptr);\nFRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=nullptr);\n\nbool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module);\n\n// Utilities to convert between ADDRESS and LPVOID\nunion Int2Ptr {\n UINT_PTR uip;\n LPVOID lpv;\n};\n\ninline UINT_PTR Ptr2Addrint(LPVOID ptr);\ninline LPVOID Addrint2Ptr(UINT_PTR ptr);\n\n// The size of a trampoline region\nconst unsigned MAX_PROBE_SIZE = 32;\n\n// The size of a jump relative instruction \"e9 00 00 00 00\"\nconst unsigned SIZE_OF_RELJUMP = 5;\n\n// The size of jump RIP relative indirect \"ff 25 00 00 00 00\"\nconst unsigned SIZE_OF_INDJUMP = 6;\n\n// The size of address we put in the location (in Intel64)\nconst unsigned SIZE_OF_ADDRESS = 8;\n\n// The size limit (in bytes) for an opcode pattern to fit into a trampoline\n// There should be enough space left for a relative jump; +1 is for the extra pattern byte.\nconst unsigned MAX_PATTERN_SIZE = MAX_PROBE_SIZE - SIZE_OF_RELJUMP + 1;\n\n// The max distance covered in 32 bits: 2^31 - 1 - C\n// where C should not be smaller than the size of a probe.\n// The latter is important to correctly handle \"backward\" jumps.\nconst __int64 MAX_DISTANCE = (((__int64)1 << 31) - 1) - MAX_PROBE_SIZE;\n\n// The maximum number of distinct buffers in memory\nconst ptrdiff_t MAX_NUM_BUFFERS = 256;\n\n#endif //__TBB_function_replacement_H\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/proxy.cpp", "content": "/*\n Copyright (c) 2005-2024 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n#if __unix__ && !__ANDROID__\n// include indirectly so that is not included\n#include \n// include indirectly so that is not included\n#include \n// Working around compiler issue with Anaconda's gcc 7.3 compiler package.\n// New gcc ported for old libc may provide their inline implementation\n// of aligned_alloc as required by new C++ standard, this makes it hard to\n// redefine aligned_alloc here. However, running on systems with new libc\n// version, it still needs it to be redefined, thus tricking system headers\n#if defined(__GLIBC_PREREQ)\n#if !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC\n// tell that there is no aligned_alloc\n#undef _GLIBCXX_HAVE_ALIGNED_ALLOC\n// trick to define another symbol instead\n#define aligned_alloc __hidden_redefined_aligned_alloc\n// Fix the state and undefine the trick\n#include \n#undef aligned_alloc\n#endif // !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC\n#endif // defined(__GLIBC_PREREQ)\n#include \n#endif // __unix__ && !__ANDROID__\n\n#include \"proxy.h\"\n\n#include \"oneapi/tbb/detail/_config.h\"\n#include \"oneapi/tbb/scalable_allocator.h\"\n#include \"../tbb/environment.h\"\n\n#if !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) && !defined(__SUNPRO_CC)\n #if TBB_USE_EXCEPTIONS\n #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0.\n #elif !defined(TBB_USE_EXCEPTIONS)\n #define TBB_USE_EXCEPTIONS 0\n #endif\n#elif !defined(TBB_USE_EXCEPTIONS)\n #define TBB_USE_EXCEPTIONS 1\n#endif\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT\n/*** internal global operator new implementation (Linux, Windows) ***/\n#include \n\n// Synchronization primitives to protect original library pointers and new_handler\n#include \"../tbbmalloc/Synchronize.h\"\n// Use MallocMutex implementation\ntypedef MallocMutex ProxyMutex;\n\n// Adds aliasing and copy attributes to function if available\n#if defined(__has_attribute)\n #if __has_attribute(__copy__)\n #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name), __copy__(name)))\n #endif\n#endif\n\n#ifndef __TBB_ALIAS_ATTR_COPY\n #define __TBB_ALIAS_ATTR_COPY(name) __attribute__((alias (#name)))\n#endif\n\n// In case there is no std::get_new_handler function\n// which provides synchronized access to std::new_handler\n#if !__TBB_CPP11_GET_NEW_HANDLER_PRESENT\nstatic ProxyMutex new_lock;\n#endif\n\nstatic inline void* InternalOperatorNew(size_t sz) {\n void* res = scalable_malloc(sz);\n#if TBB_USE_EXCEPTIONS\n while (!res) {\n std::new_handler handler;\n#if __TBB_CPP11_GET_NEW_HANDLER_PRESENT\n handler = std::get_new_handler();\n#else\n {\n ProxyMutex::scoped_lock lock(new_lock);\n handler = std::set_new_handler(0);\n std::set_new_handler(handler);\n }\n#endif\n if (handler) {\n (*handler)();\n } else {\n throw std::bad_alloc();\n }\n res = scalable_malloc(sz);\n}\n#endif /* TBB_USE_EXCEPTIONS */\n return res;\n}\n/*** end of internal global operator new implementation ***/\n#endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED\n\n#ifndef __THROW\n#define __THROW\n#endif\n\n/*** service functions and variables ***/\n#include // for memset\n#include // for sysconf\n\nstatic long memoryPageSize;\n\nstatic inline void initPageSize()\n{\n memoryPageSize = sysconf(_SC_PAGESIZE);\n}\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED\n#include \n#include // mallinfo\n\n/* __TBB_malloc_proxy used as a weak symbol by libtbbmalloc for:\n 1) detection that the proxy library is loaded\n 2) check that dlsym(\"malloc\") found something different from our replacement malloc\n*/\n\nextern \"C\" void *__TBB_malloc_proxy(size_t) __TBB_ALIAS_ATTR_COPY(malloc);\n\nstatic void *orig_msize;\n\n#elif MALLOC_ZONE_OVERLOAD_ENABLED\n\n#include \"proxy_overload_osx.h\"\n\n#endif // MALLOC_ZONE_OVERLOAD_ENABLED\n\n// Original (i.e., replaced) functions,\n// they are never changed for MALLOC_ZONE_OVERLOAD_ENABLED.\nstatic void *orig_free,\n *orig_realloc;\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED\n#define ZONE_ARG\n#define PREFIX(name) name\n\nstatic void *orig_libc_free,\n *orig_libc_realloc;\n\n// We already tried to find ptr to original functions.\nstatic std::atomic origFuncSearched{false};\n\ninline void InitOrigPointers()\n{\n // race is OK here, as different threads found same functions\n if (!origFuncSearched.load(std::memory_order_acquire)) {\n orig_free = dlsym(RTLD_NEXT, \"free\");\n orig_realloc = dlsym(RTLD_NEXT, \"realloc\");\n orig_msize = dlsym(RTLD_NEXT, \"malloc_usable_size\");\n orig_libc_free = dlsym(RTLD_NEXT, \"__libc_free\");\n orig_libc_realloc = dlsym(RTLD_NEXT, \"__libc_realloc\");\n\n origFuncSearched.store(true, std::memory_order_release);\n }\n}\n\n/*** replacements for malloc and the family ***/\nextern \"C\" {\n#elif MALLOC_ZONE_OVERLOAD_ENABLED\n\n// each impl_* function has such 1st argument, it's unused\n#define ZONE_ARG struct _malloc_zone_t *,\n#define PREFIX(name) impl_##name\n// not interested in original functions for zone overload\ninline void InitOrigPointers() {}\n\n#endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED and MALLOC_ZONE_OVERLOAD_ENABLED\n\nvoid *PREFIX(malloc)(ZONE_ARG size_t size) __THROW\n{\n return scalable_malloc(size);\n}\n\nvoid *PREFIX(calloc)(ZONE_ARG size_t num, size_t size) __THROW\n{\n return scalable_calloc(num, size);\n}\n\nvoid PREFIX(free)(ZONE_ARG void *object) __THROW\n{\n InitOrigPointers();\n __TBB_malloc_safer_free(object, (void (*)(void*))orig_free);\n}\n\nvoid *PREFIX(realloc)(ZONE_ARG void* ptr, size_t sz) __THROW\n{\n InitOrigPointers();\n return __TBB_malloc_safer_realloc(ptr, sz, orig_realloc);\n}\n\n/* The older *NIX interface for aligned allocations;\n it's formally substituted by posix_memalign and deprecated,\n so we do not expect it to cause cyclic dependency with C RTL. */\nvoid *PREFIX(memalign)(ZONE_ARG size_t alignment, size_t size) __THROW\n{\n return scalable_aligned_malloc(size, alignment);\n}\n\n/* valloc allocates memory aligned on a page boundary */\nvoid *PREFIX(valloc)(ZONE_ARG size_t size) __THROW\n{\n if (! memoryPageSize) initPageSize();\n\n return scalable_aligned_malloc(size, memoryPageSize);\n}\n\n#undef ZONE_ARG\n#undef PREFIX\n\n#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED\n\n// match prototype from system headers\n#if __ANDROID__\nsize_t malloc_usable_size(const void *ptr) __THROW\n#else\nsize_t malloc_usable_size(void *ptr) __THROW\n#endif\n{\n InitOrigPointers();\n return __TBB_malloc_safer_msize(const_cast(ptr), (size_t (*)(void*))orig_msize);\n}\n\nint posix_memalign(void **memptr, size_t alignment, size_t size) __THROW\n{\n return scalable_posix_memalign(memptr, alignment, size);\n}\n\n/* pvalloc allocates smallest set of complete pages which can hold\n the requested number of bytes. Result is aligned on page boundary. */\nvoid *pvalloc(size_t size) __THROW\n{\n if (! memoryPageSize) initPageSize();\n // align size up to the page size,\n // pvalloc(0) returns 1 page, see man libmpatrol\n size = size? ((size-1) | (memoryPageSize-1)) + 1 : memoryPageSize;\n\n return scalable_aligned_malloc(size, memoryPageSize);\n}\n\nint mallopt(int /*param*/, int /*value*/) __THROW\n{\n return 1;\n}\n\n#if defined(__GLIBC__) || defined(__ANDROID__)\nstruct mallinfo mallinfo() __THROW\n{\n struct mallinfo m;\n memset(&m, 0, sizeof(struct mallinfo));\n\n return m;\n}\n#endif\n\n#if __ANDROID__\n// Android doesn't have malloc_usable_size, provide it to be compatible\n// with Linux, in addition overload dlmalloc_usable_size() that presented\n// under Android.\nsize_t dlmalloc_usable_size(const void *ptr) __TBB_ALIAS_ATTR_COPY(malloc_usable_size);\n#else // __ANDROID__\n// TODO: consider using __typeof__ to guarantee the correct declaration types\n// C11 function, supported starting GLIBC 2.16\nvoid *aligned_alloc(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign);\n// Those non-standard functions are exported by GLIBC, and might be used\n// in conjunction with standard malloc/free, so we must overload them.\n// Bionic doesn't have them. Not removing from the linker scripts,\n// as absent entry points are ignored by the linker.\n\nvoid *__libc_malloc(size_t size) __TBB_ALIAS_ATTR_COPY(malloc);\nvoid *__libc_calloc(size_t num, size_t size) __TBB_ALIAS_ATTR_COPY(calloc);\nvoid *__libc_memalign(size_t alignment, size_t size) __TBB_ALIAS_ATTR_COPY(memalign);\nvoid *__libc_pvalloc(size_t size) __TBB_ALIAS_ATTR_COPY(pvalloc);\nvoid *__libc_valloc(size_t size) __TBB_ALIAS_ATTR_COPY(valloc);\n\n// call original __libc_* to support naive replacement of free via __libc_free etc\nvoid __libc_free(void *ptr)\n{\n InitOrigPointers();\n __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_libc_free);\n}\n\nvoid *__libc_realloc(void *ptr, size_t size)\n{\n InitOrigPointers();\n return __TBB_malloc_safer_realloc(ptr, size, orig_libc_realloc);\n}\n#endif // !__ANDROID__\n\n} /* extern \"C\" */\n\n/*** replacements for global operators new and delete ***/\n\nvoid* operator new(size_t sz) {\n return InternalOperatorNew(sz);\n}\nvoid* operator new[](size_t sz) {\n return InternalOperatorNew(sz);\n}\nvoid operator delete(void* ptr) noexcept {\n InitOrigPointers();\n __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);\n}\nvoid operator delete[](void* ptr) noexcept {\n InitOrigPointers();\n __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);\n}\nvoid* operator new(size_t sz, const std::nothrow_t&) noexcept {\n return scalable_malloc(sz);\n}\nvoid* operator new[](std::size_t sz, const std::nothrow_t&) noexcept {\n return scalable_malloc(sz);\n}\nvoid operator delete(void* ptr, const std::nothrow_t&) noexcept {\n InitOrigPointers();\n __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);\n}\nvoid operator delete[](void* ptr, const std::nothrow_t&) noexcept {\n InitOrigPointers();\n __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);\n}\n\n#endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED */\n#endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED */\n\n#ifdef _WIN32\n#include \n\n#if !__TBB_WIN8UI_SUPPORT\n\n#include \n\n#include \"function_replacement.h\"\n\ntemplate // generic function to find length of array\ninline size_t arrayLength(const T(&)[N]) {\n return N;\n}\n\nvoid __TBB_malloc_safer_delete( void *ptr)\n{\n __TBB_malloc_safer_free( ptr, nullptr );\n}\n\nvoid* safer_aligned_malloc( size_t size, size_t alignment )\n{\n // workaround for \"is power of 2 pow N\" bug that accepts zeros\n return scalable_aligned_malloc( size, alignment>sizeof(size_t*)?alignment:sizeof(size_t*) );\n}\n\n// we do not support _expand();\nvoid* safer_expand( void *, size_t )\n{\n return nullptr;\n}\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(CRTLIB) \\\nvoid (*orig_free_##CRTLIB)(void*); \\\nvoid __TBB_malloc_safer_free_##CRTLIB(void *ptr) \\\n{ \\\n __TBB_malloc_safer_free( ptr, orig_free_##CRTLIB ); \\\n} \\\n \\\nvoid (*orig__aligned_free_##CRTLIB)(void*); \\\nvoid __TBB_malloc_safer__aligned_free_##CRTLIB(void *ptr) \\\n{ \\\n __TBB_malloc_safer_free( ptr, orig__aligned_free_##CRTLIB ); \\\n} \\\n \\\nsize_t (*orig__msize_##CRTLIB)(void*); \\\nsize_t __TBB_malloc_safer__msize_##CRTLIB(void *ptr) \\\n{ \\\n return __TBB_malloc_safer_msize( ptr, orig__msize_##CRTLIB ); \\\n} \\\n \\\nsize_t (*orig__aligned_msize_##CRTLIB)(void*, size_t, size_t); \\\nsize_t __TBB_malloc_safer__aligned_msize_##CRTLIB( void *ptr, size_t alignment, size_t offset) \\\n{ \\\n return __TBB_malloc_safer_aligned_msize( ptr, alignment, offset, orig__aligned_msize_##CRTLIB ); \\\n} \\\n \\\nvoid* __TBB_malloc_safer_realloc_##CRTLIB( void *ptr, size_t size ) \\\n{ \\\n orig_ptrs func_ptrs = {orig_free_##CRTLIB, orig__msize_##CRTLIB}; \\\n return __TBB_malloc_safer_realloc( ptr, size, &func_ptrs ); \\\n} \\\n \\\nvoid* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t alignment ) \\\n{ \\\n orig_aligned_ptrs func_ptrs = {orig__aligned_free_##CRTLIB, orig__aligned_msize_##CRTLIB}; \\\n return __TBB_malloc_safer_aligned_realloc( ptr, size, alignment, &func_ptrs ); \\\n}\n\n// Only for ucrtbase: substitution for _o_free\nvoid (*orig__o_free)(void*);\nvoid __TBB_malloc__o_free(void *ptr)\n{\n __TBB_malloc_safer_free( ptr, orig__o_free );\n}\n// Only for ucrtbase: substitution for _free_base\nvoid(*orig__free_base)(void*);\nvoid __TBB_malloc__free_base(void *ptr)\n{\n __TBB_malloc_safer_free(ptr, orig__free_base);\n}\n\n// Size limit is MAX_PATTERN_SIZE (28) byte codes / 56 symbols per line.\n// * can be used to match any digit in byte codes.\n// # followed by several * indicate a relative address that needs to be corrected.\n// Purpose of the pattern is to mark an instruction bound; it should consist of several\n// full instructions plus one extra byte code. It's not required for the patterns\n// to be unique (i.e., it's OK to have same pattern for unrelated functions).\n// TODO: use hot patch prologues if exist\nconst char* known_bytecodes[] = {\n#if _WIN64\n// \"========================================================\" - 56 symbols\n \"E9********CCCC\", // multiple - jmp(0xE9) with address followed by empty space (0xCC - INT 3)\n \"4883EC284885C974\", // release free()\n \"4883EC284885C975\", // release _msize()\n \"4885C974375348\", // release free() 8.0.50727.42, 10.0\n \"C7442410000000008B\", // release free() ucrtbase.dll 10.0.14393.33\n \"48895C24085748\", // release _aligned_msize() ucrtbase.dll 10.0.14393.33\n \"48894C24084883EC28BA\", // debug prologue\n \"4C894424184889542410\", // debug _aligned_msize() 10.0\n \"48894C24084883EC2848\", // debug _aligned_free 10.0\n \"488BD1488D0D#*******E9\", // _o_free(), ucrtbase.dll\n #if __TBB_OVERLOAD_OLD_MSVCR\n \"48895C2408574883EC3049\", // release _aligned_msize 9.0\n \"4883EC384885C975\", // release _msize() 9.0\n \"4C8BC1488B0DA6E4040033\", // an old win64 SDK\n #endif\n#else // _WIN32\n// \"========================================================\" - 56 symbols\n \"8BFF558BEC8B\", // multiple\n \"8BFF558BEC83\", // release free() & _msize() 10.0.40219.325, _msize() ucrtbase.dll\n \"8BFF558BECFF\", // release _aligned_msize ucrtbase.dll\n \"8BFF558BEC51\", // release free() & _msize() ucrtbase.dll 10.0.14393.33\n \"558BEC8B450885C074\", // release _aligned_free 11.0\n \"558BEC837D08000F\", // release _msize() 11.0.51106.1\n \"558BEC837D08007419FF\", // release free() 11.0.50727.1\n \"558BEC8B450885C075\", // release _aligned_msize() 11.0.50727.1\n \"558BEC6A018B\", // debug free() & _msize() 11.0\n \"558BEC8B451050\", // debug _aligned_msize() 11.0\n \"558BEC8B450850\", // debug _aligned_free 11.0\n \"8BFF558BEC6A\", // debug free() & _msize() 10.0.40219.325\n #if __TBB_OVERLOAD_OLD_MSVCR\n \"6A1868********E8\", // release free() 8.0.50727.4053, 9.0\n \"6A1C68********E8\", // release _msize() 8.0.50727.4053, 9.0\n #endif\n#endif // _WIN64/_WIN32\n nullptr\n };\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,function_name,dbgsuffix) \\\n ReplaceFunctionWithStore( #CRT_VER #dbgsuffix \".dll\", #function_name, \\\n (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, \\\n known_bytecodes, (FUNCPTR*)&orig_##function_name##_##CRT_VER##dbgsuffix );\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,function_name,dbgsuffix) \\\n ReplaceFunctionWithStore( #CRT_VER #dbgsuffix \".dll\", #function_name, \\\n (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, 0, nullptr );\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_REDIRECT(CRT_VER,function_name,dest_func,dbgsuffix) \\\n ReplaceFunctionWithStore( #CRT_VER #dbgsuffix \".dll\", #function_name, \\\n (FUNCPTR)__TBB_malloc_safer_##dest_func##_##CRT_VER##dbgsuffix, 0, nullptr );\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,dbgsuffix) \\\n if (BytecodesAreKnown(#CRT_VER #dbgsuffix \".dll\")) { \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,free,dbgsuffix) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_msize,dbgsuffix) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,realloc,dbgsuffix) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_free,dbgsuffix) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_msize,dbgsuffix) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,_aligned_realloc,dbgsuffix) \\\n } else \\\n SkipReplacement(#CRT_VER #dbgsuffix \".dll\");\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,)\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,d)\n\n#define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(CRT_VER) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) \\\n __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER)\n\n#if __TBB_OVERLOAD_OLD_MSVCR\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90);\n#endif\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120d);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120);\n__TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(ucrtbase);\n\n/*** replacements for global operators new and delete ***/\n\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// #pragma warning( push )\n// #pragma warning( disable : 4290 )\n#endif\n\n/*** operator new overloads internals (Linux, Windows) ***/\n\nvoid* operator_new(size_t sz) {\n return InternalOperatorNew(sz);\n}\nvoid* operator_new_arr(size_t sz) {\n return InternalOperatorNew(sz);\n}\nvoid operator_delete(void* ptr) noexcept {\n __TBB_malloc_safer_delete(ptr);\n}\n#if _MSC_VER && !defined(__INTEL_COMPILER)\n// #pragma warning( pop )\n#endif\n\nvoid operator_delete_arr(void* ptr) noexcept {\n __TBB_malloc_safer_delete(ptr);\n}\nvoid* operator_new_t(size_t sz, const std::nothrow_t&) noexcept {\n return scalable_malloc(sz);\n}\nvoid* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) noexcept {\n return scalable_malloc(sz);\n}\nvoid operator_delete_t(void* ptr, const std::nothrow_t&) noexcept {\n __TBB_malloc_safer_delete(ptr);\n}\nvoid operator_delete_arr_t(void* ptr, const std::nothrow_t&) noexcept {\n __TBB_malloc_safer_delete(ptr);\n}\n\nstruct Module {\n const char *name;\n bool doFuncReplacement; // do replacement in the DLL\n};\n\nModule modules_to_replace[] = {\n {\"msvcr100d.dll\", true},\n {\"msvcr100.dll\", true},\n {\"msvcr110d.dll\", true},\n {\"msvcr110.dll\", true},\n {\"msvcr120d.dll\", true},\n {\"msvcr120.dll\", true},\n {\"ucrtbase.dll\", true},\n// \"ucrtbased.dll\" is not supported because of problems with _dbg functions\n#if __TBB_OVERLOAD_OLD_MSVCR\n {\"msvcr90d.dll\", true},\n {\"msvcr90.dll\", true},\n {\"msvcr80d.dll\", true},\n {\"msvcr80.dll\", true},\n {\"msvcr70d.dll\", true},\n {\"msvcr70.dll\", true},\n {\"msvcr71d.dll\", true},\n {\"msvcr71.dll\", true},\n#endif\n#if __TBB_TODO\n // TODO: Try enabling replacement for non-versioned system binaries below\n {\"msvcrtd.dll\", true},\n {\"msvcrt.dll\", true},\n#endif\n };\n\n/*\nWe need to replace following functions:\nmalloc\ncalloc\n_aligned_malloc\n_expand (by dummy implementation)\n??2@YAPAXI@Z operator new (ia32)\n??_U@YAPAXI@Z void * operator new[] (size_t size) (ia32)\n??3@YAXPAX@Z operator delete (ia32)\n??_V@YAXPAX@Z operator delete[] (ia32)\n??2@YAPEAX_K@Z void * operator new(unsigned __int64) (intel64)\n??_V@YAXPEAX@Z void * operator new[](unsigned __int64) (intel64)\n??3@YAXPEAX@Z operator delete (intel64)\n??_V@YAXPEAX@Z operator delete[] (intel64)\n??2@YAPAXIABUnothrow_t@std@@@Z void * operator new (size_t sz, const std::nothrow_t&) noexcept (optional)\n??_U@YAPAXIABUnothrow_t@std@@@Z void * operator new[] (size_t sz, const std::nothrow_t&) noexcept (optional)\n\nand these functions have runtime-specific replacement:\nrealloc\nfree\n_msize\n_aligned_realloc\n_aligned_free\n_aligned_msize\n*/\n\ntypedef struct FRData_t {\n //char *_module;\n const char *_func;\n FUNCPTR _fptr;\n FRR_ON_ERROR _on_error;\n} FRDATA;\n\nFRDATA c_routines_to_replace[] = {\n { \"malloc\", (FUNCPTR)scalable_malloc, FRR_FAIL },\n { \"calloc\", (FUNCPTR)scalable_calloc, FRR_FAIL },\n { \"_aligned_malloc\", (FUNCPTR)safer_aligned_malloc, FRR_FAIL },\n { \"_expand\", (FUNCPTR)safer_expand, FRR_IGNORE },\n};\n\nFRDATA cxx_routines_to_replace[] = {\n#if _WIN64\n { \"??2@YAPEAX_K@Z\", (FUNCPTR)operator_new, FRR_FAIL },\n { \"??_U@YAPEAX_K@Z\", (FUNCPTR)operator_new_arr, FRR_FAIL },\n { \"??3@YAXPEAX@Z\", (FUNCPTR)operator_delete, FRR_FAIL },\n { \"??_V@YAXPEAX@Z\", (FUNCPTR)operator_delete_arr, FRR_FAIL },\n#else\n { \"??2@YAPAXI@Z\", (FUNCPTR)operator_new, FRR_FAIL },\n { \"??_U@YAPAXI@Z\", (FUNCPTR)operator_new_arr, FRR_FAIL },\n { \"??3@YAXPAX@Z\", (FUNCPTR)operator_delete, FRR_FAIL },\n { \"??_V@YAXPAX@Z\", (FUNCPTR)operator_delete_arr, FRR_FAIL },\n#endif\n { \"??2@YAPAXIABUnothrow_t@std@@@Z\", (FUNCPTR)operator_new_t, FRR_IGNORE },\n { \"??_U@YAPAXIABUnothrow_t@std@@@Z\", (FUNCPTR)operator_new_arr_t, FRR_IGNORE }\n};\n\n#ifndef UNICODE\ntypedef char unicode_char_t;\n#define WCHAR_SPEC \"%s\"\n#else\ntypedef wchar_t unicode_char_t;\n#define WCHAR_SPEC \"%ls\"\n#endif\n\n// Check that we recognize bytecodes that should be replaced by trampolines.\n// If some functions have unknown prologue patterns, replacement should not be done.\nbool BytecodesAreKnown(const unicode_char_t *dllName)\n{\n const char *funcName[] = {\"free\", \"_msize\", \"_aligned_free\", \"_aligned_msize\", 0};\n HMODULE module = GetModuleHandle(dllName);\n\n if (!module)\n return false;\n for (int i=0; funcName[i]; i++)\n if (! IsPrologueKnown(dllName, funcName[i], known_bytecodes, module)) {\n fprintf(stderr, \"TBBmalloc: skip allocation functions replacement in \" WCHAR_SPEC\n \": unknown prologue for function \" WCHAR_SPEC \"\\n\", dllName, funcName[i]);\n return false;\n }\n return true;\n}\n\nvoid SkipReplacement(const unicode_char_t *dllName)\n{\n#ifndef UNICODE\n const char *dllStr = dllName;\n#else\n const size_t sz = 128; // all DLL name must fit\n\n char buffer[sz];\n size_t real_sz;\n char *dllStr = buffer;\n\n errno_t ret = wcstombs_s(&real_sz, dllStr, sz, dllName, sz-1);\n __TBB_ASSERT(!ret, \"Dll name conversion failed\");\n#endif\n\n for (size_t i=0; i\n\nextern \"C\" {\n TBBMALLOC_EXPORT void __TBB_malloc_safer_free( void *ptr, void (*original_free)(void*));\n TBBMALLOC_EXPORT void * __TBB_malloc_safer_realloc( void *ptr, size_t, void* );\n TBBMALLOC_EXPORT void * __TBB_malloc_safer_aligned_realloc( void *ptr, size_t, size_t, void* );\n TBBMALLOC_EXPORT size_t __TBB_malloc_safer_msize( void *ptr, size_t (*orig_msize_crt80d)(void*));\n TBBMALLOC_EXPORT size_t __TBB_malloc_safer_aligned_msize( void *ptr, size_t, size_t, size_t (*orig_msize_crt80d)(void*,size_t,size_t));\n\n#if MALLOC_ZONE_OVERLOAD_ENABLED\n void __TBB_malloc_free_definite_size(void *object, size_t size);\n#endif\n} // extern \"C\"\n\n// Struct with original free() and _msize() pointers\nstruct orig_ptrs {\n void (*free) (void*);\n size_t (*msize)(void*);\n};\n\nstruct orig_aligned_ptrs {\n void (*aligned_free) (void*);\n size_t (*aligned_msize)(void*,size_t,size_t);\n};\n\n#endif /* _TBB_malloc_proxy_H_ */\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/proxy_overload_osx.h", "content": "/*\n Copyright (c) 2005-2022 Intel Corporation\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n*/\n\n// The original source for this code is\n// Copyright (c) 2011, Google Inc.\n// All rights reserved.\n//\n// Redistribution and use in source and binary forms, with or without\n// modification, are permitted provided that the following conditions are\n// met:\n//\n// * Redistributions of source code must retain the above copyright\n// notice, this list of conditions and the following disclaimer.\n// * Redistributions in binary form must reproduce the above\n// copyright notice, this list of conditions and the following disclaimer\n// in the documentation and/or other materials provided with the\n// distribution.\n// * Neither the name of Google Inc. nor the names of its\n// contributors may be used to endorse or promote products derived from\n// this software without specific prior written permission.\n//\n// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n#include \n#include \n#include \n#include \n\nstatic kern_return_t enumerator(task_t, void *, unsigned, vm_address_t,\n memory_reader_t, vm_range_recorder_t)\n{\n return KERN_FAILURE;\n}\n\nstatic size_t good_size(malloc_zone_t *, size_t size)\n{\n return size;\n}\n\nstatic boolean_t zone_check(malloc_zone_t *) /* Consistency checker */\n{\n return true;\n}\n\nstatic void zone_print(malloc_zone_t *, boolean_t) { }\nstatic void zone_log(malloc_zone_t *, void *) {}\nstatic void zone_force_lock(malloc_zone_t *) {}\nstatic void zone_force_unlock(malloc_zone_t *) {}\n\nstatic void zone_statistics(malloc_zone_t *, malloc_statistics_t *s)\n{\n s->blocks_in_use = 0;\n s->size_in_use = s->max_size_in_use = s->size_allocated = 0;\n}\n\nstatic boolean_t zone_locked(malloc_zone_t *)\n{\n return false;\n}\n\nstatic boolean_t impl_zone_enable_discharge_checking(malloc_zone_t *)\n{\n return false;\n}\n\nstatic void impl_zone_disable_discharge_checking(malloc_zone_t *) {}\nstatic void impl_zone_discharge(malloc_zone_t *, void *) {}\nstatic void impl_zone_destroy(struct _malloc_zone_t *) {}\n\n/* note: impl_malloc_usable_size() is called for each free() call, so it must be fast */\nstatic size_t impl_malloc_usable_size(struct _malloc_zone_t *, const void *ptr)\n{\n // malloc_usable_size() is used by macOS* to recognize which memory manager\n // allocated the address, so our wrapper must not redirect to the original function.\n return __TBB_malloc_safer_msize(const_cast(ptr), nullptr);\n}\n\nstatic void *impl_malloc(struct _malloc_zone_t *, size_t size);\nstatic void *impl_calloc(struct _malloc_zone_t *, size_t num_items, size_t size);\nstatic void *impl_valloc(struct _malloc_zone_t *, size_t size);\nstatic void impl_free(struct _malloc_zone_t *, void *ptr);\nstatic void *impl_realloc(struct _malloc_zone_t *, void *ptr, size_t size);\nstatic void *impl_memalign(struct _malloc_zone_t *, size_t alignment, size_t size);\n\n/* ptr is in zone and have reported size */\nstatic void impl_free_definite_size(struct _malloc_zone_t*, void *ptr, size_t size)\n{\n __TBB_malloc_free_definite_size(ptr, size);\n}\n\n/* Empty out caches in the face of memory pressure. */\nstatic size_t impl_pressure_relief(struct _malloc_zone_t *, size_t /* goal */)\n{\n return 0;\n}\n\nstatic malloc_zone_t *system_zone = nullptr;\n\nstruct DoMallocReplacement {\n DoMallocReplacement() {\n static malloc_introspection_t introspect;\n memset(&introspect, 0, sizeof(malloc_introspection_t));\n static malloc_zone_t zone;\n memset(&zone, 0, sizeof(malloc_zone_t));\n\n introspect.enumerator = &enumerator;\n introspect.good_size = &good_size;\n introspect.check = &zone_check;\n introspect.print = &zone_print;\n introspect.log = zone_log;\n introspect.force_lock = &zone_force_lock;\n introspect.force_unlock = &zone_force_unlock;\n introspect.statistics = zone_statistics;\n introspect.zone_locked = &zone_locked;\n introspect.enable_discharge_checking = &impl_zone_enable_discharge_checking;\n introspect.disable_discharge_checking = &impl_zone_disable_discharge_checking;\n introspect.discharge = &impl_zone_discharge;\n\n zone.size = &impl_malloc_usable_size;\n zone.malloc = &impl_malloc;\n zone.calloc = &impl_calloc;\n zone.valloc = &impl_valloc;\n zone.free = &impl_free;\n zone.realloc = &impl_realloc;\n zone.destroy = &impl_zone_destroy;\n zone.zone_name = \"tbbmalloc\";\n zone.introspect = &introspect;\n zone.version = 8;\n zone.memalign = impl_memalign;\n zone.free_definite_size = &impl_free_definite_size;\n zone.pressure_relief = &impl_pressure_relief;\n\n // make sure that default purgeable zone is initialized\n malloc_default_purgeable_zone();\n void* ptr = malloc(1);\n // get all registered memory zones\n unsigned zcount = 0;\n malloc_zone_t** zone_array = nullptr;\n kern_return_t errorcode = malloc_get_all_zones(mach_task_self(),nullptr,(vm_address_t**)&zone_array,&zcount);\n if (!errorcode && zone_array && zcount>0) {\n // find the zone that allocated ptr\n for (unsigned i=0; isize(z,ptr)>0) { // the right one is found\n system_zone = z;\n break;\n }\n }\n }\n free(ptr);\n\n malloc_zone_register(&zone);\n if (system_zone) {\n // after unregistration of the system zone, the last registered (i.e. our) zone becomes the default\n malloc_zone_unregister(system_zone);\n // register the system zone back\n malloc_zone_register(system_zone);\n }\n }\n};\n\nstatic DoMallocReplacement doMallocReplacement;\n\n" }, { "path": "src/tbb/src/tbbmalloc_proxy/tbbmalloc_proxy.rc", "content": "// Copyright (c) 2005-2024 Intel Corporation\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Includes\n//\n#include \n#include \"../../include/oneapi/tbb/version.h\"\n\n/////////////////////////////////////////////////////////////////////////////\n// Neutral resources\n\n#ifdef _WIN32\nLANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL\n#pragma code_page(1252)\n#endif //_WIN32\n\n/////////////////////////////////////////////////////////////////////////////\n//\n// Version\n//\n#define TBB_VERNUMBERS TBB_VERSION_MAJOR,TBB_VERSION_MINOR,TBB_VERSION_PATCH\n#define TBB_VERSION TBB_VERSION_STRING\n\nVS_VERSION_INFO VERSIONINFO\n FILEVERSION TBB_VERNUMBERS\n PRODUCTVERSION TBB_VERNUMBERS\n FILEFLAGSMASK 0x17L\n#ifdef _DEBUG\n FILEFLAGS 0x1L\n#else\n FILEFLAGS 0x0L\n#endif\n FILEOS 0x40004L\n FILETYPE 0x2L\n FILESUBTYPE 0x0L\nBEGIN\n BLOCK \"StringFileInfo\"\n BEGIN\n BLOCK \"000004b0\"\n BEGIN\n VALUE \"CompanyName\", \"Intel Corporation\\0\"\n VALUE \"FileDescription\", \"oneAPI Threading Building Blocks (oneTBB) library\\0\"\n VALUE \"FileVersion\", TBB_VERSION \"\\0\"\n VALUE \"LegalCopyright\", \"Copyright 2005-2024 Intel Corporation. All Rights Reserved.\\0\"\n VALUE \"LegalTrademarks\", \"\\0\"\n#ifndef TBB_USE_DEBUG\n VALUE \"OriginalFilename\", \"tbbmalloc_proxy.dll\\0\"\n#else\n VALUE \"OriginalFilename\", \"tbbmalloc_proxy_debug.dll\\0\"\n#endif\n VALUE \"ProductName\", \"oneAPI Threading Building Blocks (oneTBB)\\0\"\n VALUE \"ProductVersion\", TBB_VERSION \"\\0\"\n VALUE \"PrivateBuild\", \"\\0\"\n VALUE \"SpecialBuild\", \"\\0\"\n END\n END\n BLOCK \"VarFileInfo\"\n BEGIN\n VALUE \"Translation\", 0x0, 1200\n END\nEND\n" }, { "path": "src/tbb/third-party-programs.txt", "content": "oneAPI Threading Building Blocks (oneTBB) Third Party Programs File\n\nThis file is the \"third-party-programs.txt\" file specified in the associated Intel end user license\nagreement for the Intel software you are licensing.\n\nThe third party programs and their corresponding required notices and/or license\nterms are listed below.\n_______________________________________________________________________________________________________\n\n1. Instrumentation and Tracing Technology (ITT) Notify User API:\n Copyright (c) 2005-2023 Intel Corporation. All rights reserved.\n\n Redistribution and use in source and binary forms, with or without\n modification, are permitted provided that the following conditions\n are met:\n\n 1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n 2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in \n the documentation and/or other materials provided with the \n distribution.\n 3. Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived\n from this software without specific prior written permission.\n\n THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n_______________________________________________________________________________________________________\n\n2. Portable Hardware Locality (hwloc):\n\n Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana University Research and\n Technology Corporation. All rights reserved.\n Copyright (c) 2004-2005 The University of Tennessee and The University of Tennessee Research\n Foundation. All rights reserved.\n Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, University of Stuttgart.\n All rights reserved.\n Copyright (c) 2004-2005 The Regents of the University of California. All rights reserved.\n Copyright (c) 2009 CNRS\n Copyright (c) 2009-2016 Inria. All rights reserved.\n Copyright (c) 2009-2015 Université Bordeaux\n Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved.\n Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved.\n Copyright (c) 2010 IBM\n Copyright (c) 2010 Jirka Hladky\n Copyright (c) 2012 Aleksej Saushev, The NetBSD Foundation\n Copyright (c) 2012 Blue Brain Project, EPFL. All rights reserved.\n Copyright (c) 2013-2014 University of Wisconsin-La Crosse. All rights reserved.\n Copyright (c) 2015 Research Organization for Information Science and Technology (RIST).\n All rights reserved.\n Copyright (c) 2015-2016 Intel, Inc. All rights reserved.\n See COPYING in top-level directory.\n\n Redistribution and use in source and binary forms, with or without\n modification, are permitted provided that the following conditions\n are met:\n 1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n 2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the distribution.\n 3. The name of the author may not be used to endorse or promote products\n derived from this software without specific prior written permission.\n\n THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n_______________________________________________________________________________________________________\n\n3. gperftools: Copyright (c) 2011, Google Inc.\n\n Tachyon: Copyright (c) 1994-2008 John E. Stone. All rights reserved.\n\n BSD 3-Clause \"New\" or \"Revised\" License\n\n Redistribution and use in source and binary forms, with or without\n modification, are permitted provided that the following conditions are\n met:\n\n * Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the following disclaimer\n in the documentation and/or other materials provided with the\n distribution.\n * Neither the name of Google Inc. nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\n THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n_______________________________________________________________________________________________________\n\n4. Mateusz Kwiatkowski Workaround for bug 62258 in libstdc++\n\n ********************************************************************************\n * Author: Mateusz Kwiatkowski *\n * *\n * I hereby renounce all copyright to this file and my rights resulting from *\n * it, to the broadest extent permitted by law. It may be treated as public *\n * domain. *\n * *\n * However, as this file interfaces with GCC internal ABI, it may be subject to *\n * the terms and conditions of the GNU General Public License. Please consult *\n * the GCC licensing terms and/or a lawyer for details. *\n * *\n * Note that libstdc++ licensing terms grant additional permissions described *\n * in the GCC Runtime Library Exception, version 3.1, as published by the *\n * Free Software Foundation. *\n *******************************************************************************/\n_______________________________________________________________________________________________________\n\n5. ActiveState Thread pool with same API as (multi) processing. Pool (Python recipe)\n\n #\n # Copyright (c) 2008,2016 david decotigny (this file)\n # Copyright (c) 2006-2008, R Oudkerk (multiprocessing.Pool)\n # All rights reserved.\n #\n # Redistribution and use in source and binary forms, with or without\n # modification, are permitted provided that the following conditions\n # are met:\n #\n # 1. Redistributions of source code must retain the above copyright\n # notice, this list of conditions and the following disclaimer.\n # 2. Redistributions in binary form must reproduce the above copyright\n # notice, this list of conditions and the following disclaimer in the\n # documentation and/or other materials provided with the distribution.\n # 3. Neither the name of author nor the names of any contributors may be\n # used to endorse or promote products derived from this software\n # without specific prior written permission.\n #\n # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS \"AS IS\" AND\n # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n # SUCH DAMAGE.\n\n_______________________________________________________________________________________________________\n\n6. doctest\n\n Copyright (c) 2016-2023 Viktor Kirilov\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in all\n copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n\n_______________________________________________________________________________________________________\n\n*Other names and brands may be claimed as the property of others." }, { "path": "src/tbb-compat/tbb-compat.cpp", "content": "\n#include \n\n#include \"../tbb/include/oneapi/tbb/detail/_namespace_injection.h\"\n#include \"../tbb/include/oneapi/tbb/task_arena.h\"\n\n#include \"../tbb/src/tbb/observer_proxy.h\"\n#include \"../tbb/src/tbb/main.h\"\n#include \"../tbb/src/tbb/thread_data.h\"\n\n#ifdef _WIN32\n# define DLL_EXPORT __declspec(dllexport)\n#else\n# define DLL_EXPORT\n#endif\n\nnamespace tbb {\n\nnamespace interface6 {\nclass task_scheduler_observer;\n}\n\nnamespace internal {\n\nclass task_scheduler_observer_v3 {\n friend class tbb::detail::r1::observer_proxy;\n friend class tbb::detail::r1::observer_list;\n friend class interface6::task_scheduler_observer;\n \n //! Pointer to the proxy holding this observer.\n /** Observers are proxied by the scheduler to maintain persistent lists of them. **/\n tbb::detail::r1::observer_proxy* my_proxy;\n \n //! Counter preventing the observer from being destroyed while in use by the scheduler.\n /** Valid only when observation is on. **/\n std::atomic my_busy_count;\n \npublic:\n //! Enable or disable observation\n /** For local observers the method can be used only when the current thread\n has the task scheduler initialized or is attached to an arena.\n Repeated calls with the same state are no-ops. **/\n void DLL_EXPORT __TBB_EXPORTED_METHOD observe( bool state=true );\n \n //! Returns true if observation is enabled, false otherwise.\n bool is_observing() const {return my_proxy!=NULL;}\n \n //! Construct observer with observation disabled.\n task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); }\n \n //! Entry notification\n /** Invoked from inside observe(true) call and whenever a worker enters the arena\n this observer is associated with. If a thread is already in the arena when\n the observer is activated, the entry notification is called before it\n executes the first stolen task.\n Obsolete semantics. For global observers it is called by a thread before\n the first steal since observation became enabled. **/\n virtual void on_scheduler_entry( bool /*is_worker*/ ) {}\n \n //! Exit notification\n /** Invoked from inside observe(false) call and whenever a worker leaves the\n arena this observer is associated with.\n Obsolete semantics. For global observers it is called by a thread before\n the first steal since observation became enabled. **/\n virtual void on_scheduler_exit( bool /*is_worker*/ ) {}\n \n //! Destructor automatically switches observation off if it is enabled.\n virtual ~task_scheduler_observer_v3() { if(my_proxy) observe(false);}\n};\n\n} // namespace internal\n\nnamespace interface6 {\n\nclass task_scheduler_observer : public internal::task_scheduler_observer_v3 {\n friend class internal::task_scheduler_observer_v3;\n friend class tbb::detail::r1::observer_proxy;\n friend class tbb::detail::r1::observer_list;\n \n /** Negative numbers with the largest absolute value to minimize probability\n of coincidence in case of a bug in busy count usage. **/\n // TODO: take more high bits for version number\n static const intptr_t v6_trait = (intptr_t)((~(uintptr_t)0 >> 1) + 1);\n \n //! contains task_arena pointer or tag indicating local or global semantics of the observer\n intptr_t my_context_tag;\n enum { global_tag = 0, implicit_tag = 1 };\n \npublic:\n //! Construct local or global observer in inactive state (observation disabled).\n /** For a local observer entry/exit notifications are invoked whenever a worker\n thread joins/leaves the arena of the observer's owner thread. If a thread is\n already in the arena when the observer is activated, the entry notification is\n called before it executes the first stolen task. **/\n /** TODO: Obsolete.\n Global observer semantics is obsolete as it violates master thread isolation\n guarantees and is not composable. Thus the current default behavior of the\n constructor is obsolete too and will be changed in one of the future versions\n of the library. **/\n explicit task_scheduler_observer( bool local = false ) {\n my_context_tag = local? implicit_tag : global_tag;\n }\n \n //! Construct local observer for a given arena in inactive state (observation disabled).\n /** entry/exit notifications are invoked whenever a thread joins/leaves arena.\n If a thread is already in the arena when the observer is activated, the entry notification\n is called before it executes the first stolen task. **/\n explicit task_scheduler_observer( task_arena & a) {\n my_context_tag = (intptr_t)&a;\n }\n \n /** Destructor protects instance of the observer from concurrent notification.\n It is recommended to disable observation before destructor of a derived class starts,\n otherwise it can lead to concurrent notification callback on partly destroyed object **/\n virtual ~task_scheduler_observer() { if(my_proxy) observe(false); }\n \n //! Enable or disable observation\n /** Warning: concurrent invocations of this method are not safe.\n Repeated calls with the same state are no-ops. **/\n void observe( bool state=true ) {\n if( state && !my_proxy ) {\n __TBB_ASSERT( !my_busy_count, \"Inconsistent state of task_scheduler_observer instance\");\n my_busy_count.store(v6_trait);\n }\n internal::task_scheduler_observer_v3::observe(state);\n }\n};\n\n} // namespace interface6\n\n} // namespace tbb\n\nnamespace tbb {\nnamespace internal {\n\nDLL_EXPORT\nvoid __TBB_EXPORTED_FUNC task_scheduler_observer_v3::observe( bool enable ) {\n auto* tso = (tbb::detail::d1::task_scheduler_observer*) (this);\n tbb::detail::r1::observe(*tso, enable);\n}\n\n} // namespace internal\n} // namespace tbb\n" }, { "path": "src/tbb.cpp", "content": "\n#if RCPP_PARALLEL_USE_TBB\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace RcppParallel {\n\ntbb::global_control* s_globalControl = nullptr;\n\n// TBB Tools ----\n\nstruct TBBWorker\n{\n explicit TBBWorker(Worker& worker) : worker_(worker) {}\n\n void operator()(const tbb::blocked_range& r) const {\n worker_(r.begin(), r.end());\n }\n\nprivate:\n Worker& worker_;\n};\n\nThreadStackSizeControl::ThreadStackSizeControl()\n{\n int stackSize = resolveValue(\"RCPP_PARALLEL_STACK_SIZE\", 0, 0);\n if (stackSize > 0)\n {\n s_globalControl = new tbb::global_control(\n tbb::global_control::thread_stack_size,\n stackSize\n );\n }\n}\n\nThreadStackSizeControl::~ThreadStackSizeControl()\n{\n if (s_globalControl != nullptr)\n {\n delete s_globalControl;\n s_globalControl = nullptr;\n }\n}\n\n\n// TBB Parallel For ----\n\nclass TBBParallelForExecutor\n{\npublic:\n\n TBBParallelForExecutor(Worker& worker,\n std::size_t begin,\n std::size_t end,\n std::size_t grainSize)\n : worker_(worker),\n begin_(begin),\n end_(end),\n grainSize_(grainSize)\n {\n }\n\n void operator()() const\n {\n TBBWorker tbbWorker(worker_);\n tbb::parallel_for(\n tbb::blocked_range(begin_, end_, grainSize_),\n tbbWorker\n );\n }\n\nprivate:\n Worker& worker_;\n std::size_t begin_;\n std::size_t end_;\n std::size_t grainSize_;\n};\n\nclass TBBArenaParallelForExecutor\n{\npublic:\n\n TBBArenaParallelForExecutor(tbb::task_group& group,\n Worker& worker,\n std::size_t begin,\n std::size_t end,\n std::size_t grainSize)\n : group_(group),\n worker_(worker),\n begin_(begin),\n end_(end),\n grainSize_(grainSize)\n {\n }\n\n void operator()() const\n {\n TBBParallelForExecutor executor(worker_, begin_, end_, grainSize_);\n group_.run_and_wait(executor);\n }\n\nprivate:\n\n tbb::task_group& group_;\n Worker& worker_;\n std::size_t begin_;\n std::size_t end_;\n std::size_t grainSize_;\n};\n\nvoid tbbParallelFor(std::size_t begin,\n std::size_t end,\n Worker& worker,\n std::size_t grainSize,\n int numThreads)\n{\n ThreadStackSizeControl control;\n\n tbb::task_group group;\n TBBArenaParallelForExecutor executor(group, worker, begin, end, grainSize);\n\n tbb::task_arena arena(numThreads);\n arena.execute(executor);\n}\n\n\n// TBB Parallel Reduce ----\n\nstruct TBBReducer\n{\n explicit TBBReducer(ReducerWrapper& reducer)\n : pSplitReducer_(NULL), reducer_(reducer)\n {\n }\n\n TBBReducer(TBBReducer& tbbReducer, tbb::split)\n : pSplitReducer_(new ReducerWrapper(tbbReducer.reducer_, RcppParallel::Split())),\n reducer_(*pSplitReducer_)\n {\n }\n\n virtual ~TBBReducer() { delete pSplitReducer_; }\n\n void operator()(const tbb::blocked_range& r)\n {\n reducer_(r.begin(), r.end());\n }\n\n void join(const TBBReducer& tbbReducer)\n {\n reducer_.join(tbbReducer.reducer_);\n }\n\nprivate:\n ReducerWrapper* pSplitReducer_;\n ReducerWrapper& reducer_;\n};\n\nclass TBBParallelReduceExecutor\n{\npublic:\n\n TBBParallelReduceExecutor(ReducerWrapper& reducer,\n std::size_t begin,\n std::size_t end,\n std::size_t grainSize)\n : reducer_(reducer),\n begin_(begin),\n end_(end),\n grainSize_(grainSize)\n {\n }\n\n void operator()() const\n {\n TBBReducer tbbReducer(reducer_);\n tbb::parallel_reduce(\n tbb::blocked_range(begin_, end_, grainSize_),\n tbbReducer\n );\n }\n\nprivate:\n ReducerWrapper& reducer_;\n std::size_t begin_;\n std::size_t end_;\n std::size_t grainSize_;\n};\n\nclass TBBArenaParallelReduceExecutor\n{\npublic:\n\n TBBArenaParallelReduceExecutor(tbb::task_group& group,\n ReducerWrapper& reducer,\n std::size_t begin,\n std::size_t end,\n std::size_t grainSize)\n : group_(group),\n reducer_(reducer),\n begin_(begin),\n end_(end),\n grainSize_(grainSize)\n {\n }\n\n void operator()() const\n {\n TBBParallelReduceExecutor executor(reducer_, begin_, end_, grainSize_);\n group_.run_and_wait(executor);\n }\n\nprivate:\n\n tbb::task_group& group_;\n ReducerWrapper& reducer_;\n std::size_t begin_;\n std::size_t end_;\n std::size_t grainSize_;\n};\n\nvoid tbbParallelReduceImpl(std::size_t begin,\n std::size_t end,\n ReducerWrapper& reducer,\n std::size_t grainSize,\n int numThreads)\n{\n ThreadStackSizeControl control;\n\n tbb::task_group group;\n TBBArenaParallelReduceExecutor executor(group, reducer, begin, end, grainSize);\n\n tbb::task_arena arena(numThreads);\n arena.execute(executor);\n}\n\n} // end namespace RcppParallel\n\n#endif /* RCPP_PARALLEL_USE_TBB */\n" }, { "path": "tests/doRUnit.R", "content": "stopifnot(require(RUnit, quietly = TRUE))\nstopifnot(require(Rcpp, quietly = TRUE))\nstopifnot(require(RcppParallel, quietly = TRUE))\n\n## Set a seed to make the test deterministic\nset.seed(42)\n\n## Set a default backend\nbackend <- Sys.getenv(\"RCPP_PARALLEL_BACKEND\", unset = NA)\nif (is.na(backend))\n Sys.setenv(RCPP_PARALLEL_BACKEND = \"tinythread\")\n\nwriteLines(paste(\"Using backend:\", Sys.getenv(\"RCPP_PARALLEL_BACKEND\")))\n\n## Define tests\nsuite <- defineTestSuite(\n name = \"RcppParallel Unit Tests\",\n dirs = system.file(\"tests\", package = \"RcppParallel\")\n)\n\n## Based on practice in Rcpp to avoid some test failures\nSys.setenv(\"R_TESTS\" = \"\")\n\n## Run tests\ntests <- runTestSuite(suite)\n\n## Print results\nprintTextProtocol(tests)\n\n## Return success or failure to R CMD CHECK\nif (getErrors(tests)$nFail > 0) {\n stop(\"TEST FAILED!\")\n}\n\nif (getErrors(tests)$nErr > 0) {\n stop(\"TEST HAD ERRORS!\")\n}\n\nif (getErrors(tests)$nTestFunc < 1) {\n stop(\"NO TEST FUNCTIONS RUN!\")\n}\n" }, { "path": "tools/config/cleanup.R", "content": "\n# Clean up files generated during configuration here.\n# Use 'remove_file()' to remove files generated during configuration.\n\n# unlink(\"src/tbb/build\", recursive = TRUE)\n# unlink(\"src/tbb/build-tbb\", recursive = TRUE)\nunlink(\"inst/lib\", recursive = TRUE)\nunlink(\"inst/libs\", recursive = TRUE)\nunlink(\"inst/include/index.html\", recursive = TRUE)\nunlink(\"inst/include/oneapi\", recursive = TRUE)\nunlink(\"inst/include/serial\", recursive = TRUE)\nunlink(\"inst/include/tbb\", recursive = TRUE)\n\n" }, { "path": "tools/config/configure.R", "content": "\n# make sure we call correct version of R\nrExe <- if (.Platform$OS.type == \"windows\") \"R.exe\" else \"R\"\ndefine(R = file.path(R.home(\"bin\"), rExe))\n\n# check whether user has Makevars file that might cause trouble\nmakevars <- Sys.getenv(\"R_MAKEVARS_USER\", unset = \"~/.R/Makevars\")\nif (file.exists(makevars)) {\n contents <- readLines(makevars, warn = FALSE)\n pattern <- \"^(PKG_CPPFLAGS|PKG_CXXFLAGS)\\\\s*=\"\n bad <- grep(pattern, contents, perl = TRUE, value = TRUE)\n if (length(bad)) {\n \n text <- c(\n \"\",\n sprintf(\"NOTE: '%s' contains variable declarations incompatible with RcppParallel:\", makevars),\n \"\",\n paste0(\"\\t\", bad),\n \"\",\n \"Makevars variables prefixed with 'PKG_' should be considered reserved for use by R packages.\",\n \"\"\n )\n \n writeLines(text, con = stdout())\n \n }\n}\n\n# Figure out the appropriate CXX prefix for the current\n# version of R + configuration.\ncxx <- \"/usr/bin/c++\"\ncandidates <- c(\"CXX11\", \"CXX1X\", \"CXX\")\nfor (candidate in candidates) {\n value <- r_cmd_config(candidate)\n if (!is.null(value)) {\n if (any(grepl(\"icpc\", value))) {\n define(COMPILER = \"icc\")\n }\n cxx <- candidate\n break\n }\n}\n\n# work around issue with '-Werror=format-security' being specified without\n# a prior '-Wformat', which makes gcc angry\ncxxflags <- read_r_config(sprintf(\"%sFLAGS\", cxx), envir = NULL)[[1]]\nbroken <-\n grepl(\" -Werror=format-security \", cxxflags) &&\n !grepl(\" -Wformat \", cxxflags)\n\nif (broken)\n cxxflags <- gsub(\"-Werror=format-security\", \"-Wformat -Werror=format-security\", cxxflags)\n\n# add C++ standard if not set\nif (!grepl(\"-std=\", cxxflags, fixed = TRUE)) {\n stdflag <- if (getRversion() < \"4.0\") {\n \"-std=c++0x\"\n } else {\n \"$(CXX11STD)\"\n }\n cxxflags <- paste(stdflag, cxxflags)\n}\n\n# avoid including /usr/local/include, as this can cause\n# RcppParallel to find and use a version of libtbb installed\n# there as opposed to the bundled version\ncppflags <- read_r_config(\"CPPFLAGS\", envir = NULL)[[1]]\ncppflags <- sub(\"(?: )?-I/usr/local/include\", \"\", cppflags)\ncppflags <- sub(\"(?: )?-I/opt/homebrew/include\", \"\", cppflags)\ncppflags <- sub(\"(?: )?-I/opt/local/libexec/onetbb/include\", \"\", cppflags)\n\n# define the set of flags appropriate to the current\n# configuration of R\nswitch(\n cxx,\n \n CXX11 = define(\n CC = \"$(CC)\",\n CPPFLAGS = cppflags,\n CXX11 = \"$(CXX11)\",\n CXX11FLAGS = cxxflags,\n CXX11STD = \"$(CXX11STD)\",\n CXX11PICFLAGS = \"$(CXX11PICFLAGS)\"\n ),\n \n CXX1X = define(\n CC = \"$(CC)\",\n CPPFLAGS = cppflags,\n CXX11 = \"$(CXX1X)\",\n CXX11FLAGS = cxxflags,\n CXX11STD = \"$(CXX1XSTD)\",\n CXX11PICFLAGS = \"$(CXX1XPICFLAGS)\"\n ),\n \n CXX = define(\n CC = \"$(CC)\",\n CPPFLAGS = cppflags,\n CXX11 = \"$(CXX)\",\n CXX11FLAGS = cxxflags,\n CXX11STD = \"-std=c++0x\",\n CXX11PICFLAGS = \"-fPIC\"\n ),\n \n stop(\"Failed to infer C / C++ compilation flags\")\n)\n\n# on Windows, check for Rtools; if it exists, and we have tbb, use it\nif (.Platform$OS.type == \"windows\") {\n \n gccPath <- normalizePath(Sys.which(\"gcc\"), winslash = \"/\")\n \n tbbLib <- Sys.getenv(\"TBB_LIB\", unset = NA)\n if (is.na(tbbLib))\n tbbLib <- normalizePath(file.path(gccPath, \"../../lib\"), winslash = \"/\")\n \n tbbInc <- Sys.getenv(\"TBB_INC\", unset = NA)\n if (is.na(tbbInc))\n tbbInc <- normalizePath(file.path(gccPath, \"../../include\"), winslash = \"/\")\n \n tbbFiles <- list.files(tbbLib, pattern = \"^libtbb\")\n if (length(tbbFiles)) {\n \n tbbPattern <- \"^lib(tbb\\\\d*(?:_static)?)\\\\.a$\"\n tbbName <- grep(tbbPattern, tbbFiles, perl = TRUE, value = TRUE)\n tbbName <- gsub(tbbPattern, \"\\\\1\", tbbName, perl = TRUE)\n \n tbbMallocPattern <- \"^lib(tbbmalloc\\\\d*(?:_static)?)\\\\.a$\"\n tbbMallocName <- grep(tbbMallocPattern, tbbFiles, perl = TRUE, value = TRUE)\n tbbMallocName <- gsub(tbbMallocPattern, \"\\\\1\", tbbMallocName, perl = TRUE)\n \n Sys.setenv(\n TBB_LIB = tbbLib,\n TBB_INC = tbbInc,\n TBB_NAME = tbbName,\n TBB_MALLOC_NAME = tbbMallocName\n )\n \n }\n \n}\n\n# try and figure out path to TBB\ntbbRoot <- Sys.getenv(\"TBB_ROOT\", unset = NA)\ntbbLib <- Sys.getenv(\"TBB_LIB\", unset = NA)\ntbbInc <- Sys.getenv(\"TBB_INC\", unset = NA)\n\ntbbName <- Sys.getenv(\"TBB_NAME\", unset = \"tbb\")\ntbbMallocName <- Sys.getenv(\"TBB_MALLOC_NAME\", unset = \"tbbmalloc\")\n\n# check TBB_ROOT first if defined\nif (!is.na(tbbRoot)) {\n \n if (is.na(tbbLib)) {\n tbbLib <- file.path(tbbRoot, \"lib\")\n }\n \n if (is.na(tbbInc)) {\n tbbInc <- file.path(tbbRoot, \"include\")\n }\n \n}\n\n# if TBB_LIB is defined, guess TBB_INC\nif (!is.na(tbbLib) && is.na(tbbInc)) {\n tbbIncCandidate <- file.path(tbbLib, \"../include\")\n if (file.exists(tbbIncCandidate)) {\n tbbInc <- normalizePath(tbbIncCandidate)\n }\n}\n\n# if TBB_LIB and TBB_INC are still not defined, try auto-detecting\ntryAutoDetect <-\n .Platform$OS.type == \"unix\" &&\n Sys.getenv(\"TBB_AUTODETECT\", unset = \"FALSE\") == \"TRUE\" &&\n is.na(tbbLib) &&\n is.na(tbbInc)\n\nif (tryAutoDetect) {\n \n sysInfo <- as.list(Sys.info())\n \n homebrewPrefix <- if (sysInfo$sysname == \"Darwin\") {\n \"/opt/homebrew\"\n } else {\n \"/usr/local\"\n }\n \n tbbLibSearch <- if (sysInfo$sysname == \"Darwin\") {\n file.path(homebrewPrefix, \"opt/tbb/lib/libtbb.dylib\")\n } else {\n Sys.glob(c(\n \"/usr/*/libtbb.so\",\n \"/usr/*/*/libtbb.so\",\n \"/usr/*/*/*/libtbb.so\"\n ))\n }\n \n tbbIncSearch <- if (sysInfo$sysname == \"Darwin\") {\n file.path(homebrewPrefix, \"opt/tbb/include/tbb\")\n } else {\n Sys.glob(c(\n \"/usr/include/tbb.h\",\n \"/usr/include/*/tbb.h\"\n ))\n }\n \n if (length(tbbLibSearch) &&\n length(tbbIncSearch) &&\n file.exists(tbbLibSearch[[1L]]) &&\n file.exists(tbbIncSearch[[1L]]))\n {\n tbbLib <- dirname(tbbLibSearch[[1L]])\n tbbInc <- dirname(tbbIncSearch[[1L]])\n }\n \n}\n\n# now, define TBB_LIB and TBB_INC as appropriate\ndefine(\n TBB_LIB = if (!is.na(tbbLib)) tbbLib else \"\",\n TBB_INC = if (!is.na(tbbInc)) tbbInc else \"\",\n TBB_NAME = tbbName,\n TBB_MALLOC_NAME = tbbMallocName\n)\n\n# set PKG_LIBS\npkgLibs <- if (!is.na(tbbLib)) {\n \n c(\n \"-Wl,-L\\\"$(TBB_LIB)\\\"\",\n sprintf(\"-Wl,-rpath,%s\", shQuote(tbbLib)),\n \"-l$(TBB_NAME)\",\n \"-l$(TBB_MALLOC_NAME)\"\n )\n \n} else if (.Platform$OS.type == \"windows\") {\n \n NULL\n \n} else if (R.version$os == \"emscripten\") {\n \n c(\n \"-Wl,-Ltbb/build/lib_release\",\n \"-l$(TBB_NAME)\"\n )\n \n} else {\n \n c(\n \"-Wl,-Ltbb/build/lib_release\",\n \"-l$(TBB_NAME)\",\n \"-l$(TBB_MALLOC_NAME)\"\n )\n \n}\n\n\n# on Windows, we may need to link to ssp; otherwise,\n# we see errors like\n#\n# C:\\rtools43\\x86_64-w64-mingw32.static.posix\\bin/ld.exe: C:/rtools43/x86_64-w64-mingw32.static.posix/lib/libtbb12.a(allocator.cpp.obj):allocator.cpp:(.text+0x18b): undefined reference to `__stack_chk_fail'\n#\nif (.Platform$OS.type == \"windows\") {\n pkgLibs <- c(pkgLibs, \"-lssp\")\n}\n\ndefine(PKG_LIBS = paste(pkgLibs, collapse = \" \"))\n \n# if we're going to build tbb from sources, check for cmake\ndefine(CMAKE = \"\")\nif (is.na(tbbLib)) {\n \n cmake <- local({\n \n # check for envvar\n cmake <- Sys.getenv(\"CMAKE\", unset = NA)\n if (!is.na(cmake))\n return(cmake)\n \n # check for path\n cmake <- Sys.which(\"cmake\")\n if (nzchar(cmake))\n return(cmake)\n \n # check for macOS cmake\n cmake <- \"/Applications/CMake.app/Contents/bin/cmake\"\n if (file.exists(cmake))\n return(cmake)\n \n stop(\"cmake was not found\")\n \n })\n \n # make sure we have an appropriate version of cmake installed\n output <- system(\"cmake --version\", intern = TRUE)[[1L]]\n cmakeVersion <- numeric_version(sub(\"cmake version \", \"\", output))\n if (cmakeVersion < \"3.5\") {\n stop(\"error: RcppParallel requires cmake (>= 3.6); you have \", cmakeVersion)\n }\n \n define(CMAKE = cmake)\n \n}\n\n\n# now, set up PKG_CPPFLAGS\nif (!is.na(tbbLib)) {\n define(PKG_CPPFLAGS = \"-I../inst/include -I\\\"$(TBB_INC)\\\"\")\n} else {\n define(PKG_CPPFLAGS = \"-I../inst/include\")\n}\n\n# PKG_CXXFLAGS\nif (.Platform$OS.type == \"windows\" && is.na(tbbLib)) {\n define(TBB_ENABLED = FALSE)\n define(PKG_CXXFLAGS = \"-DRCPP_PARALLEL_USE_TBB=0\")\n} else {\n define(TBB_ENABLED = TRUE)\n define(PKG_CXXFLAGS = \"-DRCPP_PARALLEL_USE_TBB=1\")\n}\n\n# macOS needs some extra flags set\nif (Sys.info()[[\"sysname\"]] == \"Darwin\") {\n define(PKG_LIBS_EXTRA = \"-Wl,-rpath,@loader_path/../lib\")\n} else if (Sys.info()[[\"sysname\"]] == \"Linux\") {\n define(PKG_LIBS_EXTRA = \"-Wl,-rpath,$(ORIGIN)/../lib\")\n} else {\n define(PKG_LIBS_EXTRA = \"\")\n \n}\n" }, { "path": "tools/config.R", "content": "# configure-database.R -------------------------------------------------------\n\n#' Retrieve the Global Configuration Database\n#'\n#' Retrieve the global configuration database.\n#' `db` is a helper alias for the database\n#' returned by `configure_database()`.\n#'\n#' @export\nconfigure_database <- local({\n database <- new.env(parent = emptyenv())\n class(database) <- \"configure_database\"\n function() database\n})\n\n#' @export\nprint.configure_database <- function(x, ...) {\n str.configure_database(x, ...)\n}\n\n#' @export\nstr.configure_database <- function(object, ...) {\n writeLines(\"\")\n objects <- mget(ls(envir = object, all.names = TRUE), object)\n output <- utils::capture.output(utils::str(objects, ...))\n writeLines(output[-1])\n invisible(output)\n}\n\n#' Define Variables for the Configuration Database\n#'\n#' Define variables to be used as part of the default configuration database.\n#' These will be used by [configure_file()] when no configuration database\n#' is explicitly supplied. [define()] is provided as a shorter alias for the\n#' same function.\n#'\n#' @param ... A set of named arguments, mapping configuration names to values.\n#'\n#' @export\nconfigure_define <- function(...) {\n envir <- configure_database()\n list2env(list(...), envir = envir)\n}\n\n#' @rdname configure_define\n#' @export\ndefine <- configure_define\n\n#' @rdname configure_database\n#' @export\ndb <- configure_database()\n\n\n# utils.R --------------------------------------------------------------------\n\n#' Configure a File\n#'\n#' Configure a file, replacing (by default) any instances of `@`-delimited\n#' variables, e.g. `@VAR@`, with the value of the variable called `VAR` in the\n#' associated `config` environment.\n#'\n#' @param source The file to be configured.\n#' @param target The file to be generated.\n#' @param config The configuration database.\n#' @param lhs The left-hand side marker; defaults to `@`.\n#' @param rhs The right-hand side marker; defaults to `@`.\n#' @param verbose Boolean; report files as they are configured?\n#'\n#' @family configure\n#'\n#' @export\nconfigure_file <- function(\n source,\n target = sub(\"[.]in$\", \"\", source),\n config = configure_database(),\n lhs = \"@\",\n rhs = \"@\",\n verbose = configure_verbose())\n{\n # read source file\n contents <- readLines(source, warn = FALSE)\n\n # replace defined variables\n enumerate(config, function(key, val) {\n needle <- paste(lhs, key, rhs, sep = \"\")\n replacement <- val\n contents <<- gsub(needle, replacement, contents, fixed = TRUE)\n })\n\n ensure_directory(dirname(target))\n\n # write configured file to target location\n # prefer unix newlines for Makevars\n mode <- if (basename(target) %in% \"Makevars\") \"wb\" else \"w\"\n conn <- file(target, open = mode)\n on.exit(close(conn), add = TRUE)\n writeLines(contents, con = conn)\n\n # copy over source permissions\n info <- file.info(source)\n Sys.chmod(target, mode = info$mode)\n\n if (isTRUE(verbose)) {\n fmt <- \"*** configured file: '%s' => '%s'\"\n message(sprintf(fmt, source, target))\n }\n}\n\n#' Configure Files in a Directory\n#'\n#' This companion function to [configure_file()] can be used to\n#' configure all `.in` files within a directory.\n#'\n#' @param path The path to a directory in which files should be configured.\n#' @param config The configuration database to be used.\n#' @param verbose Boolean; report files as they are configured?\n#'\n#' @family configure\n#'\n#' @export\nconfigure_directory <- function(\n path = \".\",\n config = configure_database(),\n verbose = configure_verbose())\n{\n files <- list.files(\n path = path,\n pattern = \"[.]in$\",\n full.names = TRUE\n )\n\n lapply(files, configure_file, config = config, verbose = verbose)\n}\n\nconfigure_auto <- function(type) {\n\n if (!isTRUE(getOption(\"configure.auto\", default = TRUE)))\n return(invisible(FALSE))\n\n if (isTRUE(getOption(\"configure.common\", default = TRUE)))\n configure_common(type = type)\n\n if (isTRUE(getOption(\"configure.platform\", default = TRUE)))\n configure_platform(type = type)\n\n}\n\nconfigure_common <- function(type) {\n\n sources <- list.files(\n path = c(\"R\", \"src\"),\n pattern = \"[.]in$\",\n full.names = TRUE\n )\n\n sources <- sub(\"[.]/\", \"\", sources)\n\n if (type == \"configure\") {\n lapply(sources, configure_file)\n } else if (type == \"cleanup\") {\n targets <- sub(\"[.]in$\", \"\", sources)\n lapply(targets, remove_file)\n }\n\n invisible(TRUE)\n}\n\nconfigure_platform <- function(type) {\n\n sysname <- tolower(Sys.info()[[\"sysname\"]])\n\n subdirs <- sysname\n if (sysname != \"windows\")\n subdirs <- c(\"unix\", subdirs)\n\n dirs <- c(\"R\", \"src\")\n for (dir in dirs) {\n\n # list files (take care to remove directories)\n sources <- Filter(\n function(file) identical(file.info(file)$isdir, FALSE),\n list.files(file.path(dir, subdirs), full.names = TRUE)\n )\n\n # configure all discovered sources\n for (source in sources) {\n target <- file.path(dir, basename(source))\n switch(type,\n configure = configure_file(source, target),\n cleanup = remove_file(target))\n }\n }\n}\n\n#' Execute R CMD config\n#'\n#' Read information about how \\R is configured as through `R CMD config`.\n#'\n#' @param ... The names of potential configuration values.\n#' @param simplify Boolean; simplify in the case where a single value was\n#' requested?\n#'\n#' @export\nr_cmd_config <- function(..., simplify = TRUE) {\n R <- file.path(R.home(\"bin\"), \"R\")\n\n # suppress cygwin path warnings for windows\n if (Sys.info()[[\"sysname\"]] == \"Windows\") {\n CYGWIN <- Sys.getenv(\"CYGWIN\")\n Sys.setenv(CYGWIN = \"nodosfilewarning\")\n on.exit(Sys.setenv(CYGWIN = CYGWIN), add = TRUE)\n }\n\n # loop through requested values and call R CMD config\n values <- unlist(list(...), recursive = TRUE)\n config <- lapply(values, function(value) {\n\n # execute it\n stdout <- tempfile(\"r-cmd-config-\", fileext = \".txt\")\n on.exit(unlink(stdout), add = TRUE)\n status <- system2(R, c(\"CMD\", \"config\", value), stdout = stdout)\n\n # report failures as NULL (distinct from empty string)\n if (status)\n return(NULL)\n\n readLines(stdout)\n\n })\n\n names(config) <- values\n\n if (simplify && length(config) == 1)\n return(config[[1]])\n\n config\n}\n\n#' Read R Configuration for a Package\n#'\n#' Read the \\R configuration, as through `R CMD config`.\n#'\n#' @param ... The \\R configuration values to read (as a character vector).\n#' If empty, all values are read as through `R CMD config --all`).\n#' @param package The path to the \\R package's sources.\n#' @param envir The environment in which the configuration information should\n#' be assigned. By default, the [configure_database()] is populated with the\n#' requested values.\n#' @param verbose Boolean; notify the user as \\R configuration is read?\n#'\n#' @export\nread_r_config <- function(\n ...,\n package = Sys.getenv(\"R_PACKAGE_DIR\", unset = \".\"),\n envir = configure_database(),\n verbose = configure_verbose())\n{\n # move to requested directory\n owd <- setwd(package)\n on.exit(setwd(owd), add = TRUE)\n R <- file.path(R.home(\"bin\"), \"R\")\n\n # suppress cygwin path warnings for windows\n if (Sys.info()[[\"sysname\"]] == \"Windows\") {\n CYGWIN <- Sys.getenv(\"CYGWIN\")\n Sys.setenv(CYGWIN = \"nodosfilewarning\")\n on.exit(Sys.setenv(CYGWIN = CYGWIN), add = TRUE)\n }\n\n values <- unlist(list(...), recursive = TRUE)\n if (length(values) == 0) {\n\n # R CMD config --all only available since R 3.4.0\n if (getRversion() < \"3.4.0\") {\n fmt <- \"'R CMD config --all' not available in R version '%s'\"\n stop(sprintf(fmt, getRversion()))\n }\n\n # execute action\n stdout <- tempfile(\"r-cmd-config-\", fileext = \".txt\")\n on.exit(unlink(stdout), add = TRUE)\n status <- system2(R, c(\"CMD\", \"config\", \"--all\"), stdout = stdout)\n if (status)\n stop(\"failed to execute 'R CMD config --all'\")\n\n # read and parse output\n output <- readLines(stdout, warn = FALSE)\n config <- parse_key_value(output)\n\n } else {\n\n # loop through requested values and call R CMD config\n config <- lapply(values, function(value) {\n\n # execute it\n stdout <- tempfile(\"r-cmd-config-\", fileext = \".txt\")\n on.exit(unlink(stdout), add = TRUE)\n status <- system2(R, c(\"CMD\", \"config\", value), stdout = stdout)\n\n # report failures as NULL (distinct from empty string)\n if (status)\n return(NULL)\n\n readLines(stdout)\n\n })\n names(config) <- values\n }\n\n if (is.null(envir))\n return(config)\n\n list2env(config, envir = envir)\n}\n\n#' Concatenate the Contents of a Set of Files\n#'\n#' Given a set of files, concatenate their contents into\n#' a single file.\n#'\n#' @param sources An \\R list of files\n#' @param target The file to use for generation.\n#' @param headers Headers to be used for each file copied.\n#' @param preamble Text to be included at the beginning of the document.\n#' @param postamble Text to be included at the end of the document.\n#' @param verbose Boolean; inform the user when the requested file is created?\n#'\n#' @export\nconcatenate_files <- function(\n sources,\n target,\n headers = section_header(basename(sources)),\n preamble = NULL,\n postamble = NULL,\n verbose = configure_verbose())\n{\n pieces <- vapply(seq_along(sources), function(i) {\n source <- sources[[i]]\n header <- headers[[i]]\n contents <- trim_whitespace(read_file(source))\n paste(header, contents, \"\", sep = \"\\n\\n\")\n }, character(1))\n\n all <- c(preamble, pieces, postamble)\n\n ensure_directory(dirname(target))\n writeLines(all, con = target)\n\n if (verbose) {\n fmt <- \"*** created file '%s'\"\n message(sprintf(fmt, target))\n }\n\n TRUE\n}\n\n#' Add Configure Infrastructure to an R Package\n#'\n#' Add the infrastructure needed to configure an R package.\n#'\n#' @param package The path to the top-level directory of an \\R package.\n#' @export\nuse_configure <- function(package = \".\") {\n\n # preserve working directory\n owd <- getwd()\n on.exit(setwd(owd), add = TRUE)\n\n # find resources\n package <- normalizePath(package, winslash = \"/\")\n resources <- system.file(\"resources\", package = \"configure\")\n\n # copy into temporary directory\n dir <- tempfile(\"configure-\")\n on.exit(unlink(dir, recursive = TRUE), add = TRUE)\n\n dir.create(dir)\n file.copy(resources, dir, recursive = TRUE)\n\n # rename resources directory\n setwd(dir)\n file.rename(basename(resources), basename(package))\n\n # now, copy these files back into the target directory\n file.copy(basename(package), dirname(package), recursive = TRUE)\n\n # ensure DESCRIPTION contains 'Biarch: TRUE' for Windows\n setwd(package)\n DESCRIPTION <- read_file(\"DESCRIPTION\")\n if (!grepl(\"(?:^|\\n)Biarch:\", DESCRIPTION)) {\n DESCRIPTION <- paste(DESCRIPTION, \"Biarch: TRUE\", sep = \"\\n\")\n DESCRIPTION <- gsub(\"\\n{2,}\", \"\\n\", DESCRIPTION)\n cat(DESCRIPTION, file = \"DESCRIPTION\", sep = \"\\n\")\n }\n\n # write placeholders for 'configure.R', 'cleanup.R' if none exist\n ensure_directory(\"tools/config\")\n configure <- \"tools/config/configure.R\"\n if (!file.exists(\"tools/config/configure.R\")) {\n text <- c(\n \"# Prepare your package for installation here.\",\n \"# Use 'define()' to define configuration variables.\",\n \"# Use 'configure_file()' to substitute configuration values.\",\n \"\",\n \"\"\n )\n writeLines(text, con = configure)\n }\n\n cleanup <- \"tools/config/cleanup.R\"\n if (!file.exists(\"tools/config/cleanup.R\")) {\n text <- c(\n \"# Clean up files generated during configuration here.\",\n \"# Use 'remove_file()' to remove files generated during configuration.\",\n \"\",\n \"\"\n )\n writeLines(text, con = cleanup)\n }\n\n # notify the user what we did\n message(\"* Copied 'configure{.win}' and 'cleanup{.win}'.\")\n message(\"* Updated 'tools/config.R'.\")\n\n # open 'configure.R', 'cleanup.R' for editing if in RStudio\n rstudio <-\n !is.na(Sys.getenv(\"RSTUDIO\", unset = NA)) &&\n requireNamespace(\"rstudioapi\", quietly = TRUE)\n\n if (rstudio) {\n rstudioapi::navigateToFile(\"tools/config/configure.R\", 5, 1)\n rstudioapi::navigateToFile(\"tools/config/cleanup.R\", 4, 1)\n } else {\n message(\"* Use 'tools/config/configure.R' for package configuration.\")\n message(\"* Use 'tools/config/cleanup.R' for package cleanup.\")\n }\n}\n\nensure_directory <- function(dir) {\n info <- file.info(dir)\n\n # no file exists at this location; try to make it\n if (is.na(info$isdir)) {\n dir.create(dir, recursive = TRUE, showWarnings = FALSE)\n if (!file.exists(dir))\n stop(\"failed to create directory '\", dir, \"'\")\n return(TRUE)\n }\n\n # a directory already exists\n if (isTRUE(info$isdir))\n return(TRUE)\n\n # a file exists, but it's not a directory\n stop(\"file already exists at path '\", dir, \"'\")\n}\n\nenumerate <- function(x, f, ...) {\n nms <- if (is.environment(x)) ls(envir = x) else names(x)\n lapply(nms, function(nm) {\n f(nm, x[[nm]], ...)\n })\n}\n\nread_file <- function(path) {\n paste(readLines(path, warn = FALSE), collapse = \"\\n\")\n}\n\nremove_file <- function(\n path,\n verbose = configure_verbose())\n{\n info <- file.info(path)\n if (is.na(info$isdir))\n return(TRUE)\n\n name <- if (info$isdir) \"directory\" else \"file\"\n\n unlink(path, recursive = isTRUE(info$isdir))\n if (file.exists(path)) {\n fmt <- \"failed to remove %s '%s' (insufficient permissions?)\"\n stop(sprintf(fmt, name, path))\n }\n\n if (verbose) {\n fmt <- \"*** removed %s '%s'\"\n message(sprintf(fmt, name, path))\n }\n\n TRUE\n}\n\nsource_file <- function(\n path,\n envir = parent.frame())\n{\n contents <- read_file(path)\n invisible(eval(parse(text = contents), envir = envir))\n}\n\ntrim_whitespace <- function(x) {\n gsub(\"^[[:space:]]*|[[:space:]]*$\", \"\", x)\n}\n\nconfigure_verbose <- function() {\n getOption(\"configure.verbose\", !interactive())\n}\n\nnamed <- function(object, nm) {\n names(object) <- nm\n object\n}\n\nparse_key_value <- function(\n text,\n separator = \"=\",\n trim = TRUE)\n{\n # find the separator\n index <- regexpr(separator, text, fixed = TRUE)\n\n # split into parts\n keys <- substring(text, 1, index - 1)\n vals <- substring(text, index + 1)\n\n # trim if requested\n if (trim) {\n keys <- trim_whitespace(keys)\n vals <- trim_whitespace(vals)\n }\n\n # put together into R list\n named(as.list(vals), keys)\n}\n\nmove_directory <- function(source, target) {\n\n # ensure we're trying to move a directory\n info <- file.info(source)\n if (is.na(info$isdir)) {\n fmt <- \"no directory exists at path '%s'\"\n stop(sprintf(fmt, source), call. = FALSE)\n }\n\n if (!info$isdir) {\n fmt <- \"'%s' exists but is not a directory\"\n stop(sprintf(fmt, source), call. = FALSE)\n }\n\n # good to go -- let's move it\n unlink(target, recursive = TRUE)\n file.rename(source, target)\n unlink(source, recursive = TRUE)\n\n}\n\nsection_header <- function(\n label,\n prefix = \"#\",\n suffix = \"-\",\n length = 78L)\n{\n\n # figure out length of full header\n n <- length - nchar(label) - nchar(prefix) - 2L\n n[n < 0] <- 0\n\n # generate '-' suffixes\n tail <- vapply(n, function(i) {\n paste(rep(suffix, i), collapse = \"\")\n }, character(1))\n\n # join it all together\n paste(prefix, label, tail)\n\n}\n\n\n# run.R ----------------------------------------------------------------------\n\nif (!interactive()) {\n\n # extract path to install script\n args <- commandArgs(TRUE)\n type <- args[[1]]\n\n # preserve working directory\n owd <- getwd()\n on.exit(setwd(owd), add = TRUE)\n\n # switch working directory to the calling scripts's directory as set\n # by the shell, in case the R working directory was set to something else\n basedir <- Sys.getenv(\"PWD\", unset = NA)\n if (!is.na(basedir))\n setwd(basedir)\n\n # report start of execution\n package <- Sys.getenv(\"R_PACKAGE_NAME\", unset = \"\")\n fmt <- \"** preparing to %s package '%s' ...\"\n message(sprintf(fmt, type, package))\n\n # execute the requested script\n path <- sprintf(\"tools/config/%s.R\", type)\n if (file.exists(path)) source_file(path)\n\n # perform automatic configuration\n configure_auto(type = type)\n\n # report end of execution\n fmt <- \"** finished %s for package '%s'\"\n message(sprintf(fmt, type, package))\n\n}\n\n\n" }, { "path": "tools/tbb/disable-pragmas.R", "content": "# Disable TBB pragmas that silence diagnostic warnings.\n# This is necessary for CRAN submissions of RcppParallel.\n\nfiles <- list.files(\n path = \"src/tbb\",\n pattern = \"[.](?:h|cpp)$\",\n all.files = TRUE,\n full.names = TRUE,\n recursive = TRUE\n)\n\nfor (file in files) {\n \n before <- readLines(file)\n \n after <- before\n after <- gsub(\"^(\\\\s*)#pragma warning\", \"\\\\1// #pragma warning\", after, perl = TRUE)\n after <- gsub(\"^(\\\\s*)#pragma GCC\", \"\\\\1// #pragma GCC\", after, perl = TRUE)\n after <- gsub(\"^(\\\\s*)#pragma clang\", \"\\\\1// #pragma clang\", after, perl = TRUE)\n \n if (!identical(before, after))\n writeLines(after, con = file)\n}\n" }, { "path": "tools/tbb/fix-memset.R", "content": "\n# Avoid usages of memset() that might cause compiler warnings.\n# This is necessary for CRAN submissions of RcppParallel.\n\nfiles <- list.files(\n path = \"src/tbb\",\n pattern = \"[.](?:h|cpp)$\",\n all.files = TRUE,\n full.names = TRUE,\n recursive = TRUE\n)\n\npattern <- \"(memset\\\\s*\\\\(\\\\s*)(\\\\w+)(\\\\s*[,)])\"\n\nfor (file in files) {\n \n before <- readLines(file)\n after <- gsub(pattern, \"\\\\1static_cast(\\\\2)\\\\3\", before, perl = TRUE)\n \n if (!identical(before, after))\n writeLines(after, con = file)\n}\n" }, { "path": "tools/tbb/update-tbb.R", "content": "\n# update as appropriate for new TBB releases, then re-run script\nurl <- \"https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2022.0.0.tar.gz\"\n\nowd <- setwd(\"src\")\nunlink(\"tbb\", recursive = TRUE)\ndownload.file(url, destfile = basename(url), mode = \"wb\")\n\nbefore <- list.files()\nuntar(basename(url))\nafter <- list.files()\n\nfolder <- setdiff(after, before)\nprint(folder)\nfile.rename(folder, \"tbb\")\n\nsetwd(\"tbb\")\nremove <- c(\".gitattributes\", \".github\", \"doc\", \"examples\", \"python\", \"test\")\nunlink(remove, recursive = TRUE)\nbazel <- list.files(pattern = \"[Bb]azel\", all.files = TRUE)\nunlink(bazel)\nsetwd(\"..\")\n\nunlink(basename(url))\n" } ]