[ { "path": ".Rbuildignore", "content": "^LICENSE$\n^src/.+\\.o$\n^src/.+\\.so$\n\\.swp$\n^\\.ignore$\n^\\.editorconfig$\n^man-roxygen$\n^.*\\.Rproj$\n^\\.Rproj\\.user$\n^docs$\n^paper$\n^_pkgdown\\.yml$\n^README.RMD$\n^.github$\nregistry/\n^\\.lintr$\n" }, { "path": ".aspell/defaults.R", "content": "Rd_files <- vignettes <- R_files <- description <-\n list(encoding = \"UTF-8\", language = \"en\", dictionaries = c(\"en_stats\", \"batchtools\"))\n" }, { "path": ".editorconfig", "content": "# See http://editorconfig.org\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\ninsert_final_newline = true\nindent_style = space\ntrim_trailing_whitespace = true\n\n[*.{r,R}]\nindent_size = 2\n\n[*.{c,h}]\nindent_size = 4\n\n[*.{cpp,hpp}]\nindent_size = 4\n\n[{NEWS,DESCRIPTION,LICENSE}]\nmax_line_length = 80\n" }, { "path": ".github/dependabot.yaml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/workflows/pkgdown.yml", "content": "# pkgdown workflow of the mlr3 ecosystem v0.1.0\n# https://github.com/mlr-org/actions\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n release:\n types:\n - published\n workflow_dispatch:\n\nname: pkgdown\n\njobs:\n pkgdown:\n runs-on: ubuntu-latest\n\n concurrency:\n group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}\n env:\n GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}\n TORCH_INSTALL: 1\n steps:\n - uses: actions/checkout@v4\n\n - uses: r-lib/actions/setup-pandoc@v2\n\n - uses: r-lib/actions/setup-r@v2\n\n - name: Install system dependencies\n if: runner.os == 'Linux'\n run: |\n while read -r cmd\n do\n eval sudo $cmd\n done < <(Rscript -e 'writeLines(remotes::system_requirements(\"ubuntu\", \"20.04\"))')\n sudo apt-get install -y libopenmpi-dev openmpi-bin\n\n - uses: r-lib/actions/setup-r-dependencies@v2\n with:\n extra-packages: any::pkgdown, local::.\n needs: website\n\n - name: Install template\n run: pak::pkg_install(\"mlr-org/mlr3pkgdowntemplate\")\n shell: Rscript {0}\n\n - name: Build site\n run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)\n shell: Rscript {0}\n\n - name: Deploy\n if: github.event_name != 'pull_request'\n uses: JamesIves/github-pages-deploy-action@v4.7.3\n with:\n clean: false\n branch: gh-pages\n folder: docs\n" }, { "path": ".github/workflows/r-cmd-check.yml", "content": "# r cmd check workflow of the mlr3 ecosystem v0.2.0\n# https://github.com/mlr-org/actions\non:\n workflow_dispatch:\n inputs:\n debug_enabled:\n type: boolean\n description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'\n required: false\n default: false\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\n schedule:\n - cron: '0 4 * * 1'\n\n\nname: r-cmd-check\n\njobs:\n r-cmd-check:\n runs-on: ${{ matrix.config.os }}\n\n name: ${{ matrix.config.os }} (${{ matrix.config.r }})\n\n strategy:\n fail-fast: false\n matrix:\n config:\n - {os: ubuntu-latest, r: 'devel'}\n - {os: ubuntu-latest, r: 'release'}\n - {os: macos-latest, r: 'release'}\n - {os: windows-latest, r: 'release'}\n\n env:\n R_REMOTES_NO_ERRORS_FROM_WARNINGS: true\n _R_CHECK_FORCE_SUGGESTS_: 0\n RSPM: ${{ matrix.config.rspm }}\n GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}\n\n steps:\n - uses: actions/checkout@v4\n\n - uses: r-lib/actions/setup-r@v2\n with:\n r-version: ${{ matrix.config.r }}\n\n - uses: r-lib/actions/setup-pandoc@v2\n\n - name: Install OpenMPI (macOS)\n if: runner.os == 'macOS'\n run: |\n brew install open-mpi\n\n - name: Install system dependencies\n if: runner.os == 'Linux'\n run: |\n while read -r cmd\n do\n eval sudo $cmd\n done < <(Rscript -e 'writeLines(remotes::system_requirements(\"ubuntu\", \"20.04\"))')\n sudo apt-get install -y libopenmpi-dev openmpi-bin\n\n - uses: r-lib/actions/setup-r-dependencies@v2\n with:\n extra-packages: any::rcmdcheck\n needs: check\n\n - uses: r-lib/actions/check-r-package@v2\n\n - uses: mxschmitt/action-tmate@v3\n if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}\n with:\n limit-access-to-actor: true\n" }, { "path": ".gitignore", "content": ".DS_Store\n/inst/doc\n/src/*.so\n/src/*.o\n/vignettes/*.html\n/vignettes/*.pdf\n/.Rproj.user\n/batchtools.Rproj\n*.tar.gz\n*.Rcheck\nREADME.html\n\n" }, { "path": ".ignore", "content": "man/\ndocs/\n" }, { "path": ".lintr", "content": "linters: linters_with_defaults(\n # lintr defaults: https://lintr.r-lib.org/reference/default_linters.html\n # the following setup changes/removes certain linters\n assignment_linter = NULL, # do not force using <- for assignments\n object_name_linter(c(\"snake_case\", \"CamelCase\")), # only allow snake case and camel case object names\n commented_code_linter = NULL, # allow code in comments\n line_length_linter(200L),\n object_length_linter(40L),\n undesirable_function_linter(fun = c(\n # base messaging\n cat = \"use catf()\",\n stop = \"use stopf()\",\n warning = \"use warningf()\",\n message = \"use messagef()\",\n # perf\n ifelse = \"use fifelse()\",\n rank = \"use frank()\"\n ))\n )\n" }, { "path": "DESCRIPTION", "content": "Package: batchtools\nTitle: Tools for Computation on Batch Systems\nVersion: 0.9.18\nAuthors@R: c(\n person(\"Michel\", \"Lang\", , \"michellang@gmail.com\", role = c(\"cre\", \"aut\"),\n comment = c(ORCID = \"0000-0001-9754-0393\")),\n person(\"Bernd\", \"Bischl\", , \"bernd_bischl@gmx.net\", role = \"aut\"),\n person(\"Dirk\", \"Surmann\", , \"surmann@statistik.tu-dortmund.de\", role = \"ctb\",\n comment = c(ORCID = \"0000-0003-0873-137X\"))\n )\nDescription: As a successor of the packages 'BatchJobs' and\n 'BatchExperiments', this package provides a parallel implementation of\n the Map function for high performance computing systems managed by\n schedulers 'IBM Spectrum LSF'\n (), 'Univa Grid\n Engine'/'Oracle Grid Engine'\n (), 'Slurm'\n (), 'TORQUE/PBS'\n (),\n or 'Docker Swarm' (). A\n multicore and socket mode allow the parallelization on a local\n machines, and multiple machines can be hooked up via SSH to create a\n makeshift cluster. Moreover, the package provides an abstraction\n mechanism to define large-scale computer experiments in a\n well-organized and reproducible way.\nLicense: LGPL-3\nURL: https://github.com/mlr-org/batchtools, https://batchtools.mlr-org.com\nBugReports: https://github.com/mlr-org/batchtools/issues\nDepends:\n R (>= 3.0.0)\nImports:\n backports (>= 1.1.2),\n base64url (>= 1.1),\n brew,\n checkmate (>= 1.8.5),\n data.table (>= 1.11.2),\n digest (>= 0.6.9),\n fs (>= 1.2.0),\n parallel,\n progress (>= 1.1.1),\n R6,\n rappdirs,\n stats,\n stringi,\n utils,\n withr (>= 2.0.0)\nSuggests:\n debugme,\n doMPI,\n doParallel,\n e1071,\n foreach,\n future,\n future.batchtools,\n jsonlite,\n knitr,\n parallelMap,\n ranger,\n rmarkdown,\n rpart,\n snow,\n testthat,\n tibble\nVignetteBuilder:\n knitr\nByteCompile: yes\nEncoding: UTF-8\nNeedsCompilation: yes\nRoxygen: list(r6 = FALSE)\nRoxygenNote: 7.3.3\n" }, { "path": "LICENSE", "content": " GNU LESSER GENERAL PUBLIC LICENSE\n Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n\n This version of the GNU Lesser General Public License incorporates\nthe terms and conditions of version 3 of the GNU General Public\nLicense, supplemented by the additional permissions listed below.\n\n 0. Additional Definitions.\n\n As used herein, \"this License\" refers to version 3 of the GNU Lesser\nGeneral Public License, and the \"GNU GPL\" refers to version 3 of the GNU\nGeneral Public License.\n\n \"The Library\" refers to a covered work governed by this License,\nother than an Application or a Combined Work as defined below.\n\n An \"Application\" is any work that makes use of an interface provided\nby the Library, but which is not otherwise based on the Library.\nDefining a subclass of a class defined by the Library is deemed a mode\nof using an interface provided by the Library.\n\n A \"Combined Work\" is a work produced by combining or linking an\nApplication with the Library. The particular version of the Library\nwith which the Combined Work was made is also called the \"Linked\nVersion\".\n\n The \"Minimal Corresponding Source\" for a Combined Work means the\nCorresponding Source for the Combined Work, excluding any source code\nfor portions of the Combined Work that, considered in isolation, are\nbased on the Application, and not on the Linked Version.\n\n The \"Corresponding Application Code\" for a Combined Work means the\nobject code and/or source code for the Application, including any data\nand utility programs needed for reproducing the Combined Work from the\nApplication, but excluding the System Libraries of the Combined Work.\n\n 1. Exception to Section 3 of the GNU GPL.\n\n You may convey a covered work under sections 3 and 4 of this License\nwithout being bound by section 3 of the GNU GPL.\n\n 2. Conveying Modified Versions.\n\n If you modify a copy of the Library, and, in your modifications, a\nfacility refers to a function or data to be supplied by an Application\nthat uses the facility (other than as an argument passed when the\nfacility is invoked), then you may convey a copy of the modified\nversion:\n\n a) under this License, provided that you make a good faith effort to\n ensure that, in the event an Application does not supply the\n function or data, the facility still operates, and performs\n whatever part of its purpose remains meaningful, or\n\n b) under the GNU GPL, with none of the additional permissions of\n this License applicable to that copy.\n\n 3. Object Code Incorporating Material from Library Header Files.\n\n The object code form of an Application may incorporate material from\na header file that is part of the Library. You may convey such object\ncode under terms of your choice, provided that, if the incorporated\nmaterial is not limited to numerical parameters, data structure\nlayouts and accessors, or small macros, inline functions and templates\n(ten or fewer lines in length), you do both of the following:\n\n a) Give prominent notice with each copy of the object code that the\n Library is used in it and that the Library and its use are\n covered by this License.\n\n b) Accompany the object code with a copy of the GNU GPL and this license\n document.\n\n 4. Combined Works.\n\n You may convey a Combined Work under terms of your choice that,\ntaken together, effectively do not restrict modification of the\nportions of the Library contained in the Combined Work and reverse\nengineering for debugging such modifications, if you also do each of\nthe following:\n\n a) Give prominent notice with each copy of the Combined Work that\n the Library is used in it and that the Library and its use are\n covered by this License.\n\n b) Accompany the Combined Work with a copy of the GNU GPL and this license\n document.\n\n c) For a Combined Work that displays copyright notices during\n execution, include the copyright notice for the Library among\n these notices, as well as a reference directing the user to the\n copies of the GNU GPL and this license document.\n\n d) Do one of the following:\n\n 0) Convey the Minimal Corresponding Source under the terms of this\n License, and the Corresponding Application Code in a form\n suitable for, and under terms that permit, the user to\n recombine or relink the Application with a modified version of\n the Linked Version to produce a modified Combined Work, in the\n manner specified by section 6 of the GNU GPL for conveying\n Corresponding Source.\n\n 1) Use a suitable shared library mechanism for linking with the\n Library. A suitable mechanism is one that (a) uses at run time\n a copy of the Library already present on the user's computer\n system, and (b) will operate properly with a modified version\n of the Library that is interface-compatible with the Linked\n Version.\n\n e) Provide Installation Information, but only if you would otherwise\n be required to provide such information under section 6 of the\n GNU GPL, and only to the extent that such information is\n necessary to install and execute a modified version of the\n Combined Work produced by recombining or relinking the\n Application with a modified version of the Linked Version. (If\n you use option 4d0, the Installation Information must accompany\n the Minimal Corresponding Source and Corresponding Application\n Code. If you use option 4d1, you must provide the Installation\n Information in the manner specified by section 6 of the GNU GPL\n for conveying Corresponding Source.)\n\n 5. Combined Libraries.\n\n You may place library facilities that are a work based on the\nLibrary side by side in a single library together with other library\nfacilities that are not Applications and are not covered by this\nLicense, and convey such a combined library under terms of your\nchoice, if you do both of the following:\n\n a) Accompany the combined library with a copy of the same work based\n on the Library, uncombined with any other library facilities,\n conveyed under the terms of this License.\n\n b) Give prominent notice with the combined library that part of it\n is a work based on the Library, and explaining where to find the\n accompanying uncombined form of the same work.\n\n 6. Revised Versions of the GNU Lesser General Public License.\n\n The Free Software Foundation may publish revised and/or new versions\nof the GNU Lesser General Public License from time to time. Such new\nversions will be similar in spirit to the present version, but may\ndiffer in detail to address new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nLibrary as you received it specifies that a certain numbered version\nof the GNU Lesser General Public License \"or any later version\"\napplies to it, you have the option of following the terms and\nconditions either of that published version or of any later version\npublished by the Free Software Foundation. If the Library as you\nreceived it does not specify a version number of the GNU Lesser\nGeneral Public License, you may choose any version of the GNU Lesser\nGeneral Public License ever published by the Free Software Foundation.\n\n If the Library as you received it specifies that a proxy can decide\nwhether future versions of the GNU Lesser General Public License shall\napply, that proxy's public statement of acceptance of any version is\npermanent authorization for you to choose that version for the\nLibrary.\n" }, { "path": "NAMESPACE", "content": "# Generated by roxygen2: do not edit by hand\n\nS3method(doJobCollection,JobCollection)\nS3method(doJobCollection,character)\nS3method(execJob,Experiment)\nS3method(execJob,Job)\nS3method(execJob,JobCollection)\nS3method(execJob,character)\nS3method(getJob,ExperimentCollection)\nS3method(getJob,JobCollection)\nS3method(getJobPars,ExperimentRegistry)\nS3method(getJobPars,Registry)\nS3method(makeJob,ExperimentRegistry)\nS3method(makeJob,Registry)\nS3method(makeJobCollection,ExperimentRegistry)\nS3method(makeJobCollection,Registry)\nS3method(print,ClusterFunctions)\nS3method(print,ExperimentRegistry)\nS3method(print,JobCollection)\nS3method(print,Registry)\nS3method(print,RuntimeEstimate)\nS3method(print,Status)\nS3method(print,SubmitJobResult)\nS3method(runHook,JobCollection)\nS3method(runHook,Registry)\nexport(Worker)\nexport(addAlgorithm)\nexport(addExperiments)\nexport(addJobTags)\nexport(addProblem)\nexport(ajoin)\nexport(assertRegistry)\nexport(batchExport)\nexport(batchMap)\nexport(batchMapResults)\nexport(batchReduce)\nexport(binpack)\nexport(btlapply)\nexport(btmapply)\nexport(cfBrewTemplate)\nexport(cfHandleUnknownSubmitError)\nexport(cfKillJob)\nexport(cfReadBrewTemplate)\nexport(chunk)\nexport(clearRegistry)\nexport(doJobCollection)\nexport(estimateRuntimes)\nexport(execJob)\nexport(findConfFile)\nexport(findDone)\nexport(findErrors)\nexport(findExperiments)\nexport(findExpired)\nexport(findJobs)\nexport(findNotDone)\nexport(findNotStarted)\nexport(findNotSubmitted)\nexport(findOnSystem)\nexport(findQueued)\nexport(findRunning)\nexport(findStarted)\nexport(findSubmitted)\nexport(findTagged)\nexport(findTemplateFile)\nexport(flatten)\nexport(getDefaultRegistry)\nexport(getErrorMessages)\nexport(getJobNames)\nexport(getJobPars)\nexport(getJobResources)\nexport(getJobStatus)\nexport(getJobTable)\nexport(getJobTags)\nexport(getLog)\nexport(getStatus)\nexport(getUsedJobTags)\nexport(grepLogs)\nexport(ijoin)\nexport(killJobs)\nexport(ljoin)\nexport(loadRegistry)\nexport(loadResult)\nexport(lpt)\nexport(makeClusterFunctions)\nexport(makeClusterFunctionsDocker)\nexport(makeClusterFunctionsHyperQueue)\nexport(makeClusterFunctionsInteractive)\nexport(makeClusterFunctionsLSF)\nexport(makeClusterFunctionsMulticore)\nexport(makeClusterFunctionsOpenLava)\nexport(makeClusterFunctionsSGE)\nexport(makeClusterFunctionsSSH)\nexport(makeClusterFunctionsSlurm)\nexport(makeClusterFunctionsSocket)\nexport(makeClusterFunctionsTORQUE)\nexport(makeExperimentRegistry)\nexport(makeJob)\nexport(makeJobCollection)\nexport(makeRegistry)\nexport(makeSubmitJobResult)\nexport(ojoin)\nexport(reduceResults)\nexport(reduceResultsDataTable)\nexport(reduceResultsList)\nexport(removeAlgorithms)\nexport(removeExperiments)\nexport(removeJobTags)\nexport(removeProblems)\nexport(removeRegistry)\nexport(resetJobs)\nexport(rjoin)\nexport(runHook)\nexport(runOSCommand)\nexport(saveRegistry)\nexport(setDefaultRegistry)\nexport(setJobNames)\nexport(showLog)\nexport(sjoin)\nexport(submitJobs)\nexport(summarizeExperiments)\nexport(sweepRegistry)\nexport(syncRegistry)\nexport(testJob)\nexport(ujoin)\nexport(unwrap)\nexport(waitForJobs)\nimport(checkmate)\nimport(data.table)\nimport(stringi)\nimport(utils)\nimportFrom(R6,R6Class)\nimportFrom(base64url,base32_decode)\nimportFrom(base64url,base32_encode)\nimportFrom(brew,brew)\nimportFrom(digest,digest)\nimportFrom(progress,progress_bar)\nimportFrom(rappdirs,site_config_dir)\nimportFrom(rappdirs,user_config_dir)\nimportFrom(stats,pexp)\nimportFrom(stats,predict)\nimportFrom(stats,runif)\nimportFrom(withr,local_dir)\nimportFrom(withr,local_options)\nimportFrom(withr,with_dir)\nimportFrom(withr,with_seed)\nuseDynLib(batchtools,c_binpack)\nuseDynLib(batchtools,c_lpt)\nuseDynLib(batchtools,count_not_missing)\nuseDynLib(batchtools,fill_gaps)\n" }, { "path": "NEWS.md", "content": "# batchtools 0.9.18\n\n* Fixed CRAN issues with documentation\n\n# batchtools 0.9.17\n\n* Fixed a bug in the finalizer of `ClusterFunctionsMulticore`.\n\n# batchtools 0.9.16\n\n* Fixed a bug in `addExperiments()` in combination with combination method `\"bind\"` and repls > 1 where experiments have been duplicated.\n* `addExperiments()` now also accepts a vector of replications (instead of a single scalar value) for argument `repls`.\n* Improved handling of jobs in `ClusterFunctionsSlurm`.\n* Fixed a bug in `waitForJobs()`\n* Fixed some assertions.\n\n\n# batchtools 0.9.15\n\n* Maintenance update.\n\n# batchtools 0.9.14\n\n* `batchMap()` now supports unnamed `more.args`.\n* Exports are now assigned with `delayedAssign()`.\n* Fix an option in the LSF template.\n\n# batchtools 0.9.13\n\n* Maintenance release for R-4.0.0.\n\n# batchtools 0.9.12\n\n* Moved `data.table` from `Depends` to `Imports`.\n User scripts might need to explicitly attach `data.table` via `library()` now.\n* Fixes for `ClusterFunctionsMulticore`.\n* Removed a workaround for `system2()` for R-devel (to be released as R-4.0.0).\n* New configuration option `compress` to select the compression algorithm (passed down to `saveRDS()`).\n\n# batchtools 0.9.11\n\n* Removed deprecated function `chunkIds()`.\n* New default for argument `fs.timeout` in the cluster function constructor is `0` (was `NA` before).\n* Fixed a unit test for OSX.\n* Improved stability and documentation.\n* Fixed memory usage calculation.\n\n# batchtools 0.9.10\n\n* Exported functions `findConfFile()` and `findTemplateFile()`.\n* Dropped support for providing a template file directly as string. A valid file is now always required.\n* Fixed writing to `TMPDIR` instead of the R session's temporary directory.\n\n# batchtools 0.9.9\n\n* RDS files are explicitly stored in version 2 to ensure backward compatibility with R versions prior to 3.5.0.\n* Package `fs` is now used internally for all file system operations.\n* Support for per-site configuration files and templates to be set up by system administrators.\n* The print of `getStatus()` now includes a time stamp.\n* `chunk()` now optionally shuffles the ids before chunking.\n* Support for setting per-job resources in `submitJobs()`.\n* Example templates now include resources for `blas.threads` and `omp.threads`.\n* Some bug fixes regarding read-only registries.\n\n# batchtools 0.9.8\n\n* Renamed column \"memory\" in the status table to \"mem.used\" to avoid name clashes with the resource specification.\n* Exported function `assertRegistry()`.\n* New function `unwrap()` as alias to `flatten()`.\n The latter causes a name clash with package `purrr` and will be deprecated in a future version.\n* Registries now contain a unique hash which is updated each time the registry is altered.\n Can be utilized to invalidate caches, e.g. the cache of knitr.\n\n# batchtools 0.9.7\n\n* Added a workaround for a test to be compatible with testthat v2.0.0.\n* Better and more customizable handling of expired jobs in `waitForJobs()`.\n* Package `foreach` is now supported for nested parallelization as an alternative to `parallelMap`.\n* Depreciated argument flatten has been removed.\n* New helper function `flatten()` to manually unnest/unwrap lists in data frames.\n* Removed functions `getProblemIds()` and `getAlgorithmIds()`.\n Instead, you can just access `reg$problems` or `reg$algorithms`, respectively.\n* The number of the maximum concurrent jobs can now also be controlled via setting resources.\n* Internal data base changes to speed up some operations.\n Old registries are updated on first load by `loadRegistry()`.\n* Fixed a bug where the sleep mechanism between queries was not working.\n* Fixed a bug where submit errors on SLURM and TORQUE were not detected as temporary.\n\n# batchtools 0.9.6\n\n* Fixed a bug where the wrong problem was retrieved from the cache. This was only triggered for chunked jobs in\n combination with an `ExperimentRegistry`.\n\n# batchtools 0.9.5\n\n* Added a missing routine to upgrade registries created with batchtools prior to v0.9.3.\n* Fixed a bug where the registry could not be synced if jobs failed during initialization (#135).\n* The sleep duration for `waitForJobs()` and `submitJobs()` can now be set via the configuration file.\n* A new heuristic will try to detect if the registry has been altered by a simultaneously running R session.\n If this is detected, the registry in the current session will be set to a read-only state.\n* `waitForJobs()` has been reworked to allow control over the heuristic to detect expired jobs.\n Jobs are treated as expired if they have been submitted but are not detected on the system for `expire.after` iterations\n (default 3 iterations, before 1 iteration).\n* New argument `writeable` for `loadRegistry()` to allow loading registries explicitly as read-only.\n* Removed argument `update.paths` from `loadRegistry()`.\n Paths are always updated, but the registry on the file system remains unchanged unless loaded in read-write mode.\n* `ClusterFunctionsSlurm` now come with an experimental nodename argument. If set, all communication with the master is\n handled via SSH which effectively allows you to submit jobs from your local machine instead of the head node.\n Note that mounting the file system (e.g., via SSHFS) is mandatory.\n\n# batchtools 0.9.4\n\n* Fixed handling of `file.dir` with special chars like whitespace.\n* All backward slashes will now be converted to forward slashes on windows.\n* Fixed order of arguments in `findExperiments()` (argument `ids` is now first).\n* Removed code to upgrade registries created with versions prior to v0.9.0 (first CRAN release).\n* `addExperiments()` now warns if a design is passed as `data.frame` with factor columns and `stringsAsFactors` is `TRUE`.\n* Added functions `setJobNames()` and `getJobNames()` to control the name of jobs on batch systems.\n Templates should be adapted to use `job.name` instead of `job.hash` for naming.\n* Argument `flatten` of `getJobResources()`, `getJobPars()` and `getJobTable()` is deprecated and will be removed.\n Future versions of the functions will behave like `flatten` is set to `FALSE` explicitly.\n Single resources/parameters must be extracted manually (or with `tidyr::unnest()`).\n\n# batchtools 0.9.3\n\n* Running jobs now are also included while querying for status \"started\". This affects `findStarted()`, `findNotStarted()` and `getStatus()`.\n* `findExperiments()` now performs an exact string match (instead of matching substrings) for patterns specified via `prob.name` and `algo.name`.\n For substring matching, use `prob.pattern` or `algo.pattern`, respectively.\n* Changed arguments for `reduceResultsDataTable()`\n * Removed `fill`, now is always `TRUE`\n * Introduced `flatten` to control if the result should be represented as a column of lists or flattened as separate columns.\n Defaults to a backward-compatible heuristic, similar to `getJobPars`.\n* Improved heuristic to lookup template files.\n Templates shipped with the package can now be used by providing just the file name (w/o extension).\n* Updated CITATION\n\n# batchtools 0.9.2\n\n* Full support for array jobs on Slurm and TORQUE.\n* Array jobs have been disabled for SGE and LSF (due to missing information about the output format) but will be re-enable in a future release.\n Note that the variable `n.array.jobs` has been removed from `JobCollection` in favor of the new variable `array.jobs` (logical).\n* `findExperiments()` now has two additional arguments to match using regular expressions.\n The possibility to prefix a string with \"~\" to enable regular expression matching has been removed.\n* New function `batchReduce()`.\n* New function `estimateRuntimes()`.\n* New function `removeRegistry()`.\n* Missing result files are now handled more consistently, raising an exception in its defaults if the result is not available.\n The argument `missing.val` has been added to `reduceResultsList()` and `reduceResultsDataTable()` and removed from `loadResult()` and `batchMapResults()`.\n* Timestamps are now stored with sub-second accuracy.\n* Renamed Torque to TORQUE. This especially affects the constructor `makeClusterFunctionsTorque` which now must be called via `makeClusterFunctionsTORQUE()`\n* `chunkIds()` has been deprecated. Use `chunk()`, `lpt()` or `binpack()` instead.\n* Fixed listing of jobs for `ClusterFunctionsLSF` and `ClusterFunctionsOpenLava` (thanks to @phaverty).\n* Job hashes are now prefixed with the literal string 'job' to ensure they start with a letter as required by some SGE systems.\n* Fixed handling of `NULL` results in `reduceResultsList()`\n* Fixed key lookup heuristic join functions.\n* Fixed a bug where `getJobTable()` returned `difftimes` with the wrong unit (e.g., in minutes instead of seconds).\n* Deactivated swap allocation for `ClusterFunctionsDocker`.\n* The package is now more patient while communicating with the scheduler or file system by using a timeout-based approach.\n This should make the package more reliable and robust under heavy load.\n\n# batchtools 0.9.0\n\nInitial CRAN release.\nSee the vignette for a brief comparison with [BatchJobs](https://cran.r-project.org/package=BatchJobs)/[BatchExperiments](https://cran.r-project.org/package=BatchExperiments).\n" }, { "path": "R/Algorithm.R", "content": "#' @title Define Algorithms for Experiments\n#'\n#' @description\n#' Algorithms are functions which get the \\code{data} part as well as the problem instance (the return value of the\n#' function defined in \\code{\\link{Problem}}) and return an arbitrary R object.\n#'\n#' This function serializes all components to the file system and registers the algorithm in the \\code{\\link{ExperimentRegistry}}.\n#'\n#' \\code{removeAlgorithm} removes all jobs from the registry which depend on the specific algorithm.\n#' \\code{reg$algorithms} holds the IDs of already defined algorithms.\n#'\n#' @param name [\\code{character(1)}]\\cr\n#' Unique identifier for the algorithm.\n#' @param fun [\\code{function}]\\cr\n#' The algorithm function. The static problem part is passed as \\dQuote{data}, the generated\n#' problem instance is passed as \\dQuote{instance} and the \\code{\\link{Job}}/\\code{\\link{Experiment}} as \\dQuote{job}.\n#' Therefore, your function must have the formal arguments \\dQuote{job}, \\dQuote{data} and \\dQuote{instance} (or dots \\code{...}).\n#'\n#' If you do not provide a function, it defaults to a function which just returns the instance.\n#' @template expreg\n#' @return [\\code{Algorithm}]. Object of class \\dQuote{Algorithm}.\n#' @aliases Algorithm\n#' @seealso \\code{\\link{Problem}}, \\code{\\link{addExperiments}}\n#' @export\naddAlgorithm = function(name, fun = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE)\n assertString(name, min.chars = 1L)\n if (!stri_detect_regex(name, \"^[[:alnum:]_.-]+$\"))\n stopf(\"Illegal characters in problem name: %s\", name)\n if (is.null(fun)) {\n fun = function(job, data, instance, ...) instance\n } else {\n assert(checkFunction(fun, args = c(\"job\", \"data\", \"instance\")), checkFunction(fun, args = \"...\"))\n }\n\n info(\"Adding algorithm '%s'\", name)\n algo = setClasses(list(fun = fun, name = name), \"Algorithm\")\n writeRDS(algo, file = getAlgorithmURI(reg, name), compress = reg$compress)\n reg$algorithms = union(reg$algorithms, name)\n saveRegistry(reg)\n invisible(algo)\n}\n\n#' @export\n#' @rdname addAlgorithm\nremoveAlgorithms = function(name, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE, running.ok = FALSE)\n assertCharacter(name, any.missing = FALSE)\n assertSubset(name, reg$algorithms)\n\n algorithm = NULL\n for (nn in name) {\n def.ids = reg$defs[algorithm == nn, \"def.id\"]\n job.ids = filter(def.ids, reg$status, \"job.id\")\n\n info(\"Removing Algorithm '%s' and %i corresponding jobs ...\", nn, nrow(job.ids))\n file_remove(getAlgorithmURI(reg, nn))\n reg$defs = reg$defs[!def.ids]\n reg$status = reg$status[!job.ids]\n reg$algorithms = chsetdiff(reg$algorithms, nn)\n }\n\n sweepRegistry(reg)\n invisible(TRUE)\n}\n\ngetAlgorithmURI = function(reg, name) {\n fs::path(dir(reg, \"algorithms\"), mangle(name))\n}\n" }, { "path": "R/ExperimentRegistry.R", "content": "#' @title ExperimentRegistry Constructor\n#'\n#' @description\n#' \\code{makeExperimentRegistry} constructs a special \\code{\\link{Registry}} which\n#' is suitable for the definition of large scale computer experiments.\n#'\n#' Each experiments consists of a \\code{\\link{Problem}} and an \\code{\\link{Algorithm}}.\n#' These can be parametrized with \\code{\\link{addExperiments}} to actually define computational\n#' jobs.\n#'\n#' @inheritParams makeRegistry\n#' @aliases ExperimentRegistry\n#' @return [\\code{ExperimentRegistry}].\n#' @export\n#' @family Registry Experiment\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # Definde one problem, two algorithms and add them with some parameters:\n#' addProblem(reg = tmp, \"p1\",\n#' fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd))\n#' addAlgorithm(reg = tmp, \"a1\", fun = function(job, data, instance, ...) mean(instance))\n#' addAlgorithm(reg = tmp, \"a2\", fun = function(job, data, instance, ...) median(instance))\n#' ids = addExperiments(reg = tmp, list(p1 = data.table::CJ(n = c(50, 100), mean = -2:2, sd = 1:4)))\n#'\n#' # Overview over defined experiments:\n#' tmp$problems\n#' tmp$algorithms\n#' summarizeExperiments(reg = tmp)\n#' summarizeExperiments(reg = tmp, by = c(\"problem\", \"algorithm\", \"n\"))\n#' ids = findExperiments(prob.pars = (n == 50), reg = tmp)\n#' print(unwrap(getJobPars(ids, reg = tmp)))\n#'\n#' # Submit jobs\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # Reduce the results of algorithm a1\n#' ids.mean = findExperiments(algo.name = \"a1\", reg = tmp)\n#' reduceResults(ids.mean, fun = function(aggr, res, ...) c(aggr, res), reg = tmp)\n#'\n#' # Join info table with all results and calculate mean of results\n#' # grouped by n and algorithm\n#' ids = findDone(reg = tmp)\n#' pars = unwrap(getJobPars(ids, reg = tmp))\n#' results = unwrap(reduceResultsDataTable(ids, fun = function(res) list(res = res), reg = tmp))\n#' tab = ljoin(pars, results)\n#' tab[, list(mres = mean(res)), by = c(\"n\", \"algorithm\")]\nmakeExperimentRegistry = function(file.dir = \"registry\", work.dir = getwd(), conf.file = findConfFile(), packages = character(0L), namespaces = character(0L),\n source = character(0L), load = character(0L), seed = NULL, make.default = TRUE) {\n\n reg = makeRegistry(file.dir = file.dir, work.dir = work.dir, conf.file = conf.file,\n packages = packages, namespaces = namespaces, source = source, load = load, seed = seed, make.default = make.default)\n\n fs::dir_create(fs::path(reg$file.dir, c(\"problems\", \"algorithms\")))\n\n reg$problems = character(0L)\n reg$algorithms = character(0L)\n reg$status$repl = integer(0L)\n reg$defs$problem = character(0L)\n reg$defs$algorithm = character(0L)\n reg$defs$job.pars = NULL\n reg$defs$prob.pars = list()\n reg$defs$algo.pars = list()\n reg$defs$pars.hash = character(0L)\n class(reg) = c(\"ExperimentRegistry\", \"Registry\")\n\n saveRegistry(reg)\n return(reg)\n}\n\n#' @export\nprint.ExperimentRegistry = function(x, ...) {\n cat(\"Experiment Registry\\n\")\n catf(\" Backend : %s\", x$cluster.functions$name)\n catf(\" File dir : %s\", x$file.dir)\n catf(\" Work dir : %s\", x$work.dir)\n catf(\" Jobs : %i\", nrow(x$status))\n catf(\" Problems : %i\", length(x$problems))\n catf(\" Algorithms: %i\", length(x$algorithms))\n catf(\" Seed : %i\", x$seed)\n catf(\" Writeable : %s\", x$writeable)\n}\n" }, { "path": "R/Export.R", "content": "#' @title Export Objects to the Slaves\n#'\n#' @description\n#' Objects are saved in subdirectory \\dQuote{exports} of the\n#' \\dQuote{file.dir} of \\code{reg}.\n#' They are automatically loaded and placed in the global environment\n#' each time the registry is loaded or a job collection is executed.\n#'\n#' @param export [\\code{list}]\\cr\n#' Named list of objects to export.\n#' @param unexport [\\code{character}]\\cr\n#' Vector of object names to unexport.\n#' @template reg\n#' @return [\\code{data.table}] with name and uri to the exported objects.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # list exports\n#' exports = batchExport(reg = tmp)\n#' print(exports)\n#'\n#' # add a job and required exports\n#' batchMap(function(x) x^2 + y + z, x = 1:3, reg = tmp)\n#' exports = batchExport(export = list(y = 99, z = 1), reg = tmp)\n#' print(exports)\n#'\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#' stopifnot(loadResult(1, reg = tmp) == 101)\n#'\n#' # Un-export z\n#' exports = batchExport(unexport = \"z\", reg = tmp)\n#' print(exports)\nbatchExport = function(export = list(), unexport = character(0L), reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = (length(export) > 0L || length(unexport) > 0L))\n assertList(export, names = \"named\")\n assertCharacter(unexport, any.missing = FALSE, min.chars = 1L)\n\n path = fs::path(reg$file.dir, \"exports\")\n\n if (length(export) > 0L) {\n nn = names(export)\n fn = fs::path(path, mangle(nn))\n found = fs::file_exists(fn)\n if (any(!found))\n info(\"Exporting new objects: '%s' ...\", stri_flatten(nn[!found], \"','\"))\n if (any(found))\n info(\"Overwriting previously exported object: '%s'\", stri_flatten(nn[found], \"','\"))\n Map(writeRDS, object = export, file = fn, compress = reg$compress)\n }\n\n if (length(unexport) > 0L) {\n fn = fs::path(path, mangle(unexport))\n found = fs::file_exists(fn)\n if (any(found))\n info(\"Un-exporting exported objects: '%s' ...\", stri_flatten(unexport[found], \"','\"))\n file_remove(fn[found])\n }\n\n fns = list.files(path, pattern = \"\\\\.rds\")\n invisible(data.table(name = unmangle(fns), uri = fs::path(path, fns)))\n}\n" }, { "path": "R/Hooks.R", "content": "#' @title Trigger Evaluation of Custom Function\n#'\n#' @description\n#' Hooks allow to trigger functions calls on specific events.\n#' They can be specified via the \\code{\\link{ClusterFunctions}} and are triggered on the following events:\n#' \\describe{\n#' \\item{\\code{pre.sync}}{\\code{function(reg, fns, ...)}: Run before synchronizing the registry on the master. \\code{fn} is the character vector of paths to the update files.}\n#' \\item{\\code{post.sync}}{\\code{function(reg, updates, ...)}: Run after synchronizing the registry on the master. \\code{updates} is the data.table of processed updates.}\n#' \\item{\\code{pre.submit.job}}{\\code{function(reg, ...)}: Run before a job is successfully submitted to the scheduler on the master.}\n#' \\item{\\code{post.submit.job}}{\\code{function(reg, ...)}: Run after a job is successfully submitted to the scheduler on the master.}\n#' \\item{\\code{pre.submit}}{\\code{function(reg, ...)}: Run before any job is submitted to the scheduler.}\n#' \\item{\\code{post.submit}}{\\code{function(reg, ...)}: Run after a jobs are submitted to the schedule.}\n#' \\item{\\code{pre.do.collection}}{\\code{function(reg, reader, ...)}: Run before starting the job collection on the slave.\n#' \\code{reader} is an internal cache object.}\n#' \\item{\\code{post.do.collection}}{\\code{function(reg, updates, reader, ...)}: Run after all jobs in the chunk are terminated on the slave.\n#' \\code{updates} is a \\code{\\link[data.table]{data.table}} of updates which will be merged with the \\code{\\link{Registry}} by the master.\n#' \\code{reader} is an internal cache object.}\n#' \\item{\\code{pre.kill}}{\\code{function(reg, ids, ...)}: Run before any job is killed.}\n#' \\item{\\code{post.kill}}{\\code{function(reg, ids, ...)}: Run after jobs are killed. \\code{ids} is the return value of \\code{\\link{killJobs}}.}\n#' }\n#'\n#' @param obj [\\link{Registry} | \\link{JobCollection}]\\cr\n#' Registry which contains the \\link{ClusterFunctions} with element \\dQuote{hooks}\n#' or a \\link{JobCollection} which holds the subset of functions which are executed\n#' remotely.\n#' @param hook [\\code{character(1)}]\\cr\n#' ID of the hook as string.\n#' @param ... [ANY]\\cr\n#' Additional arguments passed to the function referenced by \\code{hook}.\n#' See description.\n#' @return Return value of the called function, or \\code{NULL} if there is no hook\n#' with the specified ID.\n#' @aliases Hooks Hook\n#' @export\nrunHook = function(obj, hook, ...) {\n UseMethod(\"runHook\")\n}\n\n#' @export\nrunHook.Registry = function(obj, hook, ...) {\n f = obj$cluster.functions$hooks[[hook]]\n if (is.null(f))\n return(NULL)\n \"!DEBUG [runHook]: Running hook '`hook`'\"\n f(obj, ...)\n}\n\n#' @export\nrunHook.JobCollection = function(obj, hook, ...) {\n f = obj$hooks[[hook]]\n if (is.null(f))\n return(NULL)\n \"!DEBUG [runHook]: Running hook '`hook`'\"\n f(obj, ...)\n}\n" }, { "path": "R/Job.R", "content": "BaseJob = R6Class(\"BaseJob\", cloneable = FALSE,\n public = list(\n file.dir = NULL,\n id = NULL,\n seed = NULL,\n resources = NULL,\n reader = NULL,\n initialize = function(file.dir, reader, id, seed, resources) {\n self$file.dir = file.dir\n self$reader = reader\n self$id = id\n self$seed = seed\n self$resources = resources\n }\n ),\n\n active = list(\n job.id = function() {\n # alias for id. This is confusing not to have.\n self$id\n },\n\n external.dir = function() {\n fs::dir_create(fs::path(self$file.dir, \"external\", self$id))\n }\n )\n)\n\nJob = R6Class(\"Job\", cloneable = FALSE, inherit = BaseJob,\n public = list(\n job.pars = NULL,\n initialize = function(file.dir, reader, id, job.pars, seed, resources) {\n self$job.pars = job.pars\n super$initialize(file.dir, reader, id, seed, resources)\n }\n ),\n active = list(\n fun = function() {\n self$reader$get(fs::path(self$file.dir, \"user.function.rds\"))\n },\n pars = function() {\n c(self$job.pars, self$reader$get(fs::path(self$file.dir, \"more.args.rds\")))\n }\n )\n)\n\n\nExperiment = R6Class(\"Experiment\", cloneable = FALSE, inherit = BaseJob,\n public = list(\n repl = NA_integer_,\n prob.name = NULL,\n algo.name = NULL,\n prob.pars = NULL,\n algo.pars = NULL,\n compress = NULL,\n allow.access.to.instance = TRUE,\n initialize = function(file.dir, reader, id, prob.pars, algo.pars, repl, seed, resources, prob.name, algo.name, compress = \"gzip\") {\n super$initialize(file.dir, reader, id,seed, resources)\n self$repl = repl\n self$prob.name = as.character(prob.name)\n self$prob.pars = prob.pars\n self$algo.name = as.character(algo.name)\n self$algo.pars = algo.pars\n self$compress = compress\n }\n ),\n active = list(\n problem = function() {\n self$reader$get(getProblemURI(self, self$prob.name), slot = \"..problem..\")\n },\n algorithm = function() {\n self$reader$get(getAlgorithmURI(self, self$algo.name))\n },\n pars = function() {\n list(prob.pars = self$prob.pars, algo.pars = self$algo.pars)\n },\n instance = function() {\n if (!self$allow.access.to.instance)\n stop(\"You cannot access 'job$instance' in the problem generation or algorithm function\")\n p = self$problem\n if (p$cache) {\n cache.file = getProblemCacheURI(self)\n if (fs::file_exists(cache.file)) {\n result = try(readRDS(cache.file))\n if (!inherits(result, \"try-error\"))\n return(result)\n }\n }\n seed = if (is.null(p$seed)) self$seed else getSeed(p$seed, self$repl - 1L)\n wrapper = function(...) p$fun(job = self, data = p$data, ...)\n result = with_seed(seed, do.call(wrapper, self$prob.pars, envir = .GlobalEnv))\n if (p$cache)\n writeRDS(result, file = cache.file, compress = self$compress)\n return(result)\n }\n )\n)\n\n#' @title Jobs and Experiments\n#'\n#' @description\n#' Jobs and Experiments are abstract objects which hold all information necessary to execute a single computational\n#' job for a \\code{\\link{Registry}} or \\code{\\link{ExperimentRegistry}}, respectively.\n#'\n#' They can be created using the constructor \\code{makeJob} which takes a single job id.\n#' Jobs and Experiments are passed to reduce functions like \\code{\\link{reduceResults}}.\n#' Furthermore, Experiments can be used in the functions of the \\code{\\link{Problem}} and \\code{\\link{Algorithm}}.\n#' Jobs and Experiments hold these information:\n#' \\describe{\n#' \\item{\\code{job.id}}{Job ID as integer.}\n#' \\item{\\code{pars}}{\n#' Job parameters as named list.\n#' For \\code{\\link{ExperimentRegistry}}, the parameters are divided into the sublists \\dQuote{prob.pars} and \\dQuote{algo.pars}.\n#' }\n#' \\item{\\code{seed}}{Seed which is set via \\code{\\link{doJobCollection}} as scalar integer.}\n#' \\item{\\code{resources}}{Computational resources which were set for this job as named list.}\n#' \\item{\\code{external.dir}}{\n#' Path to a directory which is created exclusively for this job. You can store external files here.\n#' Directory is persistent between multiple restarts of the job and can be cleaned by calling \\code{\\link{resetJobs}}.\n#' }\n#' \\item{\\code{fun}}{Job only: User function passed to \\code{\\link{batchMap}}.}\n#' \\item{\\code{prob.name}}{Experiments only: Problem id.}\n#' \\item{\\code{algo.name}}{Experiments only: Algorithm id.}\n#' \\item{\\code{problem}}{Experiments only: \\code{\\link{Problem}}.}\n#' \\item{\\code{instance}}{Experiments only: Problem instance.}\n#' \\item{\\code{algorithm}}{Experiments only: \\code{\\link{Algorithm}}.}\n#' \\item{\\code{repl}}{Experiments only: Replication number.}\n#' }\n#'\n#' Note that the slots \\dQuote{pars}, \\dQuote{fun}, \\dQuote{algorithm} and \\dQuote{problem}\n#' lazy-load required files from the file system and construct the object on the first access.\n#' The realizations are cached for all slots except \\dQuote{instance} (which might be stochastic).\n#'\n#' Jobs and Experiments can be executed manually with \\code{\\link{execJob}}.\n#'\n#' @template id\n#' @param reader [\\code{RDSReader} | \\code{NULL}]\\cr\n#' Reader object to retrieve files. Used internally to cache reading from the file system.\n#' The default (\\code{NULL}) does not make use of caching.\n#' @template reg\n#' @return [\\code{Job} | \\code{Experiment}].\n#' @aliases Job Experiment\n#' @rdname JobExperiment\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(function(x, y) x + y, x = 1:2, more.args = list(y = 99), reg = tmp)\n#' submitJobs(resources = list(foo = \"bar\"), reg = tmp)\n#' job = makeJob(1, reg = tmp)\n#' print(job)\n#'\n#' # Get the parameters:\n#' job$pars\n#'\n#' # Get the job resources:\n#' job$resources\n#'\n#' # Execute the job locally:\n#' execJob(job)\nmakeJob = function(id, reader = NULL, reg = getDefaultRegistry()) {\n UseMethod(\"makeJob\", object = reg)\n}\n\n\n#' @export\nmakeJob.Registry = function(id, reader = NULL, reg = getDefaultRegistry()) {\n row = mergedJobs(reg, convertId(reg, id), c(\"job.id\", \"job.pars\", \"resource.id\"))\n resources = reg$resources[row, \"resources\", on = \"resource.id\", nomatch = NA]$resources[[1L]] %??% list()\n Job$new(file.dir = reg$file.dir, reader %??% RDSReader$new(FALSE), id = row$job.id, job.pars = row$job.pars[[1L]], seed = getSeed(reg$seed, row$job.id),\n resources = resources)\n}\n\n#' @export\nmakeJob.ExperimentRegistry = function(id, reader = NULL, reg = getDefaultRegistry()) {\n row = mergedJobs(reg, convertId(reg, id), c(\"job.id\", \"problem\", \"prob.pars\", \"algorithm\", \"algo.pars\", \"repl\", \"resource.id\"))\n resources = reg$resources[row, \"resources\", on = \"resource.id\", nomatch = NA]$resources[[1L]] %??% list()\n Experiment$new(file.dir = reg$file.dir, reader %??% RDSReader$new(FALSE), id = row$job.id, prob.pars = row$prob.pars[[1L]], algo.pars = row$algo.pars[[1L]], seed = getSeed(reg$seed, row$job.id),\n repl = row$repl, resources = resources, prob.name = row$problem, algo.name = row$algorithm)\n}\n\ngetJob = function(jc, i, reader = NULL) {\n UseMethod(\"getJob\")\n}\n\n#' @export\ngetJob.JobCollection = function(jc, i, reader = RDSReader$new(FALSE)) {\n row = jc$jobs[i]\n Job$new(file.dir = jc$file.dir, reader = reader, id = row$job.id, job.pars = row$job.pars[[1L]], seed = getSeed(jc$seed, row$job.id), resources = jc$resources)\n}\n\n#' @export\ngetJob.ExperimentCollection = function(jc, i, reader = RDSReader$new(FALSE)) {\n row = jc$jobs[i]\n Experiment$new(file.dir = jc$file.dir, reader = reader, id = row$job.id, prob.pars = row$prob.pars[[1L]],\n algo.pars = row$algo.pars[[1L]], seed = getSeed(jc$seed, row$job.id), repl = row$repl,\n resources = jc$resources, prob.name = row$problem, algo.name = row$algorithm, compress = jc$compress)\n}\n" }, { "path": "R/JobCollection.R", "content": "#' @title JobCollection Constructor\n#'\n#' @description\n#' \\code{makeJobCollection} takes multiple job ids and creates an object of class \\dQuote{JobCollection} which holds all\n#' necessary information for the calculation with \\code{\\link{doJobCollection}}. It is implemented as an environment\n#' with the following variables:\n#' \\describe{\n#' \\item{file.dir}{\\code{file.dir} of the \\link{Registry}.}\n#' \\item{work.dir:}{\\code{work.dir} of the \\link{Registry}.}\n#' \\item{job.hash}{Unique identifier of the job. Used to create names on the file system.}\n#' \\item{jobs}{\\code{\\link[data.table]{data.table}} holding individual job information. See examples.}\n#' \\item{log.file}{Location of the designated log file for this job.}\n#' \\item{resources:}{Named list of of specified computational resources.}\n#' \\item{uri}{Location of the job description file (saved with \\code{link[base]{saveRDS}} on the file system.}\n#' \\item{seed}{\\code{integer(1)} Seed of the \\link{Registry}.}\n#' \\item{packages}{\\code{character} with required packages to load via \\code{\\link[base]{require}}.}\n#' \\item{namespaces}{\\code{character} with required packages to load via \\code{\\link[base]{requireNamespace}}.}\n#' \\item{source}{\\code{character} with list of files to source before execution.}\n#' \\item{load}{\\code{character} with list of files to load before execution.}\n#' \\item{array.var}{\\code{character(1)} of the array environment variable specified by the cluster functions.}\n#' \\item{array.jobs}{\\code{logical(1)} signaling if jobs were submitted using \\code{chunks.as.arrayjobs}.}\n#' }\n#' If your \\link{ClusterFunctions} uses a template, \\code{\\link[brew]{brew}} will be executed in the environment of such\n#' a collection. Thus all variables available inside the job can be used in the template.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @param resources [\\code{list}]\\cr\n#' Named list of resources. Default is \\code{list()}.\n#' @template reg\n#' @return [\\code{JobCollection}].\n#' @family JobCollection\n#' @aliases JobCollection\n#' @rdname JobCollection\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE, packages = \"methods\")\n#' batchMap(identity, 1:5, reg = tmp)\n#'\n#' # resources are usually set in submitJobs()\n#' jc = makeJobCollection(1:3, resources = list(foo = \"bar\"), reg = tmp)\n#' ls(jc)\n#' jc$resources\nmakeJobCollection = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {\n UseMethod(\"makeJobCollection\", reg)\n}\n\ncreateCollection = function(jobs, resources = list(), reg = getDefaultRegistry()) {\n jc = new.env(parent = emptyenv())\n jc$jobs = setkeyv(jobs, \"job.id\")\n jc$job.hash = rnd_hash(\"job\")\n jc$job.name = if (anyMissing(jobs$job.name)) jc$job.hash else jobs$job.name[1L]\n jc$file.dir = reg$file.dir\n jc$work.dir = reg$work.dir\n jc$seed = reg$seed\n jc$uri = getJobFiles(reg, hash = jc$job.hash)\n jc$log.file = fs::path(reg$file.dir, \"logs\", sprintf(\"%s.log\", jc$job.hash))\n jc$packages = reg$packages\n jc$namespaces = reg$namespaces\n jc$source = reg$source\n jc$load = reg$load\n jc$resources = resources\n jc$array.var = reg$cluster.functions$array.var\n jc$array.jobs = isTRUE(resources$chunks.as.arrayjobs)\n jc$compress = reg$compress\n\n hooks = chintersect(names(reg$cluster.functions$hooks), batchtools$hooks$remote)\n if (length(hooks) > 0L)\n jc$hooks = reg$cluster.functions$hooks[hooks]\n return(jc)\n}\n\n#' @export\nmakeJobCollection.Registry = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {\n jc = createCollection(mergedJobs(reg, convertIds(reg, ids), c(\"job.id\", \"job.name\", \"job.pars\")), resources, reg)\n setClasses(jc, \"JobCollection\")\n}\n\n#' @export\nmakeJobCollection.ExperimentRegistry = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {\n jc = createCollection(mergedJobs(reg, convertIds(reg, ids), c(\"job.id\", \"job.name\", \"problem\", \"algorithm\", \"prob.pars\", \"algo.pars\", \"repl\")), resources, reg)\n setClasses(jc, c(\"ExperimentCollection\", \"JobCollection\"))\n}\n\n#' @export\nprint.JobCollection = function(x, ...) {\n catf(\"Collection of %i jobs\", nrow(x$jobs))\n catf(\" Hash : %s\", x$job.hash)\n catf(\" Log file: %s\", x$log.file)\n}\n" }, { "path": "R/JobNames.R", "content": "#' @title Set and Retrieve Job Names\n#' @name JobNames\n#'\n#' @description\n#' Set custom names for jobs. These are passed to the template as \\sQuote{job.name}.\n#' If no custom name is set (or any of the job names of the chunk is missing),\n#' the job hash is used as job name.\n#' Individual job names can be accessed via \\code{jobs$job.name}.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @param names [\\code{character}]\\cr\n#' Character vector of the same length as provided ids.\n#' @template reg\n#' @return \\code{setJobNames} returns \\code{NULL} invisibly, \\code{getJobTable}\n#' returns a \\code{data.table} with columns \\code{job.id} and \\code{job.name}.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' ids = batchMap(identity, 1:10, reg = tmp)\n#' setJobNames(ids, letters[1:nrow(ids)], reg = tmp)\n#' getJobNames(reg = tmp)\nsetJobNames = function(ids = NULL, names, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE)\n ids = convertIds(reg, ids, default = noIds())\n assertCharacter(names, min.chars = 1L, len = nrow(ids))\n\n reg$status[ids, \"job.name\" := names]\n saveRegistry(reg)\n invisible(NULL)\n}\n\n#' @export\n#' @rdname JobNames\ngetJobNames = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids, default = allIds(reg))\n reg$status[ids, c(\"job.id\", \"job.name\")]\n}\n" }, { "path": "R/JobTables.R", "content": "#' @title Query Job Information\n#'\n#' @description\n#' \\code{getJobStatus} returns the internal table which stores information about the computational\n#' status of jobs, \\code{getJobPars} a table with the job parameters, \\code{getJobResources} a table\n#' with the resources which were set to submit the jobs, and \\code{getJobTags} the tags of the jobs\n#' (see \\link{Tags}).\n#'\n#' \\code{getJobTable} returns all these tables joined.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with the following columns (not necessarily in this order):\n#' \\describe{\n#' \\item{job.id}{Unique Job ID as integer.}\n#' \\item{submitted}{Time the job was submitted to the batch system as \\code{\\link[base]{POSIXct}}.}\n#' \\item{started}{Time the job was started on the batch system as \\code{\\link[base]{POSIXct}}.}\n#' \\item{done}{Time the job terminated (successfully or with an error) as \\code{\\link[base]{POSIXct}}.}\n#' \\item{error}{Either \\code{NA} if the job terminated successfully or the error message.}\n#' \\item{mem.used}{Estimate of the memory usage.}\n#' \\item{batch.id}{Batch ID as reported by the scheduler.}\n#' \\item{log.file}{Log file. If missing, defaults to \\code{[job.hash].log}.}\n#' \\item{job.hash}{Unique string identifying the job or chunk.}\n#' \\item{time.queued}{Time in seconds (as \\code{\\link[base]{difftime}}) the job was queued.}\n#' \\item{time.running}{Time in seconds (as \\code{\\link[base]{difftime}}) the job was running.}\n#' \\item{pars}{List of parameters/arguments for this job.}\n#' \\item{resources}{List of computational resources set for this job.}\n#' \\item{tags}{Tags as joined string, delimited by \\dQuote{,}.}\n#' \\item{problem}{Only for \\code{\\link{ExperimentRegistry}}: the problem identifier.}\n#' \\item{algorithm}{Only for \\code{\\link{ExperimentRegistry}}: the algorithm identifier.}\n#' }\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' f = function(x) if (x < 0) stop(\"x must be > 0\") else sqrt(x)\n#' batchMap(f, x = c(-1, 0, 1), reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#' addJobTags(1:2, \"tag1\", reg = tmp)\n#' addJobTags(2, \"tag2\", reg = tmp)\n#'\n#' # Complete table:\n#' getJobTable(reg = tmp)\n#'\n#' # Job parameters:\n#' getJobPars(reg = tmp)\n#'\n#' # Set and retrieve tags:\n#' getJobTags(reg = tmp)\n#'\n#' # Job parameters with tags right-joined:\n#' rjoin(getJobPars(reg = tmp), getJobTags(reg = tmp))\ngetJobTable = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids)\n getJobStatus(ids, reg = reg)[getJobPars(ids, reg = reg)][getJobResources(ids = ids, reg = reg)][getJobTags(ids = ids, reg = reg)]\n}\n\n#' @export\n#' @rdname getJobTable\ngetJobStatus = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n submitted = started = done = NULL\n\n cols = chsetdiff(names(reg$status), c(\"def.id\", \"resource.id\"))\n tab = filter(reg$status, convertIds(reg, ids), cols)\n tab[, \"submitted\" := as.POSIXct(submitted, origin = \"1970-01-01\")]\n tab[, \"started\" := as.POSIXct(started, origin = \"1970-01-01\")]\n tab[, \"done\" := as.POSIXct(done, origin = \"1970-01-01\")]\n tab[, \"time.queued\" := difftime(started, submitted, units = \"secs\")]\n tab[, \"time.running\" := difftime(done, started, units = \"secs\")]\n tab[]\n}\n\n#' @export\n#' @rdname getJobTable\ngetJobResources = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids)\n tab = merge(filter(reg$status, ids, c(\"job.id\", \"resource.id\")), reg$resources, all.x = TRUE, by = \"resource.id\")[, c(\"job.id\", \"resources\")]\n setkeyv(tab, \"job.id\")[]\n}\n\n#' @export\n#' @rdname getJobTable\ngetJobPars = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n UseMethod(\"getJobPars\", object = reg)\n}\n\n#' @export\ngetJobPars.Registry = function(ids = NULL, reg = getDefaultRegistry()) {\n ids = convertIds(reg, ids)\n tab = mergedJobs(reg, ids, c(\"job.id\", \"job.pars\"))\n setkeyv(tab, \"job.id\")[]\n}\n\n#' @export\ngetJobPars.ExperimentRegistry = function(ids = NULL, reg = getDefaultRegistry()) {\n ids = convertIds(reg, ids)\n tab = mergedJobs(reg, ids, c(\"job.id\", \"problem\", \"prob.pars\", \"algorithm\", \"algo.pars\"))\n setkeyv(tab, \"job.id\")[]\n}\n\n\n#' @export\n#' @rdname getJobTable\ngetJobTags = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids, default = allIds(reg))\n tag = NULL\n reg$tags[ids, on = \"job.id\"][, list(tags = stri_flatten(sort(tag, na.last = TRUE), \",\")), by = \"job.id\"]\n}\n" }, { "path": "R/Joins.R", "content": "#' @title Inner, Left, Right, Outer, Semi and Anti Join for Data Tables\n#' @name JoinTables\n#'\n#' @description\n#' These helper functions perform join operations on data tables.\n#' Most of them are basically one-liners.\n#' See \\url{https://rpubs.com/ronasta/join_data_tables} for a overview of join operations in\n#' data table or alternatively \\pkg{dplyr}'s vignette on two table verbs.\n#'\n#' @param x [\\code{\\link{data.frame}}]\\cr\n#' First data.frame to join.\n#' @param y [\\code{\\link{data.frame}}]\\cr\n#' Second data.frame to join.\n#' @param by [\\code{character}]\\cr\n#' Column name(s) of variables used to match rows in \\code{x} and \\code{y}.\n#' If not provided, a heuristic similar to the one described in the \\pkg{dplyr} vignette is used:\n#' \\enumerate{\n#' \\item If \\code{x} is keyed, the existing key will be used if \\code{y} has the same column(s).\n#' \\item If \\code{x} is not keyed, the intersect of common columns names is used if not empty.\n#' \\item Raise an exception.\n#' }\n#' You may pass a named character vector to merge on columns with different names in \\code{x} and\n#' \\code{y}: \\code{by = c(\"x.id\" = \"y.id\")} will match \\code{x}'s \\dQuote{x.id} column with \\code{y}\\'s\n#' \\dQuote{y.id} column.\n#' @return [\\code{\\link[data.table]{data.table}}] with key identical to \\code{by}.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' # Create two tables for demonstration\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(identity, x = 1:6, reg = tmp)\n#' x = getJobPars(reg = tmp)\n#' y = findJobs(x >= 2 & x <= 5, reg = tmp)\n#' y$extra.col = head(letters, nrow(y))\n#'\n#' # Inner join: similar to intersect(): keep all columns of x and y with common matches\n#' ijoin(x, y)\n#'\n#' # Left join: use all ids from x, keep all columns of x and y\n#' ljoin(x, y)\n#'\n#' # Right join: use all ids from y, keep all columns of x and y\n#' rjoin(x, y)\n#'\n#' # Outer join: similar to union(): keep all columns of x and y with matches in x or y\n#' ojoin(x, y)\n#'\n#' # Semi join: filter x with matches in y\n#' sjoin(x, y)\n#'\n#' # Anti join: filter x with matches not in y\n#' ajoin(x, y)\n#'\n#' # Updating join: Replace values in x with values in y\n#' ujoin(x, y)\nijoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n setKey(x[y, nomatch = 0L, on = by], by)\n}\n\n#' @rdname JoinTables\n#' @export\nljoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n setKey(y[x, on = by], by)\n}\n\n#' @rdname JoinTables\n#' @export\nrjoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n setKey(x[y, on = by], by)\n}\n\n#' @rdname JoinTables\n#' @export\nojoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n res = if (is.null(names(by)))\n merge(x, y, all = TRUE, by = by)\n else\n merge(x, y, all = TRUE, by.x = names(by), by.y = by)\n\n setKey(res, by)\n}\n\n#' @rdname JoinTables\n#' @export\nsjoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n w = unique(x[y, on = by, nomatch = 0L, which = TRUE, allow.cartesian = TRUE])\n setKey(x[w], by)\n}\n\n#' @rdname JoinTables\n#' @export\najoin = function(x, y, by = NULL) {\n x = as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n setKey(x[!y, on = by], by)\n}\n\n#' @rdname JoinTables\n#' @param all.y [logical(1)]\\cr\n#' Keep columns of \\code{y} which are not in \\code{x}?\n#' @export\nujoin = function(x, y, all.y = FALSE, by = NULL) {\n assertFlag(all.y)\n x = if (is.data.table(x)) copy(x) else as.data.table(x)\n y = as.data.table(y)\n by = guessBy(x, y, by)\n\n cn = chsetdiff(names(y), by)\n if (!all.y)\n cn = chintersect(names(x), cn)\n if (length(cn) == 0L)\n return(x)\n\n expr = parse(text = stri_join(\"`:=`(\", stri_flatten(sprintf(\"%1$s=i.%1$s\", cn), \",\"), \")\"))\n setKey(x[y, eval(expr), on = by], by)\n}\n\nguessBy = function(x, y, by = NULL) {\n assertDataFrame(x, min.cols = 1L)\n assertDataFrame(y, min.cols = 1L)\n\n if (is.null(by)) {\n res = key(x)\n if (!is.null(res) && all(res %chin% names(y)))\n return(res)\n\n res = chintersect(names(x), names(y))\n if (length(res) > 0L)\n return(res)\n stop(\"Unable to guess columns to match on. Please specify them explicitly or set keys beforehand.\")\n } else {\n if (is.null(names(by))) {\n assertSubset(by, names(x))\n } else {\n assertSubset(names(by), names(x))\n }\n assertSubset(by, names(y))\n return(by)\n }\n}\n\nsetKey = function(res, by) {\n by = names(by) %??% unname(by)\n if (!identical(key(res), by))\n setkeyv(res, by)\n res[]\n}\n" }, { "path": "R/Logs.R", "content": "#' @useDynLib batchtools fill_gaps\nreadLog = function(id, missing.as.empty = FALSE, reg = getDefaultRegistry()) {\n log.file = getLogFiles(reg, id)\n if (is.na(log.file) || !waitForFile(log.file, timeout = reg$cluster.functions$fs.latency, must.work = FALSE)) {\n if (missing.as.empty)\n return(data.table(job.id = integer(0L), lines = character(0L)))\n stopf(\"Log file '%s' for job with id %i not available\", log.file, id$job.id)\n }\n\n lines = readLines(log.file)\n if (length(lines) > 0L) {\n job.id = as.integer(stri_match_last_regex(lines, c(\"\\\\[batchtools job\\\\.id=([0-9]+)\\\\]$\"))[, 2L])\n job.id = .Call(fill_gaps, job.id)\n } else {\n job.id = integer(0L)\n }\n\n setkeyv(data.table(job.id = job.id, lines = lines), \"job.id\", physical = FALSE)\n}\n\nextractLog = function(log, id) {\n job.id = NULL\n log[is.na(job.id) | job.id %in% id$job.id]$lines\n}\n\n#' @title Grep Log Files for a Pattern\n#'\n#' @description\n#' Crawls through log files and reports jobs with lines matching the \\code{pattern}.\n#' See \\code{\\link{showLog}} for an example.\n#'\n#' @templateVar ids.default findStarted\n#' @template ids\n#' @param pattern [\\code{character(1L)}]\\cr\n#' Regular expression or string (see \\code{fixed}).\n#' @param ignore.case [\\code{logical(1L)}]\\cr\n#' If \\code{TRUE} the match will be performed case insensitively.\n#' @param fixed [\\code{logical(1L)}]\\cr\n#' If \\code{FALSE} (default), \\code{pattern} is a regular expression and a fixed string otherwise.\n#' @template reg\n#' @export\n#' @family debug\n#' @return [\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id} and \\dQuote{message}.\ngrepLogs = function(ids = NULL, pattern, ignore.case = FALSE, fixed = FALSE, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n assertString(pattern, min.chars = 1L)\n assertFlag(ignore.case)\n assertFlag(fixed)\n job.id = job.hash = log.file = matches = NULL\n\n ids = convertIds(reg, ids, default = .findStarted(reg = reg))\n tab = filter(reg$status[!is.na(job.hash)], ids)[, list(job.id = job.id, hash = sprintf(\"%s-%s\", job.hash, log.file))]\n if (nrow(tab) == 0L)\n return(data.table(job.id = integer(0L), matches = character(0L)))\n\n setorderv(tab, \"hash\")\n res = data.table(job.id = tab$job.id, matches = NA_character_)\n hash.before = \"\"\n matcher = if (fixed) stri_detect_fixed else stri_detect_regex\n\n for (i in seq_row(tab)) {\n if (hash.before != tab$hash[i]) {\n log = readLog(tab[i], missing.as.empty = TRUE, reg = reg)\n hash.before = tab$hash[i]\n }\n\n if (nrow(log) > 0L) {\n lines = extractLog(log, tab[i])\n m = matcher(lines, pattern, case_insensitive = ignore.case)\n if (any(m))\n set(res, i, \"matches\", stri_flatten(lines[m], \"\\n\"))\n }\n }\n\n setkeyv(res[!is.na(matches)], \"job.id\")[]\n}\n\n#' @title Inspect Log Files\n#'\n#' @description\n#' \\code{showLog} opens the log in the pager. For customization, see \\code{\\link[base]{file.show}}.\n#' \\code{getLog} returns the log as character vector.\n#' @template id\n#' @template reg\n#' @export\n#' @family debug\n#' @return Nothing.\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # Create some dummy jobs\n#' fun = function(i) {\n#' if (i == 3) stop(i)\n#' if (i %% 2 == 1) warning(\"That's odd.\")\n#' }\n#' ids = batchMap(fun, i = 1:5, reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#' getStatus(reg = tmp)\n#'\n#' writeLines(getLog(ids[1], reg = tmp))\n#' \\dontrun{\n#' showLog(ids[1], reg = tmp)\n#' }\n#'\n#' grepLogs(pattern = \"warning\", ignore.case = TRUE, reg = tmp)\nshowLog = function(id, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n id = convertId(reg, id)\n lines = extractLog(readLog(id, reg = reg), id)\n log.file = fs::path(fs::path_temp(), sprintf(\"%i.log\", id$job.id))\n writeLines(text = lines, con = log.file)\n file.show(log.file, delete.file = TRUE)\n}\n\n#' @export\n#' @rdname showLog\ngetLog = function(id, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n id = convertId(reg, id)\n extractLog(readLog(id, reg = reg), id)\n}\n" }, { "path": "R/Problem.R", "content": "#' @title Define Problems for Experiments\n#'\n#' @description\n#' Problems may consist of up to two parts: A static, immutable part (\\code{data} in \\code{addProblem})\n#' and a dynamic, stochastic part (\\code{fun} in \\code{addProblem}).\n#' For example, for statistical learning problems a data frame would be the static problem part while\n#' a resampling function would be the stochastic part which creates problem instance.\n#' This instance is then typically passed to a learning algorithm like a wrapper around a statistical model\n#' (\\code{fun} in \\code{\\link{addAlgorithm}}).\n#'\n#' This function serialize all components to the file system and registers the problem in the \\code{\\link{ExperimentRegistry}}.\n#'\n#' \\code{removeProblem} removes all jobs from the registry which depend on the specific problem.\n#' \\code{reg$problems} holds the IDs of already defined problems.\n#'\n#' @param name [\\code{character(1)}]\\cr\n#' Unique identifier for the problem.\n#' @param data [\\code{ANY}]\\cr\n#' Static problem part. Default is \\code{NULL}.\n#' @param fun [\\code{function}]\\cr\n#' The function defining the stochastic problem part.\n#' The static part is passed to this function with name \\dQuote{data} and the \\code{\\link{Job}}/\\code{\\link{Experiment}}\n#' is passed as \\dQuote{job}.\n#' Therefore, your function must have the formal arguments \\dQuote{job} and \\dQuote{data} (or dots \\code{...}).\n#' If you do not provide a function, it defaults to a function which just returns the data part.\n#' @param seed [\\code{integer(1)}]\\cr\n#' Start seed for this problem. This allows the \\dQuote{synchronization} of a stochastic\n#' problem across algorithms, so that different algorithms are evaluated on the same stochastic instance.\n#' If the problem seed is defined, the seeding mechanism works as follows:\n#' (1) Before the dynamic part of a problem is instantiated,\n#' the seed of the problem + [replication number] - 1 is set, i.e. the first\n#' replication uses the problem seed. (2) The stochastic part of the problem is\n#' instantiated. (3) From now on the usual experiment seed of the registry is used,\n#' see \\code{\\link{ExperimentRegistry}}.\n#' If \\code{seed} is set to \\code{NULL} (default), the job seed is used to instantiate the problem and\n#' different algorithms see different stochastic instances of the same problem.\n#' @param cache [\\code{logical(1)}]\\cr\n#' If \\code{TRUE} and \\code{seed} is set, problem instances will be cached on the file system.\n#' This assumes that each problem instance is deterministic for each combination of hyperparameter setting\n#' and each replication number.\n#' This feature is experimental.\n#' @template expreg\n#' @return [\\code{Problem}]. Object of class \\dQuote{Problem} (invisibly).\n#' @aliases Problem\n#' @seealso \\code{\\link{Algorithm}}, \\code{\\link{addExperiments}}\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n#' addProblem(\"p1\", fun = function(job, data) data, reg = tmp)\n#' addProblem(\"p2\", fun = function(job, data) job, reg = tmp)\n#' addAlgorithm(\"a1\", fun = function(job, data, instance) instance, reg = tmp)\n#' addExperiments(repls = 2, reg = tmp)\n#'\n#' # List problems, algorithms and job parameters:\n#' tmp$problems\n#' tmp$algorithms\n#' getJobPars(reg = tmp)\n#'\n#' # Remove one problem\n#' removeProblems(\"p1\", reg = tmp)\n#'\n#' # List problems and algorithms:\n#' tmp$problems\n#' tmp$algorithms\n#' getJobPars(reg = tmp)\naddProblem = function(name, data = NULL, fun = NULL, seed = NULL, cache = FALSE, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE)\n assertString(name, min.chars = 1L)\n if (!stri_detect_regex(name, \"^[[:alnum:]_.-]+$\"))\n stopf(\"Illegal characters in problem name: %s\", name)\n if (is.null(fun)) {\n fun = function(job, data, ...) data\n } else {\n assert(checkFunction(fun, args = c(\"job\", \"data\")), checkFunction(fun, args = \"...\"))\n }\n if (is.null(seed)) {\n cache = FALSE\n } else {\n seed = asCount(seed, positive = TRUE)\n cache = assertFlag(cache)\n }\n\n info(\"Adding problem '%s'\", name)\n prob = setClasses(list(name = name, seed = seed, cache = cache, data = data, fun = fun), \"Problem\")\n writeRDS(prob, file = getProblemURI(reg, name), compress = reg$compress)\n reg$problems = union(reg$problems, name)\n cache.dir = getProblemCacheDir(reg, name)\n if (fs::dir_exists(cache.dir))\n fs::dir_delete(cache.dir)\n if (cache)\n fs::dir_create(cache.dir)\n saveRegistry(reg)\n invisible(prob)\n}\n\n#' @export\n#' @rdname addProblem\nremoveProblems = function(name, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE, running.ok = FALSE)\n assertCharacter(name, any.missing = FALSE)\n assertSubset(name, reg$problems)\n\n problem = NULL\n for (nn in name) {\n def.ids = reg$defs[problem == nn, \"def.id\"]\n job.ids = filter(def.ids, reg$status, \"job.id\")\n\n info(\"Removing Problem '%s' and %i corresponding jobs ...\", nn, nrow(job.ids))\n file_remove(getProblemURI(reg, nn))\n reg$defs = reg$defs[!def.ids]\n reg$status = reg$status[!job.ids]\n reg$problems = chsetdiff(reg$problems, nn)\n cache = getProblemCacheDir(reg, nn)\n if (fs::dir_exists(cache))\n fs::dir_delete(cache)\n }\n\n sweepRegistry(reg)\n invisible(TRUE)\n}\n\ngetProblemURI = function(reg, name) {\n fs::path(dir(reg, \"problems\"), mangle(name))\n}\n\ngetProblemCacheDir = function(reg, name) {\n fs::path(dir(reg, \"cache\"), \"problems\", base32_encode(name, use.padding = FALSE))\n}\n\ngetProblemCacheURI = function(job) {\n fs::path(getProblemCacheDir(job, job$prob.name), sprintf(\"%s.rds\", digest(list(job$prob.name, job$prob.pars, job$repl))))\n}\n" }, { "path": "R/RDSReader.R", "content": "RDSReader = R6Class(\"RDSReader\",\n cloneable = FALSE,\n public = list(\n cache = list(),\n use.cache = NA,\n\n initialize = function(use.cache = FALSE) {\n self$use.cache = use.cache\n },\n\n get = function(uri, slot = NA_character_) {\n read = function(uri) if (fs::file_exists(uri)) readRDS(uri) else NULL\n\n # no cache used, read object from disk and return\n if (!self$use.cache)\n return(read(uri))\n\n # not slotted:\n # look up object in cache. If not found, add to cache. Return cached object\n if (is.na(slot)) {\n if (! uri %chin% names(self$cache))\n self$cache[[uri]] = read(uri)\n return(self$cache[[uri]])\n }\n\n # slotted:\n # object is stored in cache[[slot]] as list(obj = [cached obj], uri = uri)\n if (is.null(self$cache[[slot]]) || self$cache[[slot]]$uri != uri)\n self$cache[[slot]] = list(obj = read(uri), uri = uri)\n return(self$cache[[slot]]$obj)\n }\n )\n)\n" }, { "path": "R/Registry.R", "content": "#' @title Registry Constructor\n#'\n#' @description\n#' \\code{makeRegistry} constructs the inter-communication object for all functions in \\code{batchtools}.\n#' All communication transactions are processed via the file system:\n#' All information required to run a job is stored as \\code{\\link{JobCollection}} in a file in the\n#' a subdirectory of the \\code{file.dir} directory.\n#' Each jobs stores its results as well as computational status information (start time, end time, error message, ...)\n#' also on the file system which is regular merged parsed by the master using \\code{\\link{syncRegistry}}.\n#' After integrating the new information into the Registry, the Registry is serialized to the file system via \\code{\\link{saveRegistry}}.\n#' Both \\code{\\link{syncRegistry}} and \\code{\\link{saveRegistry}} are called whenever required internally.\n#' Therefore it should be safe to quit the R session at any time.\n#' Work can later be resumed by calling \\code{\\link{loadRegistry}} which de-serializes the registry from\n#' the file system.\n#'\n#' The registry created last is saved in the package namespace (unless \\code{make.default} is set to\n#' \\code{FALSE}) and can be retrieved via \\code{\\link{getDefaultRegistry}}.\n#'\n#' Canceled jobs and jobs submitted multiple times may leave stray files behind.\n#' These can be swept using \\code{\\link{sweepRegistry}}.\n#' \\code{\\link{clearRegistry}} completely erases all jobs from a registry, including log files and results,\n#' and thus allows you to start over.\n#'\n#' @details\n#' Currently \\pkg{batchtools} understands the following options set via the configuration file:\n#' \\describe{\n#' \\item{\\code{cluster.functions}:}{As returned by a constructor, e.g. \\code{\\link{makeClusterFunctionsSlurm}}.}\n#' \\item{\\code{default.resources}:}{List of resources to use. Will be overruled by resources specified via \\code{\\link{submitJobs}}.}\n#' \\item{\\code{temp.dir}:}{Path to directory to use for temporary registries.}\n#' \\item{\\code{sleep}:}{Custom sleep function. See \\code{\\link{waitForJobs}}.}\n#' \\item{\\code{expire.after}:}{Number of iterations before treating jobs as expired in \\code{\\link{waitForJobs}}.}\n#' \\item{\\code{compress}:}{Compression algorithm to use via \\code{\\link{saveRDS}}.}\n#' }\n#'\n#' @param file.dir [\\code{character(1)}]\\cr\n#' Path where all files of the registry are saved.\n#' Default is directory \\dQuote{registry} in the current working directory.\n#' The provided path will get normalized unless it is given relative to the home directory\n#' (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.\n#'\n#' If you pass \\code{NA}, a temporary directory will be used.\n#' This way, you can create disposable registries for \\code{\\link{btlapply}} or examples.\n#' By default, the temporary directory \\code{\\link[base]{tempdir}()} will be used.\n#' If you want to use another directory, e.g. a directory which is shared between nodes,\n#' you can set it in your configuration file by setting the variable \\code{temp.dir}.\n#' @param work.dir [\\code{character(1)}]\\cr\n#' Working directory for R process for running jobs.\n#' Defaults to the working directory currently set during Registry construction (see \\code{\\link[base]{getwd}}).\n#' \\code{loadRegistry} uses the stored \\code{work.dir}, but you may also explicitly overwrite it,\n#' e.g., after switching to another system.\n#'\n#' The provided path will get normalized unless it is given relative to the home directory\n#' (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.\n#' @param conf.file [\\code{character(1)}]\\cr\n#' Path to a configuration file which is sourced while the registry is created.\n#' In the configuration file you can define how \\pkg{batchtools} interacts with the system via \\code{\\link{ClusterFunctions}}.\n#' Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.\n#'\n#' The file lookup is implemented in the internal (but exported) function \\code{findConfFile} which returns the first file found of the following candidates:\n#' \\enumerate{\n#' \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n#' \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n#' \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n#' \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n#' \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n#' }\n#' Set to \\code{NA} if you want to suppress reading any configuration file.\n#' If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n#' Therefore you can set and overwrite slots, e.g. \\code{default.resources = list(walltime = 3600)} to set default resources or \\dQuote{max.concurrent.jobs} to\n#' limit the number of jobs allowed to run simultaneously on the system.\n#' @param packages [\\code{character}]\\cr\n#' Packages that will always be loaded on each node.\n#' Uses \\code{\\link[base]{require}} internally.\n#' Default is \\code{character(0)}.\n#' @param namespaces [\\code{character}]\\cr\n#' Same as \\code{packages}, but the packages will not be attached.\n#' Uses \\code{\\link[base]{requireNamespace}} internally.\n#' Default is \\code{character(0)}.\n#' @param source [\\code{character}]\\cr\n#' Files which should be sourced on the slaves prior to executing a job.\n#' Calls \\code{\\link[base]{sys.source}} using the \\code{\\link[base]{.GlobalEnv}}.\n#' @param load [\\code{character}]\\cr\n#' Files which should be loaded on the slaves prior to executing a job.\n#' Calls \\code{\\link[base]{load}} using the \\code{\\link[base]{.GlobalEnv}}.\n#' @param seed [\\code{integer(1)}]\\cr\n#' Start seed for jobs. Each job uses the (\\code{seed} + \\code{job.id}) as seed.\n#' Default is a random integer between 1 and 32768.\n#' Note that there is an additional seeding mechanism to synchronize instantiation of\n#' \\code{\\link{Problem}}s in a \\code{\\link{ExperimentRegistry}}.\n#' @param make.default [\\code{logical(1)}]\\cr\n#' If set to \\code{TRUE}, the created registry is saved inside the package\n#' namespace and acts as default registry. You might want to switch this\n#' off if you work with multiple registries simultaneously.\n#' Default is \\code{TRUE}.\n#' @return [\\code{environment}] of class \\dQuote{Registry} with the following slots:\n#' \\describe{\n#' \\item{\\code{file.dir} [path]:}{File directory.}\n#' \\item{\\code{work.dir} [path]:}{Working directory.}\n#' \\item{\\code{temp.dir} [path]:}{Temporary directory. Used if \\code{file.dir} is \\code{NA} to create temporary registries.}\n#' \\item{\\code{packages} [character()]:}{Packages to load on the slaves.}\n#' \\item{\\code{namespaces} [character()]:}{Namespaces to load on the slaves.}\n#' \\item{\\code{seed} [integer(1)]:}{Registry seed. Before each job is executed, the seed \\code{seed + job.id} is set.}\n#' \\item{\\code{cluster.functions} [cluster.functions]:}{Usually set in your \\code{conf.file}. Set via a call to \\code{\\link{makeClusterFunctions}}. See example.}\n#' \\item{\\code{default.resources} [named list()]:}{Usually set in your \\code{conf.file}. Named list of default resources.}\n#' \\item{\\code{max.concurrent.jobs} [integer(1)]:}{Usually set in your \\code{conf.file}. Maximum number of concurrent jobs for a single user and current registry on the system.\n#' \\code{\\link{submitJobs}} will try to respect this setting. The resource \\dQuote{max.concurrent.jobs} has higher precedence.}\n#' \\item{\\code{defs} [data.table]:}{Table with job definitions (i.e. parameters).}\n#' \\item{\\code{status} [data.table]:}{Table holding information about the computational status. Also see \\code{\\link{getJobStatus}}.}\n#' \\item{\\code{resources} [data.table]:}{Table holding information about the computational resources used for the job. Also see \\code{\\link{getJobResources}}.}\n#' \\item{\\code{tags} [data.table]:}{Table holding information about tags. See \\link{Tags}.}\n#' \\item{\\code{hash} [character(1)]:}{Unique hash which changes each time the registry gets saved to the file system. Can be utilized to invalidate the cache of \\pkg{knitr}.}\n#' }\n#' @aliases Registry\n#' @family Registry\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' print(tmp)\n#'\n#' # Set cluster functions to interactive mode and start jobs in external R sessions\n#' tmp$cluster.functions = makeClusterFunctionsInteractive(external = TRUE)\n#'\n#' # Change packages to load\n#' tmp$packages = c(\"MASS\")\n#' saveRegistry(reg = tmp)\nmakeRegistry = function(file.dir = \"registry\", work.dir = getwd(), conf.file = findConfFile(), packages = character(0L), namespaces = character(0L),\n source = character(0L), load = character(0L), seed = NULL, make.default = TRUE) {\n assertString(file.dir, na.ok = TRUE)\n if (!is.na(file.dir))\n assertPathForOutput(file.dir, overwrite = FALSE)\n assertString(work.dir)\n assertDirectoryExists(work.dir, access = \"r\")\n assertString(conf.file, na.ok = TRUE)\n assertCharacter(packages, any.missing = FALSE, min.chars = 1L)\n assertCharacter(namespaces, any.missing = FALSE, min.chars = 1L)\n assertCharacter(source, any.missing = FALSE, min.chars = 1L)\n assertCharacter(load, any.missing = FALSE, min.chars = 1L)\n assertFlag(make.default)\n seed = if (is.null(seed)) as.integer(runif(1L, 1, 32768)) else asCount(seed, positive = TRUE)\n\n reg = new.env(parent = asNamespace(\"batchtools\"))\n reg$file.dir = file.dir\n reg$work.dir = work.dir\n reg$packages = packages\n reg$namespaces = namespaces\n reg$source = source\n reg$load = load\n reg$seed = seed\n reg$writeable = TRUE\n reg$version = packageVersion(\"batchtools\")\n\n reg$defs = data.table(\n def.id = integer(0L),\n job.pars = list(),\n key = \"def.id\")\n\n reg$status = data.table(\n job.id = integer(0L),\n def.id = integer(0L),\n submitted = double(0L),\n started = double(0L),\n done = double(0L),\n error = character(0L),\n mem.used = double(0L),\n resource.id = integer(0L),\n batch.id = character(0L),\n log.file = character(0L),\n job.hash = character(0L),\n job.name = character(0L),\n key = \"job.id\")\n\n reg$resources = data.table(\n resource.id = integer(0L),\n resource.hash = character(0L),\n resources = list(),\n key = \"resource.id\")\n\n reg$tags = data.table(\n job.id = integer(0L),\n tag = character(0L),\n key = \"job.id\")\n\n setSystemConf(reg, conf.file)\n\n if (is.na(file.dir))\n reg$file.dir = fs::file_temp(\"registry\", tmp_dir = reg$temp.dir)\n \"!DEBUG [makeRegistry]: Creating directories in '`reg$file.dir`'\"\n\n fs::dir_create(c(reg$file.dir, reg$work.dir))\n reg$file.dir = fs::path_abs(reg$file.dir)\n reg$work.dir = fs::path_abs(reg$work.dir)\n\n fs::dir_create(fs::path(reg$file.dir, c(\"jobs\", \"results\", \"updates\", \"logs\", \"exports\", \"external\")))\n with_dir(reg$work.dir, loadRegistryDependencies(reg))\n\n class(reg) = \"Registry\"\n saveRegistry(reg)\n reg$mtime = file_mtime(fs::path(reg$file.dir, \"registry.rds\"))\n reg$hash = rnd_hash()\n info(\"Created registry in '%s' using cluster functions '%s'\", reg$file.dir, reg$cluster.functions$name)\n if (make.default)\n batchtools$default.registry = reg\n return(reg)\n}\n\n#' @export\nprint.Registry = function(x, ...) {\n cat(\"Job Registry\\n\")\n catf(\" Backend : %s\", x$cluster.functions$name)\n catf(\" File dir : %s\", x$file.dir)\n catf(\" Work dir : %s\", x$work.dir)\n catf(\" Jobs : %i\", nrow(x$status))\n catf(\" Seed : %i\", x$seed)\n catf(\" Writeable: %s\", x$writeable)\n}\n\n#' @title assertRegistry\n#'\n#' @description\n#' Assert that a given object is a \\code{batchtools} registry.\n#' Additionally can sync the registry, check if it is writeable, or check if jobs are running.\n#' If any check fails, throws an error indicting the reason for the failure.\n#'\n#' @param reg [\\code{\\link{Registry}}]\\cr\n#' The object asserted to be a \\code{Registry}.\n#' @param class [\\code{character(1)}]\\cr\n#' If \\code{NULL} (default), \\code{reg} must only inherit from class \\dQuote{Registry}.\n#' Otherwise check that \\code{reg} is of class \\code{class}.\n#' E.g., if set to \\dQuote{Registry}, a \\code{\\link{ExperimentRegistry}} would not pass.\n#' @param writeable [\\code{logical(1)}]\\cr\n#' Check if the registry is writeable.\n#' @param sync [\\code{logical(1)}]\\cr\n#' Try to synchronize the registry by including pending results from the file system.\n#' See \\code{\\link{syncRegistry}}.\n#' @param running.ok [\\code{logical(1)}]\\cr\n#' If \\code{FALSE} throw an error if jobs associated with the registry are currently running.\n#' @return \\code{TRUE} invisibly.\n#' @export\nassertRegistry = function(reg, class = NULL, writeable = FALSE, sync = FALSE, running.ok = TRUE) {\n if (batchtools$debug) {\n if (!identical(key(reg$status), \"job.id\"))\n stop(\"Key of reg$job.id lost\")\n if (!identical(key(reg$defs), \"def.id\"))\n stop(\"Key of reg$defs lost\")\n if (!identical(key(reg$resources), \"resource.id\"))\n stop(\"Key of reg$resources lost\")\n }\n\n if (is.null(class)) {\n assertClass(reg, \"Registry\")\n } else {\n assertString(class)\n assertClass(reg, class, ordered = TRUE)\n }\n assertFlag(writeable)\n assertFlag(sync)\n assertFlag(running.ok)\n\n if (reg$writeable && file_mtime(fs::path(reg$file.dir, \"registry.rds\")) > reg$mtime + 1) {\n warning(\"Registry has been altered since last read. Switching to read-only mode in this session. See ?loadRegistry.\")\n reg$writeable = FALSE\n }\n\n if (writeable && !reg$writeable)\n stop(\"Registry must be writeable. See ?loadRegistry.\")\n\n if (!running.ok && nrow(.findOnSystem(reg = reg)) > 0L)\n stop(\"This operation is not allowed while jobs are running on the system\")\n\n if (sync) {\n merged = sync(reg)\n if (length(merged)) {\n saveRegistry(reg)\n file_remove(merged)\n }\n }\n\n invisible(TRUE)\n}\n\nloadRegistryDependencies = function(x, must.work = FALSE) {\n \"!DEBUG [loadRegistryDependencies]: Starting ...\"\n pkgs = union(x$packages, \"methods\")\n handler = if (must.work) stopf else warningf\n ok = vlapply(pkgs, require, character.only = TRUE)\n if (!all(ok))\n handler(\"Failed to load packages: %s\", stri_flatten(pkgs[!ok], \", \"))\n\n ok = vlapply(x$namespaces, requireNamespace)\n if (!all(ok))\n handler(\"Failed to load namespaces: %s\", stri_flatten(x$namespaces[!ok], \", \"))\n\n if (length(x$source) > 0L) {\n for (fn in x$source) {\n ok = try(sys.source(fn, envir = .GlobalEnv), silent = TRUE)\n if (is.error(ok))\n handler(\"Failed to source file '%s': %s\", fn, as.character(ok))\n }\n }\n\n if (length(x$load) > 0L) {\n for (fn in x$load) {\n ok = try(load(fn, envir = .GlobalEnv), silent = TRUE)\n if (is.error(ok))\n handler(\"Failed to load file '%s': %s\", fn, as.character(ok))\n }\n }\n\n path = fs::path(x$file.dir, \"exports\")\n fns = list.files(path, pattern = \"\\\\.rds$\")\n if (length(fns) > 0L) {\n ee = .GlobalEnv\n Map(function(name, fn) {\n delayedAssign(x = name, value = readRDS(fn), assign.env = ee)\n }, name = unmangle(fns), fn = fs::path(path, fns))\n }\n\n invisible(TRUE)\n}\n" }, { "path": "R/Tags.R", "content": "#' @title Add or Remove Job Tags\n#' @name Tags\n#' @rdname Tags\n#'\n#' @description\n#' Add and remove arbitrary tags to jobs.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @param tags [\\code{character}]\\cr\n#' Tags to add or remove as strings. Each tag may consist of letters, numbers, underscore and dots (pattern \\dQuote{^[[:alnum:]_.]+}).\n#' @return [\\code{\\link[data.table]{data.table}}] with job ids affected (invisible).\n#' @template reg\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' ids = batchMap(sqrt, x = -3:3, reg = tmp)\n#'\n#' # Add new tag to all ids\n#' addJobTags(ids, \"needs.computation\", reg = tmp)\n#' getJobTags(reg = tmp)\n#'\n#' # Add more tags\n#' addJobTags(findJobs(x < 0, reg = tmp), \"x.neg\", reg = tmp)\n#' addJobTags(findJobs(x > 0, reg = tmp), \"x.pos\", reg = tmp)\n#' getJobTags(reg = tmp)\n#'\n#' # Submit first 5 jobs and remove tag if successful\n#' ids = submitJobs(1:5, reg = tmp)\n#' if (waitForJobs(reg = tmp))\n#' removeJobTags(ids, \"needs.computation\", reg = tmp)\n#' getJobTags(reg = tmp)\n#'\n#' # Grep for warning message and add a tag\n#' addJobTags(grepLogs(pattern = \"NaNs produced\", reg = tmp), \"div.zero\", reg = tmp)\n#' getJobTags(reg = tmp)\n#'\n#' # All tags where tag x.neg is set:\n#' ids = findTagged(\"x.neg\", reg = tmp)\n#' getUsedJobTags(ids, reg = tmp)\naddJobTags = function(ids = NULL, tags, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE)\n ids = convertIds(reg, ids, default = allIds(reg))\n assertCharacter(tags, any.missing = FALSE, pattern = \"^[[:alnum:]_.]+$\", min.len = 1L)\n\n for (cur in tags) {\n ids[, (\"tag\") := cur]\n reg$tags = rbind(reg$tags, ids)\n }\n reg$tags = setkeyv(unique(reg$tags, by = NULL), \"job.id\")\n\n saveRegistry(reg)\n invisible(ids[, \"job.id\"])\n}\n\n#' @export\n#' @rdname Tags\nremoveJobTags = function(ids = NULL, tags, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE)\n ids = convertIds(reg, ids)\n assertCharacter(tags, any.missing = FALSE, pattern = \"^[[:alnum:]_.]+$\", min.len = 1L)\n job.id = tag = NULL\n\n if (is.null(ids)) {\n i = reg$tags[tag %in% tags, which = TRUE]\n } else {\n i = reg$tags[job.id %in% ids$job.id & tag %in% tags, which = TRUE]\n }\n if (length(i) > 0L) {\n ids = unique(reg$tags[i, \"job.id\"], by = \"job.id\")\n reg$tags = reg$tags[-i]\n saveRegistry(reg)\n } else {\n ids = noIds()\n }\n\n invisible(ids)\n}\n\n#' @export\n#' @rdname Tags\ngetUsedJobTags = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids)\n unique(filter(reg$tags, ids), by = \"tag\")$tag\n}\n" }, { "path": "R/Worker.R", "content": "#' @title Create a Linux-Worker\n#' @docType class\n#' @format An \\code{\\link[R6]{R6Class}} generator object\n#'\n#' @description\n#' \\code{\\link[R6]{R6Class}} to create local and remote linux workers.\n#'\n#' @field nodename Host name. Set via constructor.\n#' @field ncpus Number of CPUs. Set via constructor and defaults to a heuristic which tries to detect the number of CPUs of the machine.\n#' @field max.load Maximum load average (of the last 5 min). Set via constructor and defaults to the number of CPUs of the machine.\n#' @field status Status of the worker; one of \\dQuote{unknown}, \\dQuote{available}, \\dQuote{max.cpus} and \\dQuote{max.load}.\n#' @section Methods:\n#' \\describe{\n#' \\item{\\code{new(nodename, ncpus, max.load)}}{Constructor.}\n#' \\item{\\code{update(reg)}}{Update the worker status.}\n#' \\item{\\code{list(reg)}}{List running jobs.}\n#' \\item{\\code{start(reg, fn, outfile)}}{Start job collection in file \\dQuote{fn} and output to \\dQuote{outfile}.}\n#' \\item{\\code{kill(reg, batch.id)}}{Kill job matching the \\dQuote{batch.id}.}\n#' }\n#' @return [\\code{\\link{Worker}}].\n#' @export\n#' @examples\n#' \\dontrun{\n#' # create a worker for the local machine and use 4 CPUs.\n#' Worker$new(\"localhost\", ncpus = 4)\n#' }\nWorker = R6Class(\"Worker\",\n cloneable = FALSE,\n public = list(\n nodename = NULL,\n ncpus = NULL,\n max.load = NULL,\n script = NULL,\n status = \"unknown\",\n\n initialize = function(nodename, ncpus = NULL, max.load = NULL) {\n if (testOS(\"windows\"))\n stop(\"Windows is not supported by the Worker Class\")\n\n self$nodename = assertString(nodename)\n if (!is.null(ncpus))\n ncpus = asCount(ncpus)\n if (!is.null(max.load))\n assertNumber(max.load)\n\n if (nodename == \"localhost\") {\n self$script = system.file(\"bin\", \"linux-helper\", package = \"batchtools\")\n } else {\n args = c(\"-e\", shQuote(\"message(\\\"[bt] --BOF--\\\\n\\\", \\\"[bt] \\\", system.file(\\\"bin/linux-helper\\\", package = \\\"batchtools\\\"), \\\"\\\\n[bt] --EOF--\\\\n\\\")\"))\n res = runOSCommand(\"Rscript\", args, nodename = nodename)\n script = private$filter_output(res)$output\n if (!testString(script, min.chars = 1L)) {\n stopf(\"Unable to locate helper script on SSH node '%s'. Is batchtools installed on the node?\", nodename)\n }\n }\n\n self$ncpus = ncpus %??% as.integer(private$run(\"number-of-cpus\")$output)\n self$max.load = max.load %??% self$ncpus\n },\n\n list = function(reg) {\n stri_join(self$nodename, \"#\", stri_trim_both(private$run(c(\"list-jobs\", reg$file.dir))$output))\n },\n\n start = function(reg, fn, outfile) {\n private$run(c(\"start-job\", fn, outfile))\n },\n\n kill = function(reg, batch.id) {\n pid = stri_split_fixed(batch.id, \"#\", n = 2L)[[1L]][2L]\n cfKillJob(reg, self$script, c(\"kill-job\", pid))\n },\n\n update = function(reg) {\n \"!DEBUG [Worker]: Updating Worker '`self$nodename`'\"\n res = private$run(c(\"status\", reg$file.dir))\n res = as.numeric(stri_split_regex(res$output, \"\\\\s+\")[[1L]])\n names(res) = c(\"load\", \"n.rprocs\", \"n.rprocs.50\", \"n.jobs\")\n self$status = if (res[\"load\"] > self$max.load) {\n \"max.load\"\n } else if (res[\"n.jobs\"] >= self$ncpus) {\n \"max.cpus\"\n } else {\n \"available\"\n }\n return(res)\n }\n ),\n\n private = list(\n filter_output = function(res) {\n output = stri_trim_both(res$output)\n marker = stri_detect_regex(output, \"^\\\\[bt\\\\] --[BE]OF--$\")\n if (sum(marker) != 2L) {\n stopf(\"runOSCommand failed: Expected BOF+EOF markers for '%s %s', but got:\\n %s\",\n res$sys.cmd, stri_flatten(res$sys.args, \" \"), stri_flatten(res$output, \"\\n\") %??% \"\")\n }\n info = stri_startswith_fixed(output, \"[bt]\") & !marker\n res$output = stri_trim_left(stri_sub(output[info], 5L))\n res\n },\n\n run = function(args) {\n private$filter_output(runOSCommand(self$script, args, nodename = self$nodename))\n }\n )\n)\n" }, { "path": "R/addExperiments.R", "content": "#' @title Add Experiments to the Registry\n#'\n#' @description\n#' Adds experiments (parametrized combinations of problems with algorithms) to the registry and thereby defines batch jobs.\n#'\n#' If multiple problem designs or algorithm designs are provided, they are combined via the Cartesian product.\n#' E.g., if you have two problems \\code{p1} and \\code{p2} and three algorithms \\code{a1}, \\code{a2} and \\code{a3},\n#' \\code{addExperiments} creates experiments for all parameters for the combinations \\code{(p1, a1)}, \\code{(p1, a2)},\n#' \\code{(p1, a3)}, \\code{(p2, a1)}, \\code{(p2, a2)} and \\code{(p2, a3)}.\n#'\n#' @note\n#' R's \\code{data.frame} converts character vectors to factors by default in R versions prior to 4.0.0 which frequently resulted in problems using \\code{addExperiments}.\n#' Therefore, this function will warn about factor variables if the following conditions hold:\n#' \\enumerate{\n#' \\item R version is < 4.0.0\n#' \\item The design is passed as a \\code{data.frame}, not a \\code{\\link[data.table]{data.table}} or \\code{\\link[tibble]{tibble}}.\n#' \\item The option \\dQuote{stringsAsFactors} is not set or set to \\code{TRUE}.\n#' }\n#'\n#' @param prob.designs [named list of \\code{\\link[base]{data.frame}}]\\cr\n#' Named list of data frames (or \\code{\\link[data.table]{data.table}}).\n#' The name must match the problem name while the column names correspond to parameters of the problem.\n#' If \\code{NULL}, experiments for all defined problems without any parameters are added.\n#' @param algo.designs [named list of \\code{\\link[data.table]{data.table}} or \\code{\\link[base]{data.frame}}]\\cr\n#' Named list of data frames (or \\code{\\link[data.table]{data.table}}).\n#' The name must match the algorithm name while the column names correspond to parameters of the algorithm.\n#' If \\code{NULL}, experiments for all defined algorithms without any parameters are added.\n#' @param repls [\\code{integer()}]\\cr\n#' Number of replications for each problem design in `prob.designs` (automatically replicated to\n#' the correct length).\n#' @param combine [\\code{character(1)}]\\cr\n#' How to combine the rows of a single problem design with the rows of a single algorithm design?\n#' Default is \\dQuote{crossprod} which combines each row of the problem design which each row of the algorithm design\n#' in a cross-product fashion. Set to \\dQuote{bind} to just \\code{\\link[base]{cbind}} the tables of\n#' problem and algorithm designs where the shorter table is repeated if necessary.\n#' @template expreg\n#' @return [\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n#' @export\n#' @family Experiment\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # add first problem\n#' fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\n#' addProblem(\"rnorm\", fun = fun, reg = tmp)\n#'\n#' # add second problem\n#' fun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\n#' addProblem(\"rexp\", fun = fun, reg = tmp)\n#'\n#' # add first algorithm\n#' fun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\n#' addAlgorithm(\"average\", fun = fun, reg = tmp)\n#'\n#' # add second algorithm\n#' fun = function(instance, ...) sd(instance)\n#' addAlgorithm(\"deviation\", fun = fun, reg = tmp)\n#'\n#' # define problem and algorithm designs\n#' library(data.table)\n#' prob.designs = algo.designs = list()\n#' prob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\n#' prob.designs$rexp = data.table(n = 100, lambda = 1:5)\n#' algo.designs$average = data.table(method = c(\"mean\", \"median\"))\n#' algo.designs$deviation = data.table()\n#'\n#' # add experiments and submit\n#' addExperiments(prob.designs, algo.designs, reg = tmp)\n#'\n#' # check what has been created\n#' summarizeExperiments(reg = tmp)\n#' unwrap(getJobPars(reg = tmp))\naddExperiments = function(prob.designs = NULL, algo.designs = NULL, repls = 1L, combine = \"crossprod\", reg = getDefaultRegistry()) {\n convertDesigns = function(type, designs, keywords) {\n check.factors = getRversion() < \"4.0.0\" && default.stringsAsFactors()\n\n Map(function(id, design) {\n if (check.factors && identical(class(design)[1L], \"data.frame\")) {\n i = which(vlapply(design, is.factor))\n if (length(i) > 0L) {\n warningf(\"%s design '%s' passed as 'data.frame' and 'stringsAsFactors' is TRUE. Column(s) '%s' may be encoded as factors accidentally.\", type, id, stri_flatten(names(design)[i]), \"','\")\n }\n }\n if (!is.data.table(design))\n design = as.data.table(design)\n i = wf(keywords %chin% names(design))\n if (length(i) > 0L)\n stopf(\"%s design %s contains reserved keyword '%s'\", type, id, keywords[i])\n design\n }, id = names(designs), design = designs)\n }\n\n increment = function(ids, n = 1L) {\n if (length(ids) == 0L) seq_len(n) else max(ids) + seq_len(n)\n }\n\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE)\n if (is.null(prob.designs)) {\n prob.designs = replicate(length(reg$problems), data.table(), simplify = FALSE)\n names(prob.designs) = reg$problems\n } else {\n assertList(prob.designs, types = \"data.frame\", names = \"named\")\n assertSubset(names(prob.designs), reg$problems)\n prob.designs = convertDesigns(\"Problem\", prob.designs, c(\"job\", \"data\"))\n }\n if (is.null(algo.designs)) {\n algo.designs = replicate(length(reg$algorithms), data.table(), simplify = FALSE)\n names(algo.designs) = reg$algorithms\n } else {\n assertList(algo.designs, types = \"data.frame\", names = \"named\")\n assertSubset(names(algo.designs), reg$algorithms)\n algo.designs = convertDesigns(\"Algorithm\", algo.designs, c(\"job\", \"data\", \"instance\"))\n }\n repls = asInteger(repls, lower = 1L, any.missing = FALSE)\n repls = rep_len(repls, length(prob.designs))\n assertChoice(combine, c(\"crossprod\", \"bind\"))\n\n all.ids = integer(0L)\n\n for (i in seq_along(prob.designs)) {\n pn = names(prob.designs)[i]\n pd = prob.designs[[i]]\n n.pd = max(nrow(pd), 1L)\n repls_cur = repls[i]\n\n for (j in seq_along(algo.designs)) {\n an = names(algo.designs)[j]\n ad = algo.designs[[j]]\n n.ad = max(nrow(ad), 1L)\n\n if (combine == \"crossprod\") {\n n.jobs = n.pd * n.ad * repls_cur\n info(\"Adding %i experiments ('%s'[%i] x '%s'[%i] x repls[%i]) ...\", n.jobs, pn, n.pd, an, n.ad, repls_cur)\n idx = CJ(.i = seq_len(n.pd), .j = seq_len(n.ad))\n } else {\n recycle = max(n.pd, n.ad)\n n.jobs = recycle * repls_cur\n info(\"Adding %i experiments (('%s'[%i] | '%s'[%i]) x repls[%i]) ...\", n.jobs, pn, n.pd, an, n.ad, repls_cur)\n idx = data.table(.i = rep_len(seq_len(n.pd), recycle), .j = rep_len(seq_len(n.ad), recycle))\n }\n\n # create temp tab with prob name, algo name and pars as list\n tab = data.table(\n problem = pn,\n algorithm = an,\n prob.pars = if (nrow(pd) > 0L) .mapply(list, pd[idx$.i], list()) else list(list()),\n algo.pars = if (nrow(ad) > 0L) .mapply(list, ad[idx$.j], list()) else list(list())\n )\n\n # create hash of each row of tab\n tab$pars.hash = calculateHash(tab)\n\n # merge with already defined experiments to get def.ids\n if (nrow(reg$defs) == 0L) {\n # this is no optimization, but fixes an strange error on r-devel/windows for merging empty data.tables\n tab$def.id = NA_integer_\n } else {\n tab = merge(reg$defs[, !c(\"problem\", \"algorithm\", \"prob.pars\", \"algo.pars\")], tab, by = \"pars.hash\", all.x = FALSE, all.y = TRUE, sort = FALSE)\n }\n\n # generate def ids for new experiments\n w = which(is.na(tab$def.id))\n if (length(w) > 0L) {\n tab[w, \"def.id\" := increment(reg$defs$def.id, length(w))]\n reg$defs = rbind(reg$defs, tab[w])\n }\n\n # create rows in status table for new defs and each repl and filter for defined\n tab = CJ(def.id = tab$def.id, repl = seq_len(repls_cur))[!reg$status, on = c(\"def.id\", \"repl\")]\n if (nrow(tab) < n.jobs)\n info(\"Skipping %i duplicated experiments ...\", n.jobs - nrow(tab))\n\n if (nrow(tab) > 0L) {\n # rbind new status\n tab$job.id = increment(reg$status$job.id, nrow(tab))\n reg$status = rbind(reg$status, tab, fill = TRUE)\n }\n\n all.ids = c(all.ids, tab$job.id)\n }\n }\n\n if (length(all.ids)) {\n setkeyv(reg$defs, \"def.id\")\n setkeyv(reg$status, \"job.id\")\n saveRegistry(reg)\n }\n invisible(data.table(job.id = all.ids, key = \"job.id\"))\n}\n\ncalculateHash = function(tab) {\n cols = c(\"problem\", \"algorithm\", \"prob.pars\", \"algo.pars\")\n unlist(.mapply(function(...) digest(list(...)), tab[, cols, with = FALSE], list()))\n}\n" }, { "path": "R/batchMap.R", "content": "#' @title Map Operation for Batch Systems\n#'\n#' @description\n#' A parallel and asynchronous \\code{\\link[base]{Map}}/\\code{\\link[base]{mapply}} for batch systems.\n#' Note that this function only defines the computational jobs.\n#' The actual computation is started with \\code{\\link{submitJobs}}.\n#' Results and partial results can be collected with \\code{\\link{reduceResultsList}}, \\code{\\link{reduceResults}} or\n#' \\code{\\link{loadResult}}.\n#'\n#' For a synchronous \\code{\\link[base]{Map}}-like execution, see \\code{\\link{btmapply}}.\n#'\n#' @param fun [\\code{function}]\\cr\n#' Function to map over arguments provided via \\code{...}.\n#' Parameters given via \\code{args} or \\code{...} are passed as-is, in the respective order and possibly named.\n#' If the function has the named formal argument \\dQuote{.job}, the \\code{\\link{Job}} is passed to the function\n#' on the slave.\n#' @param ... [ANY]\\cr\n#' Arguments to vectorize over (list or vector).\n#' Shorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\n#' Mutually exclusive with \\code{args}.\n#' Note that although it is possible to iterate over large objects (e.g., lists of data frames or matrices), this usually\n#' hurts the overall performance and thus is discouraged.\n#' @param args [\\code{list} | \\code{data.frame}]\\cr\n#' Arguments to vectorize over as (named) list or data frame.\n#' Shorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\n#' Mutually exclusive with \\code{...}.\n#' @template more.args\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n#' @export\n#' @seealso \\code{\\link{batchReduce}}\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(3) }\n#' # example using \"...\" and more.args\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' f = function(x, y) x^2 + y\n#' ids = batchMap(f, x = 1:10, more.args = list(y = 100), reg = tmp)\n#' getJobPars(reg = tmp)\n#' testJob(6, reg = tmp) # 100 + 6^2 = 136\n#'\n#' # vector recycling\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' f = function(...) list(...)\n#' ids = batchMap(f, x = 1:3, y = 1:6, reg = tmp)\n#' getJobPars(reg = tmp)\n#'\n#' # example for an expand.grid()-like operation on parameters\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' ids = batchMap(paste, args = data.table::CJ(x = letters[1:3], y = 1:3), reg = tmp)\n#' getJobPars(reg = tmp)\n#' testJob(6, reg = tmp)\nbatchMap = function(fun, ..., args = list(), more.args = list(), reg = getDefaultRegistry()) {\n list2dt = function(x) { # converts a list to a data.table, but avoids creating column names\n nn = names(x)\n if (is.null(nn))\n names(x) = rep.int(\"\", length(x))\n as.data.table(x)\n }\n\n assertRegistry(reg, class = \"Registry\", writeable = TRUE)\n if (nrow(reg$defs) > 0L)\n stop(\"Registry must be empty\")\n assertFunction(fun)\n assert(checkList(args), checkDataFrame(args))\n assertList(more.args)\n\n if (length(args) > 0L) {\n if (...length() > 0L)\n stop(\"You may only provide arguments via '...' *or* 'args'\")\n ddd = list2dt(args)\n } else {\n ddd = list2dt(list(...))\n }\n\n if (\".job\" %chin% names(ddd))\n stop(\"Name '.job' not allowed as parameter name (reserved keyword)\")\n\n if (any(dim(ddd) == 0L))\n return(noIds())\n info(\"Adding %i jobs ...\", nrow(ddd))\n\n writeRDS(fun, file = fs::path(reg$file.dir, \"user.function.rds\"), compress = reg$compress)\n if (length(more.args) > 0L)\n writeRDS(more.args, file = fs::path(reg$file.dir, \"more.args.rds\"), compress = reg$compress)\n ids = seq_row(ddd)\n\n reg$defs = data.table(\n def.id = ids,\n job.pars = .mapply(list, dots = ddd, MoreArgs = list()),\n key = \"def.id\")\n\n reg$status = data.table(\n job.id = ids,\n def.id = ids,\n submitted = NA_real_,\n started = NA_real_,\n done = NA_real_,\n error = NA_character_,\n mem.used = NA_real_,\n resource.id = NA_integer_,\n batch.id = NA_character_,\n log.file = NA_character_,\n job.hash = NA_character_,\n job.name = NA_character_,\n key = \"job.id\")\n\n saveRegistry(reg)\n invisible(allIds(reg))\n}\n" }, { "path": "R/batchMapResults.R", "content": "#' @title Map Over Results to Create New Jobs\n#'\n#' @description\n#' This function allows you to create new computational jobs (just like \\code{\\link{batchMap}} based on the results of\n#' a \\code{\\link{Registry}}.\n#'\n#' @note\n#' The URI to the result files in registry \\code{source} is hard coded as parameter in the \\code{target} registry.\n#' This means that \\code{target} is currently not portable between systems for computation.\n#'\n#' @templateVar ids.default findDone\n#' @param fun [\\code{function}]\\cr\n#' Function which takes the result as first (unnamed) argument.\n#' @template ids\n#' @param ... [ANY]\\cr\n#' Arguments to vectorize over (list or vector). Passed to \\code{\\link{batchMap}}.\n#' @template more.args\n#' @param target [\\code{\\link{Registry}}]\\cr\n#' Empty Registry where new jobs are created for.\n#' @param source [\\code{\\link{Registry}}]\\cr\n#' Registry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).\n#' @return [\\code{\\link[data.table]{data.table}}] with ids of jobs added to \\code{target}.\n#' @export\n#' @family Results\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(2) }\n#' # Source registry: calculate square of some numbers\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(function(x) list(square = x^2), x = 1:10, reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # Target registry: calculate the square root on results of first registry\n#' target = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMapResults(fun = function(x, y) list(sqrt = sqrt(x$square)), ids = 4:8,\n#' target = target, source = tmp)\n#' submitJobs(reg = target)\n#' waitForJobs(reg = target)\n#'\n#' # Map old to new ids. First, get a table with results and parameters\n#' results = unwrap(rjoin(getJobPars(reg = target), reduceResultsDataTable(reg = target)))\n#' print(results)\n#'\n#' # Parameter '.id' points to job.id in 'source'. Use a inner join to combine:\n#' ijoin(results, unwrap(reduceResultsDataTable(reg = tmp)), by = c(\".id\" = \"job.id\"))\nbatchMapResults = function(fun, ids = NULL, ..., more.args = list(), target, source = getDefaultRegistry()) {\n assertRegistry(source, sync = TRUE)\n assertRegistry(target, writeable = TRUE, sync = TRUE)\n assertFunction(fun)\n ids = convertIds(source, ids, default = .findDone(reg = source))\n assertList(more.args, names = \"strict\")\n\n if (nrow(target$status) > 0L)\n stop(\"Target registry 'target' must be empty\")\n\n fns = getResultFiles(source, ids)\n names(fns) = ids$job.id\n more.args = c(list(.fn = fns, .fun = fun), more.args)\n args = c(list(.id = ids$job.id), list(...))\n\n batchMap(batchMapResultsWrapper, args = args, more.args = more.args, reg = target)\n}\n\nbatchMapResultsWrapper = function(.fun, .fn, .id, ...) {\n .fun(readRDS(.fn[[as.character(.id)]]), ...)\n}\n" }, { "path": "R/batchReduce.R", "content": "#' @title Reduce Operation for Batch Systems\n#'\n#' @description\n#' A parallel and asynchronous \\code{\\link[base]{Reduce}} for batch systems.\n#' Note that this function only defines the computational jobs.\n#' Each job reduces a certain number of elements on one slave.\n#' The actual computation is started with \\code{\\link{submitJobs}}.\n#' Results and partial results can be collected with \\code{\\link{reduceResultsList}}, \\code{\\link{reduceResults}} or\n#' \\code{\\link{loadResult}}.\n#'\n#' @param fun [\\code{function(aggr, x, ...)}]\\cr\n#' Function to reduce \\code{xs} with.\n#' @param xs [\\code{vector}]\\cr\n#' Vector to reduce.\n#' @param init [ANY]\\cr\n#' Initial object for reducing. See \\code{\\link[base]{Reduce}}.\n#' @param chunks [\\code{integer(length(xs))}]\\cr\n#' Group for each element of \\code{xs}. Can be generated with \\code{\\link{chunk}}.\n#' @param more.args [\\code{list}]\\cr\n#' A list of additional arguments passed to \\code{fun}.\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n#' @export\n#' @seealso \\code{\\link{batchMap}}\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' # define function to reduce on slave, we want to sum a vector\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' xs = 1:100\n#' f = function(aggr, x) aggr + x\n#'\n#' # sum 20 numbers on each slave process, i.e. 5 jobs\n#' chunks = chunk(xs, chunk.size = 5)\n#' batchReduce(fun = f, 1:100, init = 0, chunks = chunks, reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # now reduce one final time on master\n#' reduceResults(fun = function(aggr, job, res) f(aggr, res), reg = tmp)\nbatchReduce = function(fun, xs, init = NULL, chunks = seq_along(xs), more.args = list(), reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"Registry\", writeable = TRUE)\n if (nrow(reg$defs) > 0L)\n stop(\"Registry must be empty\")\n assertFunction(fun, c(\"aggr\", \"x\"))\n assertAtomicVector(xs)\n assertIntegerish(chunks, len = length(xs), any.missing = FALSE, lower = 0L)\n assertList(more.args, names = \"strict\")\n\n more.args = c(more.args, list(.fun = fun, .init = init))\n batchMap(batchReduceWrapper, unname(split(xs, chunks)), more.args = more.args, reg = reg)\n}\n\nbatchReduceWrapper = function(xs.block, .fun, .init, ...) {\n fun = function(aggr, x) .fun(aggr, x, ...)\n Reduce(fun, xs.block, init = .init)\n}\n" }, { "path": "R/btlapply.R", "content": "#' @title Synchronous Apply Functions\n#'\n#' @description\n#' This is a set of functions acting as counterparts to the sequential popular apply functions in base R:\n#' \\code{btlapply} for \\code{\\link[base]{lapply}} and \\code{btmapply} for \\code{\\link[base]{mapply}}.\n#'\n#' Internally, jobs are created using \\code{\\link{batchMap}} on the provided registry.\n#' If no registry is provided, a temporary registry (see argument \\code{file.dir} of \\code{\\link{makeRegistry}}) and \\code{\\link{batchMap}}\n#' will be used.\n#' After all jobs are terminated (see \\code{\\link{waitForJobs}}), the results are collected and returned as a list.\n#'\n#' Note that these functions are only suitable for short and fail-safe operations\n#' on batch system. If some jobs fail, you have to retrieve partial results from the\n#' registry directory yourself.\n#'\n#' @param X [\\code{\\link[base]{vector}}]\\cr\n#' Vector to apply over.\n#' @param fun [\\code{function}]\\cr\n#' Function to apply.\n#' @param more.args [\\code{list}]\\cr\n#' Additional arguments passed to \\code{fun}.\n#' @param ... [\\code{ANY}]\\cr\n#' Additional arguments passed to \\code{fun} (\\code{btlapply}) or vectors to map over (\\code{btmapply}).\n#' @inheritParams submitJobs\n#' @param n.chunks [\\code{integer(1)}]\\cr\n#' Passed to \\code{\\link{chunk}} before \\code{\\link{submitJobs}}.\n#' @param chunk.size [\\code{integer(1)}]\\cr\n#' Passed to \\code{\\link{chunk}} before \\code{\\link{submitJobs}}.\n#' @template reg\n#' @return [\\code{list}] List with the results of the function call.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' btlapply(1:3, function(x) x^2)\n#' btmapply(function(x, y, z) x + y + z, x = 1:3, y = 1:3, more.args = list(z = 1), simplify = TRUE)\nbtlapply = function(X, fun, ..., resources = list(), n.chunks = NULL, chunk.size = NULL, reg = makeRegistry(file.dir = NA)) {\n assertVector(X)\n assertFunction(fun)\n assertRegistry(reg, class = \"Registry\", writeable = TRUE)\n\n ids = batchMap(fun, X, more.args = list(...), reg = reg)\n if (!is.null(n.chunks) || !is.null(chunk.size))\n ids$chunk = chunk(ids$job.id, n.chunks = n.chunks, chunk.size = chunk.size)\n submitJobs(ids = ids, resources = resources, reg = reg)\n waitForJobs(ids = ids, reg = reg)\n reduceResultsList(ids = ids, reg = reg)\n}\n\n#' @export\n#' @param simplify [\\code{logical(1)}]\\cr\n#' Simplify the results using \\code{\\link[base]{simplify2array}}?\n#' @param use.names [\\code{logical(1)}]\\cr\n#' Use names of the input to name the output?\n#' @rdname btlapply\nbtmapply = function(fun, ..., more.args = list(), simplify = FALSE, use.names = TRUE, resources = list(), n.chunks = NULL, chunk.size = NULL, reg = makeRegistry(file.dir = NA)) {\n assertFunction(fun)\n assertFlag(simplify)\n assertFlag(use.names)\n assertRegistry(reg, class = \"Registry\", writeable = TRUE)\n\n ids = batchMap(fun, ..., more.args = more.args, reg = reg)\n if (!is.null(n.chunks) || !is.null(chunk.size))\n ids$chunk = chunk(ids$job.id, n.chunks = n.chunks, chunk.size = chunk.size)\n submitJobs(ids = ids, resources = resources, reg = reg)\n waitForJobs(ids = ids, reg = reg)\n res = reduceResultsList(ids = ids, reg = reg)\n\n if (use.names) {\n x = head(list(...), 1L)\n if (length(x) > 0L) {\n x = x[[1L]]\n if (is.null(names(x))) {\n if(is.character(x))\n names(res) = x\n } else {\n names(res) = names(x)\n }\n }\n }\n\n if (simplify) simplify2array(res) else res\n}\n" }, { "path": "R/chunkIds.R", "content": "#' @title Chunk Jobs for Sequential Execution\n#'\n#' @description\n#' Jobs can be partitioned into \\dQuote{chunks} to be executed sequentially on the computational nodes.\n#' Chunks are defined by providing a data frame with columns \\dQuote{job.id} and \\dQuote{chunk} (integer)\n#' to \\code{\\link{submitJobs}}.\n#' All jobs with the same chunk number will be grouped together on one node to form a single\n#' computational job.\n#'\n#' The function \\code{chunk} simply splits \\code{x} into either a fixed number of groups, or\n#' into a variable number of groups with a fixed number of maximum elements.\n#'\n#' The function \\code{lpt} also groups \\code{x} into a fixed number of chunks,\n#' but uses the actual values of \\code{x} in a greedy \\dQuote{Longest Processing Time} algorithm.\n#' As a result, the maximum sum of elements in minimized.\n#'\n#' \\code{binpack} splits \\code{x} into a variable number of groups whose sum of elements do\n#' not exceed the upper limit provided by \\code{chunk.size}.\n#'\n#' See examples of \\code{\\link{estimateRuntimes}} for an application of \\code{binpack} and \\code{lpt}.\n#'\n#' @param x [\\code{numeric}]\\cr\n#' For \\code{chunk} an atomic vector (usually the \\code{job.id}).\n#' For \\code{binpack} and \\code{lpt}, the weights to group.\n#' @param chunk.size [\\code{integer(1)}]\\cr\n#' Requested chunk size for each single chunk.\n#' For \\code{chunk} this is the number of elements in \\code{x}, for \\code{binpack} the size\n#' is determined by the sum of values in \\code{x}.\n#' Mutually exclusive with \\code{n.chunks}.\n#' @param n.chunks [\\code{integer(1)}]\\cr\n#' Requested number of chunks.\n#' The function \\code{chunk} distributes the number of elements in \\code{x} evenly while\n#' \\code{lpt} tries to even out the sum of elements in each chunk.\n#' If more chunks than necessary are requested, empty chunks are ignored.\n#' Mutually exclusive with \\code{chunks.size}.\n#' @param shuffle [\\code{logical(1)}]\\cr\n#' Shuffles the groups. Default is \\code{TRUE}.\n#' @return [\\code{integer}] giving the chunk number for each element of \\code{x}.\n#' @seealso \\code{\\link{estimateRuntimes}}\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(2) }\n#' ch = chunk(1:10, n.chunks = 2)\n#' table(ch)\n#'\n#' ch = chunk(rep(1, 10), chunk.size = 2)\n#' table(ch)\n#'\n#' set.seed(1)\n#' x = runif(10)\n#' ch = lpt(x, n.chunks = 2)\n#' sapply(split(x, ch), sum)\n#'\n#' set.seed(1)\n#' x = runif(10)\n#' ch = binpack(x, 1)\n#' sapply(split(x, ch), sum)\n#'\n#' # Job chunking\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' ids = batchMap(identity, 1:25, reg = tmp)\n#'\n#' ### Group into chunks with 10 jobs each\n#' library(data.table)\n#' ids[, chunk := chunk(job.id, chunk.size = 10)]\n#' print(ids[, .N, by = chunk])\n#'\n#' ### Group into 4 chunks\n#' ids[, chunk := chunk(job.id, n.chunks = 4)]\n#' print(ids[, .N, by = chunk])\n#'\n#' ### Submit to batch system\n#' submitJobs(ids = ids, reg = tmp)\n#'\n#' # Grouped chunking\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n#' prob = addProblem(reg = tmp, \"prob1\", data = iris, fun = function(job, data) nrow(data))\n#' prob = addProblem(reg = tmp, \"prob2\", data = Titanic, fun = function(job, data) nrow(data))\n#' algo = addAlgorithm(reg = tmp, \"algo\", fun = function(job, data, instance, i, ...) problem)\n#' prob.designs = list(prob1 = data.table(), prob2 = data.table(x = 1:2))\n#' algo.designs = list(algo = data.table(i = 1:3))\n#' addExperiments(prob.designs, algo.designs, repls = 3, reg = tmp)\n#'\n#' ### Group into chunks of 5 jobs, but do not put multiple problems into the same chunk\n#' # -> only one problem has to be loaded per chunk, and only once because it is cached\n#' ids = getJobTable(reg = tmp)[, .(job.id, problem, algorithm)]\n#' ids[, chunk := chunk(job.id, chunk.size = 5), by = \"problem\"]\n#' ids[, chunk := .GRP, by = c(\"problem\", \"chunk\")]\n#' dcast(ids, chunk ~ problem)\nchunk = function(x, n.chunks = NULL, chunk.size = NULL, shuffle = TRUE) {\n assertAtomicVector(x)\n\n if (!xor(is.null(n.chunks), is.null(chunk.size)))\n stop(\"You must provide either 'n.chunks' (x)or 'chunk.size'\")\n assertCount(n.chunks, positive = TRUE, null.ok = TRUE)\n assertCount(chunk.size, positive = TRUE, null.ok = TRUE)\n assertFlag(shuffle)\n\n n = length(x)\n if (n == 0L)\n return(integer(0L))\n if (is.null(n.chunks))\n n.chunks = (n %/% chunk.size + (n %% chunk.size > 0L))\n chunks = as.integer((seq.int(0L, n - 1L) %% min(n.chunks, n))) + 1L\n if (shuffle)\n chunks = sample(chunks)\n else\n chunks = sort(chunks)\n return(chunks)\n}\n\n#' @rdname chunk\n#' @useDynLib batchtools c_lpt\n#' @export\nlpt = function(x, n.chunks = 1L) {\n assertNumeric(x, lower = 0, any.missing = FALSE, finite = TRUE)\n assertCount(n.chunks, positive = TRUE)\n\n x = as.double(x)\n ord = order(x, decreasing = TRUE)\n n.chunks = as.integer(n.chunks)\n\n .Call(c_lpt, x, ord, n.chunks)\n}\n\n#' @rdname chunk\n#' @useDynLib batchtools c_binpack\n#' @export\nbinpack = function(x, chunk.size = max(x)) {\n assertNumeric(x, lower = 0, any.missing = FALSE, finite = TRUE)\n assertNumber(chunk.size, lower = 0)\n if (length(x) == 0L)\n return(integer(0L))\n\n x = as.double(x)\n ord = order(x, decreasing = TRUE)\n chunk.size = as.double(chunk.size)\n\n .Call(c_binpack, x, ord, chunk.size)\n}\n" }, { "path": "R/clearRegistry.R", "content": "#' @title Remove All Jobs\n#' @description\n#' Removes all jobs from a registry and calls \\code{\\link{sweepRegistry}}.\n#'\n#' @template reg\n#' @family Registry\n#' @export\nclearRegistry = function(reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n info(\"Removing %i jobs ...\", nrow(reg$status))\n reg$status = reg$status[FALSE]\n reg$defs = reg$defs[FALSE]\n reg$resources = reg$resources[FALSE]\n user.fun = fs::path(reg$file.dir, \"user.function.rds\")\n if (fs::file_exists(user.fun)) {\n info(\"Removing user function ...\")\n file_remove(user.fun)\n }\n sweepRegistry(reg = reg)\n}\n" }, { "path": "R/clusterFunctions.R", "content": "#' @title ClusterFunctions Constructor\n#'\n#' @description\n#' This is the constructor used to create \\emph{custom} cluster functions.\n#' Note that some standard implementations for TORQUE, Slurm, LSF, SGE, etc. ship\n#' with the package.\n#'\n#' @param name [\\code{character(1)}]\\cr\n#' Name of cluster functions.\n#' @param submitJob [\\code{function(reg, jc, ...)}]\\cr\n#' Function to submit new jobs. Must return a \\code{\\link{SubmitJobResult}} object.\n#' The arguments are \\code{reg} (\\code{\\link{Registry}}) and \\code{jobs} (\\code{\\link{JobCollection}}).\n#' @param killJob [\\code{function(reg, batch.id)}]\\cr\n#' Function to kill a job on the batch system. Make sure that you definitely kill the job! Return\n#' value is currently ignored. Must have the arguments \\code{reg} (\\code{\\link{Registry}}) and\n#' \\code{batch.id} (\\code{character(1)} as returned by \\code{submitJob}).\n#' Note that there is a helper function \\code{\\link{cfKillJob}} to repeatedly try to kill jobs.\n#' Set \\code{killJob} to \\code{NULL} if killing jobs cannot be supported.\n#' @param listJobsQueued [\\code{function(reg)}]\\cr\n#' List all queued jobs on the batch system for the current user.\n#' Must return an character vector of batch ids, same format as they\n#' are returned by \\code{submitJob}.\n#' Set \\code{listJobsQueued} to \\code{NULL} if listing of queued jobs is not supported.\n#' @param listJobsRunning [\\code{function(reg)}]\\cr\n#' List all running jobs on the batch system for the current user.\n#' Must return an character vector of batch ids, same format as they\n#' are returned by \\code{submitJob}. It does not matter if you return a few job ids too many (e.g.\n#' all for the current user instead of all for the current registry), but you have to include all\n#' relevant ones. Must have the argument are \\code{reg} (\\code{\\link{Registry}}).\n#' Set \\code{listJobsRunning} to \\code{NULL} if listing of running jobs is not supported.\n#' @param array.var [\\code{character(1)}]\\cr\n#' Name of the environment variable set by the scheduler to identify IDs of job arrays.\n#' Default is \\code{NA} for no array support.\n#' @param store.job.collection [\\code{logical(1)}]\\cr\n#' Flag to indicate that the cluster function implementation of \\code{submitJob} can not directly handle \\code{\\link{JobCollection}} objects.\n#' If set to \\code{FALSE}, the \\code{\\link{JobCollection}} is serialized to the file system before submitting the job.\n#' @param store.job.files [\\code{logical(1)}]\\cr\n#' Flag to indicate that job files need to be stored in the file directory.\n#' If set to \\code{FALSE} (default), the job file is created in a temporary directory, otherwise (or if the debug mode is enabled) in\n#' the subdirectory \\code{jobs} of the \\code{file.dir}.\n#' @param scheduler.latency [\\code{numeric(1)}]\\cr\n#' Time to sleep after important interactions with the scheduler to ensure a sane state.\n#' Currently only triggered after calling \\code{\\link{submitJobs}}.\n#' @param fs.latency [\\code{numeric(1)}]\\cr\n#' Expected maximum latency of the file system, in seconds.\n#' Set to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\n#' access files and directories.\n#' Usually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.\n#' @param hooks [\\code{list}]\\cr\n#' Named list of functions which will we called on certain events like \\dQuote{pre.submit} or \\dQuote{post.sync}.\n#' See \\link{Hooks}.\n#' @export\n#' @aliases ClusterFunctions\n#' @family ClusterFunctions\n#' @family ClusterFunctionsHelper\nmakeClusterFunctions = function(name, submitJob, killJob = NULL, listJobsQueued = NULL, listJobsRunning = NULL,\n array.var = NA_character_, store.job.collection = FALSE, store.job.files = FALSE, scheduler.latency = 0,\n fs.latency = 0, hooks = list()) {\n assertList(hooks, types = \"function\", names = \"unique\")\n assertSubset(names(hooks), unlist(batchtools$hooks, use.names = FALSE))\n\n setClasses(list(\n name = assertString(name, min.chars = 1L),\n submitJob = assertFunction(submitJob, c(\"reg\", \"jc\"), null.ok = TRUE),\n killJob = assertFunction(killJob, c(\"reg\", \"batch.id\"), null.ok = TRUE),\n listJobsQueued = assertFunction(listJobsQueued, \"reg\", null.ok = TRUE),\n listJobsRunning = assertFunction(listJobsRunning, \"reg\", null.ok = TRUE),\n array.var = assertString(array.var, na.ok = TRUE),\n store.job.collection = assertFlag(store.job.collection),\n store.job.files = assertFlag(store.job.files),\n scheduler.latency = assertNumber(scheduler.latency, lower = 0),\n fs.latency = assertNumber(fs.latency, lower = 0),\n hooks = hooks),\n \"ClusterFunctions\")\n}\n\n#' @export\nprint.ClusterFunctions = function(x, ...) {\n catf(\"ClusterFunctions for mode: %s\", x$name)\n catf(\" List queued Jobs : %s\", !is.null(x$listJobsQueued))\n catf(\" List running Jobs: %s\", !is.null(x$listJobsRunning))\n catf(\" Kill Jobs : %s\", !is.null(x$killJob))\n catf(\" Hooks : %s\", if (length(x$hooks)) stri_flatten(names(x$hooks), \",\") else \"-\")\n}\n\n#' @title Create a SubmitJobResult\n#'\n#' @description\n#' This function is only intended for use in your own cluster functions implementation.\n#'\n#' Use this function in your implementation of \\code{\\link{makeClusterFunctions}} to create a return\n#' value for the \\code{submitJob} function.\n#'\n#' @param status [\\code{integer(1)}]\\cr\n#' Launch status of job. 0 means success, codes between 1 and 100 are temporary errors and any\n#' error greater than 100 is a permanent failure.\n#' @param batch.id [\\code{character()}]\\cr\n#' Unique id of this job on batch system, as given by the batch system.\n#' Must be globally unique so that the job can be terminated using just this information.\n#' For array jobs, this may be a vector of length equal to the number of jobs in the array.\n#' @param log.file [\\code{character()}]\\cr\n#' Log file. If \\code{NA}, defaults to \\code{[job.hash].log}.\n#' Some cluster functions set this for array jobs.\n#' @param msg [\\code{character(1)}]\\cr\n#' Optional error message in case \\code{status} is not equal to 0. Default is \\dQuote{OK},\n#' \\dQuote{TEMPERROR}, \\dQuote{ERROR}, depending on \\code{status}.\n#' @return [\\code{\\link{SubmitJobResult}}]. A list, containing\n#' \\code{status}, \\code{batch.id} and \\code{msg}.\n#' @family ClusterFunctionsHelper\n#' @aliases SubmitJobResult\n#' @export\nmakeSubmitJobResult = function(status, batch.id, log.file = NA_character_, msg = NA_character_) {\n status = asInt(status)\n if (is.na(msg)) {\n msg = if (status == 0L)\n \"OK\"\n else if (status <= 100L)\n \"TEMPERROR\"\n else\n \"ERROR\"\n }\n \"!DEBUG [makeSubmitJobResult]: Result for batch.id '`paste0(batch.id, sep = ',')`': `status` (`msg`)\"\n\n setClasses(list(status = status, batch.id = batch.id, log.file = log.file, msg = msg), \"SubmitJobResult\")\n}\n\n#' @export\nprint.SubmitJobResult = function(x, ...) {\n cat(\"Job submission result\\n\")\n catf(\" ID : %s\", stri_flatten(x$batch.id, \",\"))\n catf(\" Status: %i\", x$status)\n catf(\" Msg : %s\", x$msg)\n}\n\n#' @title Cluster Functions Helper to Parse a Brew Template\n#'\n#' @description\n#' This function is only intended for use in your own cluster functions implementation.\n#'\n#' This function is only intended for use in your own cluster functions implementation.\n#' Simply reads your template file and returns it as a character vector.\n#'\n#' @param template [\\code{character(1)}]\\cr\n#' Path to template file which is then passed to \\code{\\link[brew]{brew}}.\n#' @param comment.string [\\code{character(1)}]\\cr\n#' Ignore lines starting with this string.\n#' @return [\\code{character}].\n#' @family ClusterFunctionsHelper\n#' @export\ncfReadBrewTemplate = function(template, comment.string = NA_character_) {\n \"!DEBUG [cfReadBrewTemplate]: Parsing template file '`template`'\"\n lines = stri_trim_both(readLines(template))\n\n lines = lines[!stri_isempty(lines)]\n if (!is.na(comment.string))\n lines = lines[!stri_startswith_fixed(lines, comment.string)]\n if (length(lines) == 0L)\n stopf(\"Error reading template '%s' or empty template\", template)\n return(stri_flatten(lines, \"\\n\"))\n}\n\n#' @title Cluster Functions Helper to Write Job Description Files\n#'\n#' @description\n#' This function is only intended for use in your own cluster functions implementation.\n#'\n#' Calls brew silently on your template, any error will lead to an exception.\n#' The file is stored at the same place as the corresponding job file in the \\dQuote{jobs}-subdir\n#' of your files directory.\n#'\n#' @template reg\n#' @param text [\\code{character(1)}]\\cr\n#' String ready to be brewed. See \\code{\\link{cfReadBrewTemplate}} to read a template from the file system.\n#' @param jc [\\code{\\link{JobCollection})}]\\cr\n#' Will be used as environment to brew the template file in. See \\code{\\link{JobCollection}} for a list of all\n#' available variables.\n#' @return [\\code{character(1)}]. File path to brewed template file.\n#' @family ClusterFunctionsHelper\n#' @export\ncfBrewTemplate = function(reg, text, jc) {\n assertString(text)\n outfile = fs::path(dir(reg, \"jobs\"), sprintf(\"%s.job\", jc$job.hash))\n\n parent.env(jc) = asNamespace(\"batchtools\")\n on.exit(parent.env(jc) <- emptyenv())\n \"!DEBUG [cfBrewTemplate]: Brewing template to file '`outfile`'\"\n\n z = try(brew(text = text, output = outfile, envir = jc), silent = TRUE)\n if (is.error(z))\n stopf(\"Error brewing template: %s\", as.character(z))\n waitForFile(outfile, reg$cluster.functions$fs.latency)\n return(outfile)\n}\n\n#' @title Cluster Functions Helper to Handle Unknown Errors\n#'\n#' @description\n#' This function is only intended for use in your own cluster functions implementation.\n#'\n#' Simply constructs a \\code{\\link{SubmitJobResult}} object with status code 101, NA as batch id and\n#' an informative error message containing the output of the OS command in \\code{output}.\n#'\n#' @param cmd [\\code{character(1)}]\\cr\n#' OS command used to submit the job, e.g. qsub.\n#' @param exit.code [\\code{integer(1)}]\\cr\n#' Exit code of the OS command, should not be 0.\n#' @param output [\\code{character}]\\cr\n#' Output of the OS command, hopefully an informative error message.\n#' If these are multiple lines in a vector, they are automatically joined.\n#' @return [\\code{\\link{SubmitJobResult}}].\n#' @family ClusterFunctionsHelper\n#' @export\ncfHandleUnknownSubmitError = function(cmd, exit.code, output) {\n assertString(cmd, min.chars = 1L)\n exit.code = asInt(exit.code)\n assertCharacter(output, any.missing = FALSE)\n msg = sprintf(\"Command '%s' produced exit code %i. Output: '%s'\", cmd, exit.code, stri_flatten(output, \"\\n\"))\n makeSubmitJobResult(status = 101L, batch.id = NA_character_, msg = msg)\n}\n\n#' @title Cluster Functions Helper to Kill Batch Jobs\n#'\n#' @description\n#' This function is only intended for use in your own cluster functions implementation.\n#'\n#' Calls the OS command to kill a job via \\code{\\link[base]{system}} like this: \\dQuote{cmd batch.job.id}. If the\n#' command returns an exit code > 0, the command is repeated after a 1 second sleep\n#' \\code{max.tries-1} times. If the command failed in all tries, an error is generated.\n#'\n#' @template reg\n#' @param cmd [\\code{character(1)}]\\cr\n#' OS command, e.g. \\dQuote{qdel}.\n#' @param args [\\code{character}]\\cr\n#' Arguments to \\code{cmd}, including the batch id.\n#' @param max.tries [\\code{integer(1)}]\\cr\n#' Number of total times to try execute the OS command in cases of failures.\n#' Default is \\code{3}.\n#' @inheritParams runOSCommand\n#' @return \\code{TRUE} on success. An exception is raised otherwise.\n#' @family ClusterFunctionsHelper\n#' @export\ncfKillJob = function(reg, cmd, args = character(0L), max.tries = 3L, nodename = \"localhost\") {\n assertString(cmd, min.chars = 1L)\n assertCharacter(args, any.missing = FALSE)\n assertString(nodename)\n max.tries = asCount(max.tries)\n\n for (i in seq_len(max.tries)) {\n res = runOSCommand(cmd, args, nodename = nodename)\n if (res$exit.code == 0L)\n return(TRUE)\n Sys.sleep(1)\n }\n\n stopf(\"Really tried to kill job, but failed %i times with '%s'.\\nMessage: %s\",\n max.tries, stri_flatten(c(cmd, args), \" \"), stri_flatten(res$output, \"\\n\"))\n}\n\ngetBatchIds = function(reg, status = \"all\") {\n cf = reg$cluster.functions\n tab = data.table(batch.id = character(0L), status = character(0L))\n batch.id = NULL\n\n if (status %chin% c(\"all\", \"running\") && !is.null(cf$listJobsRunning)) {\n \"!DEBUG [getBatchIds]: Getting running Jobs\"\n x = unique(cf$listJobsRunning(reg))\n if (length(x) > 0L)\n tab = rbind(tab, data.table(batch.id = x, status = \"running\"))\n }\n\n if (status %chin% c(\"all\", \"queued\") && !is.null(cf$listJobsQueued)) {\n \"!DEBUG [getBatchIds]: Getting queued Jobs\"\n x = chsetdiff(cf$listJobsQueued(reg), tab$batch.id)\n if (length(x) > 0L)\n tab = rbind(tab, data.table(batch.id = unique(x), status = \"queued\"))\n }\n\n submitted = done = batch.id = NULL\n batch.ids = reg$status[!is.na(submitted) & is.na(done) & !is.na(batch.id), unique(batch.id)]\n tab[batch.id %in% batch.ids]\n}\n\n\n#' @title Find a batchtools Template File\n#'\n#' @description\n#' This functions returns the path to a template file on the file system.\n#' @template template\n#' @return [\\code{character}] Path to the file or \\code{NA} if no template template file was found.\n#' @keywords internal\n#' @export\nfindTemplateFile = function(template) {\n assertString(template, min.chars = 1L)\n\n if (stri_endswith_fixed(template, \".tmpl\")) {\n assertFileExists(template, access = \"r\")\n return(fs::path_abs(template))\n }\n\n x = Sys.getenv(\"R_BATCHTOOLS_SEARCH_PATH\")\n if (nzchar(x)) {\n x = fs::path(x, sprintf(\"batchtools.%s.tmpl\", template))\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n }\n\n x = sprintf(\"batchtools.%s.tmpl\", template)\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n\n x = fs::path(user_config_dir(\"batchtools\", expand = FALSE), sprintf(\"%s.tmpl\", template))\n if (fs::file_access(x, \"read\"))\n return(x)\n\n x = fs::path(\"~\", sprintf(\".batchtools.%s.tmpl\", template))\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n\n x = fs::path(site_config_dir(\"batchtools\"), sprintf(\"%s.tmpl\", template))\n if (fs::file_access(x, \"read\"))\n return(x)\n\n x = system.file(\"templates\", sprintf(\"%s.tmpl\", template), package = \"batchtools\")\n if (fs::file_access(x, \"read\"))\n return(x)\n\n return(NA_character_)\n}\n" }, { "path": "R/clusterFunctionsDocker.R", "content": "#' @title ClusterFunctions for Docker\n#'\n#' @description\n#' Cluster functions for Docker/Docker Swarm (\\url{https://docs.docker.com/engine/swarm/}).\n#'\n#' The \\code{submitJob} function executes\n#' \\code{docker [docker.args] run --detach=true [image.args] [resources] [image] [cmd]}.\n#' Arguments \\code{docker.args}, \\code{image.args} and \\code{image} can be set on construction.\n#' The \\code{resources} part takes the named resources \\code{ncpus} and \\code{memory}\n#' from \\code{\\link{submitJobs}} and maps them to the arguments \\code{--cpu-shares} and \\code{--memory}\n#' (in Megabytes). The resource \\code{threads} is mapped to the environment variables \\dQuote{OMP_NUM_THREADS}\n#' and \\dQuote{OPENBLAS_NUM_THREADS}.\n#' To reliably identify jobs in the swarm, jobs are labeled with \\dQuote{batchtools=[job.hash]} and named\n#' using the current login name (label \\dQuote{user}) and the job hash (label \\dQuote{batchtools}).\n#'\n#' \\code{listJobsRunning} uses \\code{docker [docker.args] ps --format=\\{\\{.ID\\}\\}} to filter for running jobs.\n#'\n#' \\code{killJobs} uses \\code{docker [docker.args] kill [batch.id]} to filter for running jobs.\n#'\n#' These cluster functions use a \\link{Hook} to remove finished jobs before a new submit and every time the \\link{Registry}\n#' is synchronized (using \\code{\\link{syncRegistry}}).\n#' This is currently required because docker does not remove terminated containers automatically.\n#' Use \\code{docker ps -a --filter 'label=batchtools' --filter 'status=exited'} to identify and remove terminated\n#' containers manually (or usa a cron job).\n#'\n#' @param image [\\code{character(1)}]\\cr\n#' Name of the docker image to run.\n#' @param docker.args [\\code{character}]\\cr\n#' Additional arguments passed to \\dQuote{docker} *before* the command (\\dQuote{run}, \\dQuote{ps} or \\dQuote{kill}) to execute (e.g., the docker host).\n#' @param image.args [\\code{character}]\\cr\n#' Additional arguments passed to \\dQuote{docker run} (e.g., to define mounts or environment variables).\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsDocker = function(image, docker.args = character(0L), image.args = character(0L), scheduler.latency = 1, fs.latency = 65) { # nocov start\n assertString(image)\n assertCharacter(docker.args, any.missing = FALSE)\n assertCharacter(image.args, any.missing = FALSE)\n user = Sys.info()[\"user\"]\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n assertIntegerish(jc$resources$ncpus, lower = 1L, any.missing = FALSE, .var.name = \"resources$ncpus\")\n assertIntegerish(jc$resources$memory, lower = 1L, any.missing = FALSE, .var.name = \"resources$memory\")\n timeout = if (is.null(jc$resources$walltime)) character(0L) else sprintf(\"timeout %i\", asInt(jc$resources$walltime, lower = 0L))\n\n cmd = c(\"docker\", docker.args, \"run\", \"--detach=true\", image.args,\n sprintf(\"-e DEBUGME='%s'\", Sys.getenv(\"DEBUGME\")),\n sprintf(\"-e OMP_NUM_THREADS=%i\", jc$resources$omp.threads %??% jc$resources$threads),\n sprintf(\"-e OPENBLAS_NUM_THREADS=%i\", jc$resources$blas.threads %??% jc$resources$threads),\n sprintf(\"-e MKL_NUM_THREADS=%i\", jc$resources$blas.threads %??% jc$resources$threads),\n sprintf(\"-c %i\", jc$resources$ncpus),\n sprintf(\"-m %im\", jc$resources$memory),\n sprintf(\"--memory-swap %im\", jc$resources$memory),\n sprintf(\"--label batchtools=%s\", jc$job.hash),\n sprintf(\"--label user=%s\", user),\n sprintf(\"--name=%s_bt_%s\", user, jc$job.hash),\n image, timeout, \"Rscript\", stri_join(\"-e\", shQuote(sprintf(\"batchtools::doJobCollection('%s', '%s')\", jc$uri, jc$log.file)), sep = \" \"))\n res = runOSCommand(cmd[1L], cmd[-1L])\n\n if (res$exit.code > 0L) {\n housekeeping(reg)\n no.res.msg = \"no resources available\"\n if (res$exit.code == 1L && any(stri_detect_fixed(res$output, no.res.msg)))\n return(makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = no.res.msg))\n return(cfHandleUnknownSubmitError(stri_flatten(cmd, \" \"), res$exit.code, res$output))\n } else {\n if (length(res$output != 1L)) {\n matches = which(stri_detect_regex(res$output, \"^[[:alnum:]]{64}$\"))\n if (length(matches) != 1L)\n stopf(\"Command '%s' did not return a long UUID identitfier\", stri_flatten(cmd, \" \"))\n res$output = res$output[matches]\n }\n return(makeSubmitJobResult(status = 0L, batch.id = stri_sub(res$output, 1L, 12L)))\n }\n }\n\n listJobs = function(reg, filter = character(0L)) {\n assertRegistry(reg, writeable = FALSE)\n # use a workaround for DockerSwarm: docker ps does not list all jobs correctly, only\n # docker inspect reports the status correctly\n args = c(docker.args, \"ps\", \"--format={{.ID}}\", \"--filter 'label=batchtools'\", filter)\n res = runOSCommand(\"docker\", args)\n if (res$exit.code > 0L)\n OSError(\"Listing of jobs failed\", res)\n if (length(res$output) == 0L || !nzchar(res$output))\n return(character(0L))\n res$output\n }\n\n housekeeping = function(reg, ...) {\n batch.ids = chintersect(listJobs(reg, \"--filter 'status=exited'\"), reg$status$batch.id)\n if (length(batch.ids) > 0L)\n runOSCommand(\"docker\", c(docker.args, \"rm\", batch.ids))\n invisible(TRUE)\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"docker\", c(docker.args, \"kill\", batch.id))\n }\n\n listJobsRunning = function(reg) {\n assertRegistry(reg, writeable = FALSE)\n listJobs(reg, sprintf(\"--filter 'user=%s'\", user))\n }\n\n makeClusterFunctions(name = \"Docker\", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,\n store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency,\n hooks = list(post.submit = housekeeping, post.sync = housekeeping))\n} # nocov end\n" }, { "path": "R/clusterFunctionsHyperQueue.R", "content": "#' @title ClusterFunctions for HyperQueue\n#'\n#' @description\n#' Cluster functions for HyperQueue (\\url{https://it4innovations.github.io/hyperqueue/stable/}).\n#'\n#' Jobs are submitted via the HyperQueue CLI using \\code{hq submit} and executed by calling \\code{Rscript -e \"batchtools::doJobCollection(...)\"}.\n#' The job name is set to the job hash and logs are handled internally by batchtools.\n#' Listing jobs uses \\code{hq job list} and cancelling jobs uses \\code{hq job cancel}.\n#' A running HyperQueue server and workers are required.\n#'\n#'\n#' @inheritParams makeClusterFunctions\n#' @return [ClusterFunctions].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsHyperQueue = function(scheduler.latency = 1, fs.latency = 65) {\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n ncpus = if (!is.null(jc$resources$ncpus)) sprintf(\"--cpus=%i\", jc$resources$ncpus)\n memory = if (!is.null(jc$resources$memory)) sprintf(\"--resource mem=%iMiB\", jc$resources$memory)\n # time-limit is the maximum time the job can run, time-request is the minimum remaining lifetime a worker must have\n walltime = if (!is.null(jc$resources$walltime)) sprintf(\"--time-limit=%is --time-request=%is\", jc$resources$walltime, jc$resources$walltime)\n\n\n args = c(\n \"submit\",\n sprintf(\"--name=%s\", jc$job.hash),\n # hyperqueue cannot write stdout and stderr to the same file\n \"--stdout=none\",\n \"--stderr=none\",\n ncpus,\n memory,\n walltime,\n \"--\",\n \"Rscript\", \"-e\",\n shQuote(sprintf(\"batchtools::doJobCollection('%s', '%s')\", jc$uri, jc$log.file))\n )\n res = runOSCommand(\"hq\", args)\n if (res$exit.code > 0L) {\n return(cfHandleUnknownSubmitError(\"hq\", res$exit.code, res$output))\n }\n batch_ids = sub(\".*job ID: ([0-9]+).*\", \"\\\\1\", res$output)\n makeSubmitJobResult(status = 0L, batch.id = batch_ids)\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n args = c(\"job\", \"cancel\", batch.id)\n res = runOSCommand(\"hq\", args)\n if (res$exit.code > 0L) {\n OSError(\"Killing of job failed\", res)\n }\n makeSubmitJobResult(status = 0L, batch.id = batch.id)\n }\n\n\n listJobsQueued = function(reg) {\n requireNamespace(\"jsonlite\")\n assertRegistry(reg, writeable = FALSE)\n args = c(\"job\", \"list\", \"--filter\", \"waiting\", \"--output-mode\", \"json\")\n res = runOSCommand(\"hq\", args)\n if (res$exit.code > 0L) {\n OSError(\"Listing of jobs failed\", res)\n }\n jobs = jsonlite::fromJSON(res$output)\n as.character(jobs$id)\n }\n\n listJobsRunning = function(reg) {\n requireNamespace(\"jsonlite\")\n assertRegistry(reg, writeable = FALSE)\n args = c(\"job\", \"list\", \"--filter\", \"running\", \"--output-mode\", \"json\")\n res = runOSCommand(\"hq\", args)\n if (res$exit.code > 0L) {\n OSError(\"Listing of jobs failed\", res)\n }\n jobs = jsonlite::fromJSON(res$output)\n as.character(jobs$id)\n }\n\n makeClusterFunctions(\n name = \"HyperQueue\",\n submitJob = submitJob,\n killJob = killJob,\n listJobsRunning = listJobsRunning,\n listJobsQueued = listJobsQueued,\n store.job.collection = TRUE,\n scheduler.latency = scheduler.latency,\n fs.latency = fs.latency)\n}\n" }, { "path": "R/clusterFunctionsInteractive.R", "content": "#' @title ClusterFunctions for Sequential Execution in the Running R Session\n#'\n#' @description\n#' All jobs are executed sequentially using the current R process in which \\code{\\link{submitJobs}} is called.\n#' Thus, \\code{submitJob} blocks the session until the job has finished.\n#' The main use of this \\code{ClusterFunctions} implementation is to test and debug programs on a local computer.\n#'\n#' Listing jobs returns an empty vector (as no jobs can be running when you call this)\n#' and \\code{killJob} is not implemented for the same reasons.\n#'\n#' @param external [\\code{logical(1)}]\\cr\n#' If set to \\code{TRUE}, jobs are started in a fresh R session instead of currently active but still\n#' waits for its termination.\n#' Default is \\code{FALSE}.\n#' @param write.logs [\\code{logical(1)}]\\cr\n#' Sink the output to log files. Turning logging off can increase the speed of\n#' calculations but makes it very difficult to debug.\n#' Default is \\code{TRUE}.\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsInteractive = function(external = FALSE, write.logs = TRUE, fs.latency = 0) {\n assertFlag(external)\n assertFlag(write.logs)\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n if (external) {\n runOSCommand(Rscript(), sprintf(\"-e \\\"batchtools::doJobCollection('%s', output = '%s')\\\"\", jc$uri, jc$log.file))\n } else {\n doJobCollection(jc, output = jc$log.file)\n }\n makeSubmitJobResult(status = 0L, batch.id = \"cfInteractive\")\n }\n\n makeClusterFunctions(name = \"Interactive\", submitJob = submitJob, store.job.collection = external, fs.latency = fs.latency)\n}\n" }, { "path": "R/clusterFunctionsLSF.R", "content": "#' @title ClusterFunctions for LSF Systems\n#'\n#' @description\n#' Cluster functions for LSF (\\url{https://www.ibm.com/products/hpc-workload-management}).\n#'\n#' Job files are created based on the brew template \\code{template.file}. This\n#' file is processed with brew and then submitted to the queue using the\n#' \\code{bsub} command. Jobs are killed using the \\code{bkill} command and the\n#' list of running jobs is retrieved using \\code{bjobs -u $USER -w}. The user\n#' must have the appropriate privileges to submit, delete and list jobs on the\n#' cluster (this is usually the case).\n#'\n#' The template file can access all resources passed to \\code{\\link{submitJobs}}\n#' as well as all variables stored in the \\code{\\link{JobCollection}}.\n#' It is the template file's job to choose a queue for the job and handle the desired resource\n#' allocations.\n#'\n#' @note\n#' Array jobs are currently not supported.\n#'\n#' @template template\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsLSF = function(template = \"lsf\", scheduler.latency = 1, fs.latency = 65) { # nocov start\n template = findTemplateFile(template)\n if (testScalarNA(template))\n stopf(\"Argument 'template' (=\\\"%s\\\") must point to a readable template file or contain the template itself as string (containing at least one newline)\", template)\n template = cfReadBrewTemplate(template)\n\n # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only\n # when unfinished jobs are found, and 255 when no jobs are found,\n # or a non-existent job ID is entered.\n Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = \"Y\")\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n outfile = cfBrewTemplate(reg, template, jc)\n res = runOSCommand(\"bsub\", stdin = outfile)\n\n if (res$exit.code > 0L) {\n cfHandleUnknownSubmitError(\"bsub\", res$exit.code, res$output)\n } else {\n batch.id = stri_extract_first_regex(stri_flatten(res$output, \" \"), \"\\\\d+\")\n makeSubmitJobResult(status = 0L, batch.id = batch.id)\n }\n }\n\n listJobs = function(reg, args) {\n assertRegistry(reg, writeable = FALSE)\n res = runOSCommand(\"bjobs\", args)\n if (res$exit.code > 0L) {\n if (res$exit.code == 255L || any(stri_detect_regex(res$output, \"No (unfinished|pending|running) job found\")))\n return(character(0L))\n OSError(\"Listing of jobs failed\", res)\n }\n stri_extract_first_regex(tail(res$output, -1L), \"\\\\d+\")\n }\n\n listJobsQueued = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-w\", \"-p\"))\n }\n\n listJobsRunning = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-w\", \"-r\"))\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"bkill\", batch.id)\n }\n\n makeClusterFunctions(name = \"LSF\", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,\n listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/clusterFunctionsMulticore.R", "content": "if (getRversion() < \"3.3.2\" && .Platform$OS.type != \"windows\") {\n # Provided patch for upstream which is shipped with R >= 3.3.2:\n # https://stat.ethz.ch/pipermail/r-devel/2016-August/073035.html\n selectChildren = getFromNamespace(\"selectChildren\", \"parallel\")\n readChild = getFromNamespace(\"readChild\", \"parallel\")\n\n mccollect = function(pids, timeout = 0) {\n if (!length(pids)) return (NULL)\n if (!is.integer(pids)) stop(\"invalid 'jobs' argument\")\n\n s = selectChildren(pids, timeout)\n if (is.logical(s) || !length(s)) return(NULL)\n res = lapply(s, function(x) {\n r = readChild(x)\n if (is.raw(r)) unserialize(r) else NULL\n })\n names(res) = as.character(pids)[match(s, pids)]\n res\n }\n} else {\n mccollect = function(jobs, timeout = 0) {\n parallel::mccollect(jobs, wait = FALSE, timeout = timeout)\n }\n}\n\nMulticore = R6Class(\"Multicore\",\n cloneable = FALSE,\n public = list(\n jobs = NULL,\n ncpus = NA_integer_,\n\n initialize = function(ncpus) {\n self$jobs = data.table(pid = integer(0L), count = integer(0L))\n self$ncpus = ncpus\n reg.finalizer(self, function(e) mccollect(self$jobs$pid, timeout = 1), onexit = FALSE)\n },\n\n spawn = function(jc) {\n force(jc)\n repeat {\n self$collect(0)\n if (nrow(self$jobs) < self$ncpus)\n break\n Sys.sleep(1)\n }\n pid = parallel::mcparallel(doJobCollection(jc, output = jc$log.file), mc.set.seed = FALSE)$pid\n self$jobs = rbind(self$jobs, data.table(pid = pid, count = 0L))\n invisible(as.character(pid))\n },\n\n list = function() {\n self$collect(0)\n as.character(self$jobs$pid)\n },\n\n collect = function(timeout) {\n repeat {\n res = mccollect(self$jobs$pid, timeout = timeout)\n if (is.null(res))\n break\n pids = as.integer(names(res))\n self$jobs[pid %in% pids, count := count + 1L]\n self$jobs = self$jobs[count < 1L]\n }\n }\n )\n)\n\n#' @title ClusterFunctions for Parallel Multicore Execution\n#'\n#' @description\n#' Jobs are spawned asynchronously using the functions \\code{mcparallel} and \\code{mccollect} (both in \\pkg{parallel}).\n#' Does not work on Windows, use \\code{\\link{makeClusterFunctionsSocket}} instead.\n#'\n#' @template ncpus\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsMulticore = function(ncpus = NA_integer_, fs.latency = 0) {\n if (testOS(\"windows\"))\n stop(\"ClusterFunctionsMulticore do not support Windows. Use makeClusterFunctionsSocket instead.\")\n if (is.na(ncpus)) {\n ncpus = max(as.numeric(getOption(\"mc.cores\")), parallel::detectCores(), 1L, na.rm = TRUE)\n info(\"Auto-detected %i CPUs\", ncpus)\n }\n ncpus = asCount(ncpus, na.ok = FALSE, positive = TRUE)\n p = Multicore$new(ncpus)\n\n submitJob = function(reg, jc) {\n force(jc)\n pid = p$spawn(jc)\n makeSubmitJobResult(status = 0L, batch.id = pid)\n }\n\n listJobsRunning = function(reg) {\n assertRegistry(reg, writeable = FALSE)\n p$list()\n }\n\n makeClusterFunctions(name = \"Multicore\", submitJob = submitJob, listJobsRunning = listJobsRunning,\n store.job.collection = FALSE, fs.latency = fs.latency, hooks = list(pre.sync = function(reg, fns) p$collect(1)))\n}\n" }, { "path": "R/clusterFunctionsOpenLava.R", "content": "#' @title ClusterFunctions for OpenLava\n#'\n#' @description\n#' Cluster functions for OpenLava.\n#'\n#' Job files are created based on the brew template \\code{template}. This\n#' file is processed with brew and then submitted to the queue using the\n#' \\code{bsub} command. Jobs are killed using the \\code{bkill} command and the\n#' list of running jobs is retrieved using \\code{bjobs -u $USER -w}. The user\n#' must have the appropriate privileges to submit, delete and list jobs on the\n#' cluster (this is usually the case).\n#'\n#' The template file can access all resources passed to \\code{\\link{submitJobs}}\n#' as well as all variables stored in the \\code{\\link{JobCollection}}.\n#' It is the template file's job to choose a queue for the job and handle the desired resource\n#' allocations.\n#'\n#' @note\n#' Array jobs are currently not supported.\n#'\n#' @template template\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsOpenLava = function(template = \"openlava\", scheduler.latency = 1, fs.latency = 65) { # nocov start\n template = findTemplateFile(template)\n if (testScalarNA(template))\n stopf(\"Argument 'template' (=\\\"%s\\\") must point to a readable template file\", template)\n template = cfReadBrewTemplate(template)\n\n # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only\n # when unfinished jobs are found, and 255 when no jobs are found,\n # or a non-existent job ID is entered.\n Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = \"Y\")\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n outfile = cfBrewTemplate(reg, template, jc)\n res = runOSCommand(\"bsub\", stdin = shQuote(outfile))\n\n if (res$exit.code > 0L) {\n cfHandleUnknownSubmitError(\"bsub\", res$exit.code, res$output)\n } else {\n batch.id = stri_extract_first_regex(stri_flatten(res$output, \" \"), \"\\\\d+\")\n makeSubmitJobResult(status = 0L, batch.id = batch.id)\n }\n }\n\n listJobs = function(reg, args) {\n assertRegistry(reg, writeable = FALSE)\n res = runOSCommand(\"bjobs\", args)\n if (res$exit.code > 0L) {\n if (res$exit.code == 255L || any(stri_detect_regex(res$output, \"No (unfinished|pending|running) job found\")))\n return(character(0L))\n OSError(\"Listing of jobs failed\", res)\n }\n stri_extract_first_regex(tail(res$output, -1L), \"\\\\d+\")\n }\n\n listJobsQueued = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-w\", \"-p\"))\n }\n\n listJobsRunning = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-w\", \"-r\"))\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"bkill\", batch.id)\n }\n\n makeClusterFunctions(name = \"OpenLava\", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,\n listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/clusterFunctionsSGE.R", "content": "#' @title ClusterFunctions for SGE Systems\n#'\n#' @description\n#' Cluster functions for Univa Grid Engine / Oracle Grid Engine /\n#' Sun Grid Engine (\\url{https://altair.com/hpc-cloud-applications/}).\n#'\n#' Job files are created based on the brew template \\code{template}. This\n#' file is processed with brew and then submitted to the queue using the\n#' \\code{qsub} command. Jobs are killed using the \\code{qdel} command and the\n#' list of running jobs is retrieved using \\code{qselect}. The user must have\n#' the appropriate privileges to submit, delete and list jobs on the cluster\n#' (this is usually the case).\n#'\n#' The template file can access all resources passed to \\code{\\link{submitJobs}}\n#' as well as all variables stored in the \\code{\\link{JobCollection}}.\n#' It is the template file's job to choose a queue for the job and handle the desired resource\n#' allocations.\n#'\n#' @note\n#' Array jobs are currently not supported.\n#'\n#' @template template\n#' @inheritParams makeClusterFunctions\n#' @template nodename\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsSGE = function(template = \"sge\", nodename = \"localhost\", scheduler.latency = 1, fs.latency = 65) { # nocov start\n assertString(nodename)\n template = findTemplateFile(template)\n if (testScalarNA(template))\n stopf(\"Argument 'template' (=\\\"%s\\\") must point to a readable template file\", template)\n template = cfReadBrewTemplate(template)\n quote = if (isLocalHost(nodename)) identity else shQuote\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n outfile = cfBrewTemplate(reg, template, jc)\n res = runOSCommand(\"qsub\", shQuote(outfile), nodename = nodename)\n\n if (res$exit.code > 0L) {\n cfHandleUnknownSubmitError(\"qsub\", res$exit.code, res$output)\n } else {\n batch.id = stri_extract_first_regex(stri_flatten(res$output, \" \"), \"\\\\d+\")\n makeSubmitJobResult(status = 0L, batch.id = batch.id)\n }\n }\n\n listJobs = function(reg, args) {\n assertRegistry(reg, writeable = FALSE)\n res = runOSCommand(\"qstat\", args, nodename = nodename)\n if (res$exit.code > 0L)\n OSError(\"Listing of jobs failed\", res)\n stri_extract_first_regex(tail(res$output, -2L), \"\\\\d+\")\n }\n\n listJobsQueued = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-s p\"))\n }\n\n listJobsRunning = function(reg) {\n listJobs(reg, c(\"-u $USER\", \"-s rs\"))\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"qdel\", batch.id, nodename = nodename)\n }\n\n makeClusterFunctions(name = \"SGE\", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,\n listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/clusterFunctionsSSH.R", "content": "#' @title ClusterFunctions for Remote SSH Execution\n#'\n#' @description\n#' Jobs are spawned by starting multiple R sessions via \\code{Rscript} over SSH.\n#' If the hostname of the \\code{\\link{Worker}} equals \\dQuote{localhost},\n#' \\code{Rscript} is called directly so that you do not need to have an SSH client installed.\n#'\n#' @param workers [\\code{list} of \\code{\\link{Worker}}]\\cr\n#' List of Workers as constructed with \\code{\\link{Worker}}.\n#' @inheritParams makeClusterFunctions\n#'\n#' @note\n#' If you use a custom \\dQuote{.ssh/config} file, make sure your\n#' ProxyCommand passes \\sQuote{-q} to ssh, otherwise each output will\n#' end with the message \\dQuote{Killed by signal 1} and this will break\n#' the communication with the nodes.\n#'\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\n#' @examples\n#' \\dontrun{\n#' # cluster functions for multicore execution on the local machine\n#' makeClusterFunctionsSSH(list(Worker$new(\"localhost\", ncpus = 2)))\n#' }\nmakeClusterFunctionsSSH = function(workers, fs.latency = 65) { # nocov start\n assertList(workers, types = \"Worker\")\n names(workers) = vcapply(workers, \"[[\", \"nodename\")\n if (anyDuplicated(names(workers)))\n stop(\"Duplicated hostnames found in list of workers\")\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n lapply(workers, function(w) w$update(reg))\n rload = vnapply(workers, function(w) w$max.load / w$ncpus)\n worker = Find(function(w) w$status == \"available\", sample(workers, prob = 1 / (rload + 0.1)), nomatch = NULL)\n\n if (!is.null(worker) && worker$status == \"available\") {\n pid = try(worker$start(reg, jc$uri, jc$log.file))\n if (is.error(pid)) {\n makeSubmitJobResult(status = 101L, batch.id = NA_character_, msg = \"Submit failed.\")\n } else {\n makeSubmitJobResult(status = 0L, batch.id = sprintf(\"%s#%s\", worker$nodename, pid$output))\n }\n } else {\n makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = sprintf(\"Busy: %s\", workers[[1L]]$status))\n }\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n nodename = stri_split_fixed(batch.id, \"#\", n = 2L)[[1L]][1L]\n workers[[nodename]]$kill(reg, batch.id)\n }\n\n listJobsRunning = function(reg) {\n assertRegistry(reg, writeable = FALSE)\n unlist(lapply(workers, function(w) w$list(reg)), use.names = FALSE)\n }\n\n makeClusterFunctions(name = \"SSH\", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,\n store.job.collection = TRUE, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/clusterFunctionsSlurm.R", "content": "#' @title ClusterFunctions for Slurm Systems\n#'\n#' @description\n#' Cluster functions for Slurm (\\url{https://slurm.schedmd.com/}).\n#'\n#' Job files are created based on the brew template \\code{template.file}. This\n#' file is processed with brew and then submitted to the queue using the\n#' \\code{sbatch} command. Jobs are killed using the \\code{scancel} command and\n#' the list of running jobs is retrieved using \\code{squeue}. The user must\n#' have the appropriate privileges to submit, delete and list jobs on the\n#' cluster (this is usually the case).\n#'\n#' The template file can access all resources passed to \\code{\\link{submitJobs}}\n#' as well as all variables stored in the \\code{\\link{JobCollection}}.\n#' It is the template file's job to choose a queue for the job and handle the desired resource\n#' allocations.\n#'\n#' Note that you might have to specify the cluster name here if you do not want to use the default,\n#' otherwise the commands for listing and killing jobs will not work.\n#'\n#' @template template\n#' @param array.jobs [\\code{logical(1)}]\\cr\n#' If array jobs are disabled on the computing site, set to \\code{FALSE}.\n#' @template nodename\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsSlurm = function(template = \"slurm\", array.jobs = TRUE, nodename = \"localhost\", scheduler.latency = 1, fs.latency = 65) { # nocov start\n assertFlag(array.jobs)\n assertString(nodename)\n template = findTemplateFile(template)\n if (testScalarNA(template))\n stopf(\"Argument 'template' (=\\\"%s\\\") must point to a readable template file\", template)\n template = cfReadBrewTemplate(template, \"##\")\n quote = if (isLocalHost(nodename)) identity else shQuote\n\n getClusters = function(reg) {\n clusters = filterNull(lapply(reg$resources$resources, \"[[\", \"cluster\"))\n if (length(clusters))\n return(stri_flatten(unique(as.character(clusters)), \",\"))\n return(character(0L))\n }\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n if (jc$array.jobs) {\n logs = sprintf(\"%s_%i\", fs::path_file(jc$log.file), seq_row(jc$jobs))\n jc$log.file = stri_join(jc$log.file, \"_%a\")\n }\n outfile = cfBrewTemplate(reg, template, jc)\n res = runOSCommand(\"sbatch\", shQuote(outfile), nodename = nodename)\n output = stri_flatten(stri_trim_both(res$output), \"\\n\")\n\n if (res$exit.code > 0L) {\n temp.errors = c(\n \"Batch job submission failed: Job violates accounting policy (job submit limit, user's size and/or time limits)\",\n \"Socket timed out on send/recv operation\",\n \"Submission rate too high, suggest using job arrays\"\n )\n i = wf(stri_detect_fixed(output, temp.errors))\n if (length(i) == 1L)\n return(makeSubmitJobResult(status = i, batch.id = NA_character_, msg = temp.errors[i]))\n return(cfHandleUnknownSubmitError(\"sbatch\", res$exit.code, res$output))\n }\n\n id = stri_split_fixed(output[1L], \" \")[[1L]][4L]\n if (jc$array.jobs) {\n if (!array.jobs)\n stop(\"Array jobs not supported by cluster function\")\n makeSubmitJobResult(status = 0L, batch.id = sprintf(\"%s_%i\", id, seq_row(jc$jobs)), log.file = logs)\n } else {\n makeSubmitJobResult(status = 0L, batch.id = id)\n }\n }\n\n listJobs = function(reg, args) {\n assertRegistry(reg, writeable = FALSE)\n args = c(args, \"--noheader\", \"--format=%i\")\n if (array.jobs)\n args = c(args, \"-r\")\n clusters = getClusters(reg)\n if (length(clusters))\n args = c(args, sprintf(\"--clusters=%s\", clusters))\n res = runOSCommand(\"squeue\", args, nodename = nodename)\n if (res$exit.code > 0L)\n OSError(\"Listing of jobs failed\", res)\n if (length(clusters)) tail(res$output, -1L) else res$output\n }\n\n\n # Full List of Slurm job state codes:\n # https://slurm.schedmd.com/squeue.html\n # BF,CA,CD,CF,CG,DL,F,NF,OOM,PD,PR,R,RD,RF,RH,RS,RV,SI,SE,SO,ST,S,TO\n # Querying by RD (RESV_DEL_HOLD) status throwing error on slurm v20.11.4\n \n\n listJobsQueued = function(reg) {\n args = c(quote(\"--user=$USER\"), \"--states=PD,CF,RF,RH,RQ,SE\")\n listJobs(reg, args)\n }\n\n listJobsRunning = function(reg) {\n args = c(quote(\"--user=$USER\"), \"--states=R,S,CG,RS,SI,SO,ST\")\n listJobs(reg, args)\n }\n\n # Slurm job state codes that will result in an expired status:\n # BF,CA,CD,DL,F,NF,OOM,PR,RV,TO,RD\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"scancel\", c(sprintf(\"--clusters=%s\", getClusters(reg)), batch.id), nodename = nodename)\n }\n\n makeClusterFunctions(name = \"Slurm\", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,\n listJobsQueued = listJobsQueued, array.var = \"SLURM_ARRAY_TASK_ID\", store.job.collection = TRUE,\n store.job.files = !isLocalHost(nodename), scheduler.latency = scheduler.latency, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/clusterFunctionsSocket.R", "content": "Socket = R6Class(\"Socket\",\n cloneable = FALSE,\n public = list(\n cl = NULL,\n pids = NULL,\n\n initialize = function(ncpus) {\n loadNamespace(\"snow\")\n self$cl = snow::makeSOCKcluster(rep.int(\"localhost\", ncpus))\n self$pids = character(ncpus)\n reg.finalizer(self, function(e) if (!is.null(e$cl)) { snow::stopCluster(e$cl); self$cl = NULL }, onexit = TRUE)\n },\n\n spawn = function(jc, ...) {\n force(jc)\n if (all(nzchar(self$pids))) {\n res = snow::recvOneResult(self$cl)\n self$pids[self$pids == res$tag] = \"\"\n }\n i = wf(!nzchar(self$pids))\n snow::sendCall(self$cl[[i]], doJobCollection, list(jc = jc, output = jc$log.file), return = FALSE, tag = jc$job.hash)\n self$pids[i] = jc$job.hash\n invisible(jc$job.hash)\n },\n\n list = function() {\n if (is.null(self$cl))\n return(character(0L))\n\n sl = lapply(self$cl, function(x) x$con)\n finished = which(socketSelect(sl, write = FALSE, timeout = 1))\n for (i in seq_along(finished)) {\n res = snow::recvOneResult(self$cl)\n self$pids[self$pids == res$tag] = \"\"\n }\n\n self$pids[nzchar(self$pids)]\n }\n )\n)\n\n#' @title ClusterFunctions for Parallel Socket Execution\n#'\n#' @description\n#' Jobs are spawned asynchronously using the package \\pkg{snow}.\n#'\n#' @template ncpus\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsSocket = function(ncpus = NA_integer_, fs.latency = 65) {\n assertCount(ncpus, positive = TRUE, na.ok = TRUE)\n if (is.na(ncpus)) {\n ncpus = max(getOption(\"mc.cores\", parallel::detectCores()), 1L, na.rm = TRUE)\n info(\"Auto-detected %i CPUs\", ncpus)\n }\n p = Socket$new(ncpus)\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n p$spawn(jc)\n makeSubmitJobResult(status = 0L, batch.id = jc$job.hash)\n }\n\n listJobsRunning = function(reg) {\n assertRegistry(reg, writeable = FALSE)\n p$list()\n }\n\n makeClusterFunctions(name = \"Socket\", submitJob = submitJob, listJobsRunning = listJobsRunning,\n store.job.collection = FALSE, fs.latency = fs.latency, hooks = list(pre.sync = function(reg, fns) p$list()))\n}\n" }, { "path": "R/clusterFunctionsTORQUE.R", "content": "#' @title ClusterFunctions for OpenPBS/TORQUE Systems\n#'\n#' @description\n#' Cluster functions for TORQUE/PBS (\\url{https://adaptivecomputing.com/cherry-services/torque-resource-manager/}).\n#'\n#' Job files are created based on the brew template \\code{template.file}. This file is processed\n#' with brew and then submitted to the queue using the \\code{qsub} command. Jobs are killed using\n#' the \\code{qdel} command and the list of running jobs is retrieved using \\code{qselect}. The user\n#' must have the appropriate privileges to submit, delete and list jobs on the cluster (this is\n#' usually the case).\n#'\n#' The template file can access all resources passed to \\code{\\link{submitJobs}}\n#' as well as all variables stored in the \\code{\\link{JobCollection}}.\n#' It is the template file's job to choose a queue for the job and handle the desired resource\n#' allocations.\n#'\n#' @template template\n#' @inheritParams makeClusterFunctions\n#' @return [\\code{\\link{ClusterFunctions}}].\n#' @family ClusterFunctions\n#' @export\nmakeClusterFunctionsTORQUE = function(template = \"torque\", scheduler.latency = 1, fs.latency = 65) { # nocov start\n template = findTemplateFile(template)\n if (testScalarNA(template))\n stopf(\"Argument 'template' (=\\\"%s\\\") must point to a readable template\", template)\n template = cfReadBrewTemplate(template, \"##\")\n\n submitJob = function(reg, jc) {\n assertRegistry(reg, writeable = TRUE)\n assertClass(jc, \"JobCollection\")\n\n outfile = cfBrewTemplate(reg, template, jc)\n res = runOSCommand(\"qsub\", shQuote(outfile))\n output = stri_flatten(stri_trim_both(res$output), \"\\n\")\n\n if (res$exit.code > 0L) {\n max.jobs.msg = \"Maximum number of jobs already in queue\"\n if (stri_detect_fixed(output, max.jobs.msg) || res$exit.code == 228L)\n return(makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = max.jobs.msg))\n return(cfHandleUnknownSubmitError(\"qsub\", res$exit.code, res$output))\n }\n\n if (jc$array.jobs) {\n logs = sprintf(\"%s-%i\", fs::path_file(jc$log.file), seq_row(jc$jobs))\n makeSubmitJobResult(status = 0L, batch.id = stri_replace_first_fixed(output, \"[]\", stri_paste(\"[\", seq_row(jc$jobs), \"]\")), log.file = logs)\n } else {\n makeSubmitJobResult(status = 0L, batch.id = output)\n }\n }\n\n killJob = function(reg, batch.id) {\n assertRegistry(reg, writeable = TRUE)\n assertString(batch.id)\n cfKillJob(reg, \"qdel\", batch.id)\n }\n\n listJobs = function(reg, args) {\n assertRegistry(reg, writeable = FALSE)\n res = runOSCommand(\"qselect\", args)\n if (res$exit.code > 0L)\n OSError(\"Listing of jobs failed\", res)\n res$output\n }\n\n listJobsQueued = function(reg) {\n args = c(\"-u $USER\", \"-s QW\")\n listJobs(reg, args)\n }\n\n listJobsRunning = function(reg) {\n args = c(\"-u $USER\", \"-s EHRT\")\n listJobs(reg, args)\n }\n\n makeClusterFunctions(name = \"TORQUE\", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,\n listJobsRunning = listJobsRunning, array.var = \"PBS_ARRAYID\", store.job.collection = TRUE,\n scheduler.latency = scheduler.latency, fs.latency = fs.latency)\n} # nocov end\n" }, { "path": "R/config.R", "content": "#' @title Find a batchtools Configuration File\n#'\n#' @description\n#' This functions returns the path to the first configuration file found in the following locations:\n#' \\enumerate{\n#' \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n#' \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n#' \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n#' \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n#' \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n#' }\n#' @return [\\code{character(1)}] Path to the configuration file or \\code{NA} if no configuration file was found.\n#' @keywords internal\n#' @export\nfindConfFile = function() {\n x = Sys.getenv(\"R_BATCHTOOLS_SEARCH_PATH\")\n if (nzchar(x)) {\n x = fs::path(x, \"batchtools.conf.R\")\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n }\n\n x = \"batchtools.conf.R\"\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n\n x = fs::path(user_config_dir(\"batchtools\", expand = FALSE), \"config.R\")\n if (fs::file_access(x, \"read\"))\n return(x)\n\n x = fs::path(\"~\", \".batchtools.conf.R\")\n if (fs::file_access(x, \"read\"))\n return(fs::path_abs(x))\n\n x = fs::path(site_config_dir(\"batchtools\"), \"config.R\")\n if (fs::file_access(x, \"read\"))\n return(x)\n\n return(NA_character_)\n}\n\nsetSystemConf = function(reg, conf.file) {\n reg$cluster.functions = makeClusterFunctionsInteractive()\n reg$default.resources = list()\n reg$temp.dir = fs::path_temp()\n reg$compress = \"gzip\"\n\n if (!is.na(conf.file)) {\n assertString(conf.file)\n info(\"Sourcing configuration file '%s' ...\", conf.file)\n sys.source(conf.file, envir = reg, keep.source = FALSE)\n\n assertClass(reg$cluster.functions, \"ClusterFunctions\")\n assertList(reg$default.resources, names = \"unique\")\n fs::dir_create(reg$temp.dir)\n } else {\n info(\"No readable configuration file found\")\n }\n}\n" }, { "path": "R/doJobCollection.R", "content": "#' @title Execute Jobs of a JobCollection\n#'\n#' @description\n#' Executes every job in a \\code{\\link{JobCollection}}.\n#' This function is intended to be called on the slave.\n#'\n#' @param jc [\\code{\\link{JobCollection}}]\\cr\n#' Either an object of class \\dQuote{JobCollection} as returned by\n#' \\code{\\link{makeJobCollection}} or a string with the path to file\n#' containing a \\dQuote{JobCollection} as RDS file (as stored by \\code{\\link{submitJobs}}).\n#' @param output [\\code{character(1)}]\\cr\n#' Path to a file to write the output to. Defaults to \\code{NULL} which means\n#' that output is written to the active \\code{\\link[base]{sink}}.\n#' Do not set this if your scheduler redirects output to a log file.\n#' @return [\\code{character(1)}]: Hash of the \\code{\\link{JobCollection}} executed.\n#' @family JobCollection\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(identity, 1:2, reg = tmp)\n#' jc = makeJobCollection(1:2, reg = tmp)\n#' doJobCollection(jc)\ndoJobCollection = function(jc, output = NULL) {\n UseMethod(\"doJobCollection\")\n}\n\n\n#' @export\ndoJobCollection.character = function(jc, output = NULL) {\n obj = readRDS(jc)\n force(obj)\n if (!batchtools$debug && !obj$array.jobs) {\n fs::file_delete(jc)\n job = fs::path_ext_set(jc, \"job\")\n if (fs::file_exists(job))\n fs::file_delete(job)\n }\n doJobCollection.JobCollection(obj, output = output)\n}\n\n\n#' @export\ndoJobCollection.JobCollection = function(jc, output = NULL) {\n error = function(msg, ...) {\n now = ustamp()\n updates = data.table(job.id = jc$jobs$job.id, started = now, done = now,\n error = stri_trunc(stri_trim_both(sprintf(msg, ...)), 500L, \" [truncated]\"),\n mem.used = NA_real_, key = \"job.id\")\n writeRDS(updates, file = fs::path(jc$file.dir, \"updates\", sprintf(\"%s.rds\", jc$job.hash)), compress = jc$compress)\n invisible(NULL)\n }\n\n # signal warnings immediately\n opts = options(\"warn\")\n options(warn = 1L)\n on.exit(options(opts))\n\n # setup output connection\n if (!is.null(output)) {\n if (!testPathForOutput(output, overwrite = TRUE))\n return(error(\"Cannot create output file for logging\"))\n fp = file(output, open = \"wt\")\n sink(file = fp)\n sink(file = fp, type = \"message\")\n on.exit({ sink(type = \"message\"); sink(type = \"output\"); close(fp) }, add = TRUE)\n }\n\n # subset array jobs\n if (jc$array.jobs) {\n i = as.integer(Sys.getenv(jc$array.var))\n if (!testInteger(i, any.missing = FALSE, lower = 1L, upper = nrow(jc$jobs)))\n return(error(\"Failed to subset JobCollection using array environment variable '%s' [='%s']\", jc$array.var, i))\n jc$jobs = jc$jobs[i]\n }\n\n # say hi\n n.jobs = nrow(jc$jobs)\n s = now()\n catf(\"### [bt%s]: This is batchtools v%s\", s, packageVersion(\"batchtools\"))\n catf(\"### [bt%s]: Starting calculation of %i jobs\", s, n.jobs)\n catf(\"### [bt%s]: Setting working directory to '%s'\", s, jc$work.dir)\n\n # set work dir\n if (!fs::dir_exists(jc$work.dir))\n return(error(\"Work dir does not exist\"))\n local_dir(jc$work.dir)\n\n # load registry dependencies: packages, source files, ...\n # note that this should happen _before_ parallelMap or foreach is initialized\n ok = try(loadRegistryDependencies(jc, must.work = TRUE), silent = TRUE)\n if (is.error(ok))\n return(error(\"Error loading registry dependencies: %s\", as.character(ok)))\n\n # setup inner parallelization with parallelMap\n if (hasName(jc$resources, \"pm.backend\")) {\n if (!requireNamespace(\"parallelMap\", quietly = TRUE))\n return(error(\"parallelMap not installed\"))\n pm.opts = filterNull(insert(list(mode = jc$resources$pm.backend, cpus = jc$resources$ncpus, show.info = FALSE), jc$resources$pm.opts))\n do.call(parallelMap::parallelStart, pm.opts)\n on.exit(parallelMap::parallelStop(), add = TRUE)\n pm.opts = parallelMap::parallelGetOptions()$settings\n catf(\"### [bt%s]: Using %i CPUs for parallelMap/%s on level '%s'\", s, pm.opts$cpus, pm.opts$mode, if (is.na(pm.opts$level)) \"default\" else pm.opts$level)\n }\n\n # setup inner parallelization with foreach\n if (hasName(jc$resources, \"foreach.backend\")) {\n if (!requireNamespace(\"foreach\", quietly = TRUE))\n return(error(\"Package 'foreach' is not installed\"))\n backend = jc$resources$foreach.backend\n ncpus = jc$resources$ncpus\n\n if (backend == \"seq\") {\n foreach::registerDoSEQ()\n } else if (backend == \"parallel\") {\n if (!requireNamespace(\"doParallel\", quietly = TRUE))\n return(error(\"Package 'doParallel' is not installed\"))\n doParallel::registerDoParallel(cores = ncpus)\n } else if (backend == \"mpi\") {\n if (!requireNamespace(\"doMPI\", quietly = TRUE))\n return(error(\"Package 'doMPI' is not installed\"))\n cl = doMPI::startMPIcluster(count = ncpus)\n doMPI::registerDoMPI(cl)\n on.exit(doMPI::closeCluster(cl), add = TRUE)\n } else {\n return(error(\"Unknwon foreach backend: '%s'\", backend))\n }\n catf(\"### [bt%s]: Using %i CPUs for foreach/%s\", s, ncpus, backend)\n }\n\n # setup memory measurement\n measure.memory = isTRUE(jc$resources$measure.memory)\n catf(\"### [bt%s]: Memory measurement %s\", s, ifelse(measure.memory, \"enabled\", \"disabled\"))\n if (measure.memory) {\n memory.mult = c(if (.Machine$sizeof.pointer == 4L) 28L else 56L, 8L)\n }\n\n # try to pre-fetch some objects from the file system\n reader = RDSReader$new(n.jobs > 1L)\n buf = UpdateBuffer$new(jc$jobs$job.id)\n\n runHook(jc, \"pre.do.collection\", reader = reader)\n\n for (i in seq_len(n.jobs)) {\n job = getJob(jc, i, reader = reader)\n id = job$id\n\n update = list(started = ustamp(), done = NA_integer_, error = NA_character_, mem.used = NA_real_)\n catf(\"### [bt%s]: Starting job [batchtools job.id=%i]\", now(), id)\n if (measure.memory) {\n gc(reset = TRUE)\n result = try(execJob(job))\n update$mem.used = sum(gc()[, 1L] * memory.mult) / 1000000L\n } else {\n result = try(execJob(job))\n }\n update$done = ustamp()\n\n if (is.error(result)) {\n catf(\"\\n### [bt%s]: Job terminated with an exception [batchtools job.id=%i]\", now(), id)\n update$error = stri_trunc(stri_trim_both(as.character(result)), 500L, \" [truncated]\")\n } else {\n catf(\"\\n### [bt%s]: Job terminated successfully [batchtools job.id=%i]\", now(), id)\n writeRDS(result, file = getResultFiles(jc, id), compress = jc$compress)\n }\n buf$add(i, update)\n buf$flush(jc)\n }\n\n runHook(jc, \"post.do.collection\", updates = buf$updates, reader = reader)\n buf$save(jc)\n catf(\"### [bt%s]: Calculation finished!\", now())\n\n invisible(jc$job.hash)\n}\n\n\nUpdateBuffer = R6Class(\"UpdateBuffer\",\n cloneable = FALSE,\n public = list(\n updates = NULL,\n next.update = NA_real_,\n initialize = function(ids) {\n self$updates = data.table(job.id = ids, started = NA_real_, done = NA_real_, error = NA_character_, mem.used = NA_real_, written = FALSE, key = \"job.id\")\n self$next.update = Sys.time() + runif(1L, 60, 300)\n },\n\n add = function(i, x) {\n set(self$updates, i, names(x), x)\n },\n\n save = function(jc) {\n i = self$updates[!is.na(started) & (!written), which = TRUE]\n if (length(i) > 0L) {\n first.id = self$updates$job.id[i[1L]]\n writeRDS(self$updates[i, !\"written\"], file = fs::path(jc$file.dir, \"updates\", sprintf(\"%s-%i.rds\", jc$job.hash, first.id)), compress = jc$compress)\n set(self$updates, i, \"written\", TRUE)\n }\n },\n\n flush = function(jc) {\n now = Sys.time()\n if (now > self$next.update) {\n self$save(jc)\n self$next.update = now + runif(1L, 60, 300)\n }\n }\n\n )\n)\n" }, { "path": "R/estimateRuntimes.R", "content": "#' @title Estimate Remaining Runtimes\n#'\n#' @description\n#' Estimates the runtimes of jobs using the random forest implemented in \\pkg{ranger}.\n#' Observed runtimes are retrieved from the \\code{\\link{Registry}} and runtimes are\n#' predicted for unfinished jobs.\n#'\n#' The estimated remaining time is calculated in the \\code{print} method.\n#' You may also pass \\code{n} here to determine the number of parallel jobs which is then used\n#' in a simple Longest Processing Time (LPT) algorithm to give an estimate for the parallel runtime.\n#'\n#' @param tab [\\code{\\link[data.table]{data.table}}]\\cr\n#' Table with column \\dQuote{job.id} and additional columns to predict the runtime.\n#' Observed runtimes will be looked up in the registry and serve as dependent variable.\n#' All columns in \\code{tab} except \\dQuote{job.id} will be passed to \\code{\\link[ranger]{ranger}} as\n#' independent variables to fit the model.\n#' @param ... [ANY]\\cr\n#' Additional parameters passed to \\code{\\link[ranger]{ranger}}. Ignored for the \\code{print} method.\n#' @template reg\n#' @return [\\code{RuntimeEstimate}] which is a \\code{list} with two named elements:\n#' \\dQuote{runtimes} is a \\code{\\link[data.table]{data.table}} with columns \\dQuote{job.id},\n#' \\dQuote{runtime} (in seconds) and \\dQuote{type} (\\dQuote{estimated} if runtime is estimated,\n#' \\dQuote{observed} if runtime was observed).\n#' The other element of the list named \\dQuote{model}] contains the fitted random forest object.\n#' @export\n#' @seealso \\code{\\link{binpack}} and \\code{\\link{lpt}} to chunk jobs according to their estimated runtimes.\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' # Create a simple toy registry\n#' set.seed(1)\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE, seed = 1)\n#' addProblem(name = \"iris\", data = iris, fun = function(data, ...) nrow(data), reg = tmp)\n#' addAlgorithm(name = \"nrow\", function(instance, ...) nrow(instance), reg = tmp)\n#' addAlgorithm(name = \"ncol\", function(instance, ...) ncol(instance), reg = tmp)\n#' addExperiments(algo.designs = list(nrow = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\n#' addExperiments(algo.designs = list(ncol = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\n#'\n#' # We use the job parameters to predict runtimes\n#' tab = unwrap(getJobPars(reg = tmp))\n#'\n#' # First we need to submit some jobs so that the forest can train on some data.\n#' # Thus, we just sample some jobs from the registry while grouping by factor variables.\n#' library(data.table)\n#' ids = tab[, .SD[sample(nrow(.SD), 5)], by = c(\"problem\", \"algorithm\", \"y\")]\n#' setkeyv(ids, \"job.id\")\n#' submitJobs(ids, reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # We \"simulate\" some more realistic runtimes here to demonstrate the functionality:\n#' # - Algorithm \"ncol\" is 5 times more expensive than \"nrow\"\n#' # - x has no effect on the runtime\n#' # - If y is \"a\" or \"b\", the runtimes are really high\n#' runtime = function(algorithm, x, y) {\n#' ifelse(algorithm == \"nrow\", 100L, 500L) + 1000L * (y %in% letters[1:2])\n#' }\n#' tmp$status[ids, done := done + tab[ids, runtime(algorithm, x, y)]]\n#' rjoin(sjoin(tab, ids), getJobStatus(ids, reg = tmp)[, c(\"job.id\", \"time.running\")])\n#'\n#' # Estimate runtimes:\n#' est = estimateRuntimes(tab, reg = tmp)\n#' print(est)\n#' rjoin(tab, est$runtimes)\n#' print(est, n = 10)\n#'\n#' # Submit jobs with longest runtime first:\n#' ids = est$runtimes[type == \"estimated\"][order(runtime, decreasing = TRUE)]\n#' print(ids)\n#' \\dontrun{\n#' submitJobs(ids, reg = tmp)\n#' }\n#'\n#' # Group jobs into chunks with runtime < 1h\n#' ids = est$runtimes[type == \"estimated\"]\n#' ids[, chunk := binpack(runtime, 3600)]\n#' print(ids)\n#' print(ids[, list(runtime = sum(runtime)), by = chunk])\n#' \\dontrun{\n#' submitJobs(ids, reg = tmp)\n#' }\n#'\n#' # Group jobs into 10 chunks with similar runtime\n#' ids = est$runtimes[type == \"estimated\"]\n#' ids[, chunk := lpt(runtime, 10)]\n#' print(ids[, list(runtime = sum(runtime)), by = chunk])\nestimateRuntimes = function(tab, ..., reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n data = copy(convertIds(reg, tab, keep.extra = names(tab)))\n\n if (!requireNamespace(\"ranger\", quietly = TRUE))\n stop(\"Please install package 'ranger' for runtime estimation\")\n\n data[, \"runtime\" := as.numeric(getJobStatus(tab, reg)$time.running)]\n i = is.na(data$runtime)\n if (all(i))\n stop(\"No training data available. Some jobs must be finished before estimating runtimes.\")\n\n rf = ranger::ranger(runtime ~ ., data = data[!i, !\"job.id\"], ...)\n data[i, \"runtime\" := predict(rf, .SD)$predictions, .SDcols = chsetdiff(names(data), c(\"job.id\", \"runtime\"))]\n data$type = factor(ifelse(i, \"estimated\", \"observed\"), levels = c(\"observed\", \"estimated\"))\n setClasses(list(runtimes = data[, c(\"job.id\", \"type\", \"runtime\")], model = rf), c(\"RuntimeEstimate\", class(data)))\n}\n\n\n#' @rdname estimateRuntimes\n#' @param x [\\code{RuntimeEstimate}]\\cr\n#' Object to print.\n#' @param n [\\code{integer(1)}]\\cr\n#' Number of parallel jobs to assume for runtime estimation.\n#' @export\nprint.RuntimeEstimate = function(x, n = 1L, ...) {\n ps = function(x, nc = 2L) {\n sprintf(paste0(\"%0\", nc, \"id %02ih %02im %.1fs\"),\n floor(x / 86400),\n floor((x / 3600) %% 24L),\n floor((x / 60) %% 60L),\n x %% 60L\n )\n }\n\n assertCount(n, positive = TRUE)\n runtime = type = NULL\n calculated = x$runtimes[type == \"observed\", sum(runtime)]\n remaining = x$runtimes[type == \"estimated\", sum(runtime)]\n total = calculated + remaining\n nc = max(1L, nchar(total %/% 86400))\n\n catf(\"Runtime Estimate for %i jobs with %i CPUs\", nrow(x$runtimes), n)\n catf(\" Done : %s\", ps(calculated, nc = nc))\n if (x$runtimes[type == \"estimated\", .N] > 0L) {\n catf(\" Remaining: %s\", ps(remaining, nc = nc))\n if (n >= 2L) {\n rt = x$runtimes[type == \"estimated\"]$runtime\n bins = lpt(rt, n)\n bins = vnapply(split(rt, bins), sum)\n catf(\" Parallel : %s\", ps(max(bins), nc = nc))\n }\n }\n catf(\" Total : %s\", ps(total, nc = nc))\n}\n" }, { "path": "R/execJob.R", "content": "#' @title Execute a Single Jobs\n#'\n#' @description\n#' Executes a single job (as created by \\code{\\link{makeJob}}) and returns\n#' its result. Also works for Experiments.\n#'\n#' @param job [\\code{\\link{Job}} | \\code{\\link{Experiment}}]\\cr\n#' Job/Experiment to execute.\n#' @return Result of the job.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(identity, 1:2, reg = tmp)\n#' job = makeJob(1, reg = tmp)\n#' execJob(job)\nexecJob = function(job) {\n UseMethod(\"execJob\")\n}\n\n#' @export\nexecJob.character = function(job) {\n execJob(readRDS(job))\n}\n\n#' @export\nexecJob.JobCollection = function(job) {\n if (nrow(job$jobs) != 1L)\n stop(\"You must provide a JobCollection with exactly one job\")\n execJob(getJob(job, i = 1L))\n}\n\n#' @export\nexecJob.Job = function(job) {\n opts = options(\"error\")\n options(error = function(e) traceback(2L))\n on.exit(options(opts))\n # this needs to be cat, message outputs to stderr which R cannot capture properly\n catf(\"### [bt%s]: Setting seed to %i ...\", now(), job$seed)\n if (\".job\" %chin% names(formals(job$fun))) {\n with_seed(job$seed, do.call(job$fun, c(job$pars, list(.job = job)), envir = .GlobalEnv))\n } else {\n with_seed(job$seed, do.call(job$fun, job$pars, envir = .GlobalEnv))\n }\n}\n\n#' @export\nexecJob.Experiment = function(job) {\n opts = options(\"error\")\n options(error = function(e) traceback(2L))\n on.exit(options(opts))\n # this needs to be cat, message outputs to stderr which R cannot capture properly\n catf(\"### [bt%s]: Generating problem instance for problem '%s' ...\", now(), job$prob.name)\n instance = job$instance\n force(instance)\n job$allow.access.to.instance = FALSE\n\n wrapper = function(...) job$algorithm$fun(job = job, data = job$problem$data, instance = instance, ...)\n # this needs to be cat, message outputs to stderr which R cannot capture properly\n catf(\"### [bt%s]: Applying algorithm '%s' on problem '%s' for job %i (seed = %i) ...\", now(), job$algo.name, job$prob.name, job$id, job$seed)\n with_seed(job$seed, do.call(wrapper, job$algo.pars, envir = .GlobalEnv))\n}\n" }, { "path": "R/files.R", "content": "dir = function(reg, what) {\n fs::path(fs::path_expand(reg$file.dir), what)\n}\n\ngetResultFiles = function(reg, ids) {\n fs::path(dir(reg, \"results\"), sprintf(\"%i.rds\", if (is.atomic(ids)) ids else ids$job.id))\n}\n\ngetLogFiles = function(reg, ids) {\n job.hash = log.file = NULL\n tab = reg$status[list(ids), c(\"job.id\", \"job.hash\", \"log.file\")]\n tab[is.na(log.file) & !is.na(job.hash), log.file := sprintf(\"%s.log\", job.hash)]\n tab[!is.na(log.file), log.file := fs::path(dir(reg, \"logs\"), log.file)]$log.file\n}\n\ngetJobFiles = function(reg, hash) {\n fs::path(reg$file.dir, \"jobs\", sprintf(\"%s.rds\", hash))\n}\n\ngetExternalDirs = function(reg, ids) {\n fs::path(dir(reg, \"external\"), if (is.atomic(ids)) ids else ids$job.id)\n}\n\nmangle = function(x) {\n sprintf(\"%s.rds\", base32_encode(x, use.padding = FALSE))\n}\n\nunmangle = function(x) {\n base32_decode(stri_sub(x, to = -5L), use.padding = FALSE)\n}\n\nfile_remove = function(x) {\n fs::file_delete(x[fs::file_exists(x)])\n\n while(any(i <- fs::file_exists(x))) {\n Sys.sleep(0.5)\n fs::file_delete(x[i])\n }\n}\n\nfile_mtime = function(x) {\n fs::file_info(x)$modification_time\n}\n\nwriteRDS = function(object, file, compress = \"gzip\") {\n file_remove(file)\n saveRDS(object, file = file, version = 2L, compress = compress)\n waitForFile(file, 300)\n invisible(TRUE)\n}\n" }, { "path": "R/findJobs.R", "content": "#' @title Find and Filter Jobs\n#'\n#' @description\n#' These functions are used to find and filter jobs, depending on either their parameters (\\code{findJobs} and\n#' \\code{findExperiments}), their tags (\\code{findTagged}), or their computational status (all other functions,\n#' see \\code{\\link{getStatus}} for an overview).\n#'\n#' Note that \\code{findQueued}, \\code{findRunning}, \\code{findOnSystem} and \\code{findExpired} are somewhat heuristic\n#' and may report misleading results, depending on the state of the system and the \\code{\\link{ClusterFunctions}} implementation.\n#'\n#' See \\code{\\link{JoinTables}} for convenient set operations (unions, intersects, differences) on tables with job ids.\n#'\n#' @param expr [\\code{expression}]\\cr\n#' Predicate expression evaluated in the job parameters.\n#' Jobs for which \\code{expr} evaluates to \\code{TRUE} are returned.\n#' @templateVar ids.default all\n#' @template ids\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with column \\dQuote{job.id} containing matched jobs.\n#' @seealso \\code{\\link{getStatus}} \\code{\\link{JoinTables}}\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(identity, i = 1:3, reg = tmp)\n#' ids = findNotSubmitted(reg = tmp)\n#'\n#' # get all jobs:\n#' findJobs(reg = tmp)\n#'\n#' # filter for jobs with parameter i >= 2\n#' findJobs(i >= 2, reg = tmp)\n#'\n#' # filter on the computational status\n#' findSubmitted(reg = tmp)\n#' findNotDone(reg = tmp)\n#'\n#' # filter on tags\n#' addJobTags(2:3, \"my_tag\", reg = tmp)\n#' findTagged(tags = \"my_tag\", reg = tmp)\n#'\n#' # combine filter functions using joins\n#' # -> jobs which are not done and not tagged (using an anti-join):\n#' ajoin(findNotDone(reg = tmp), findTagged(\"my_tag\", reg = tmp))\nfindJobs = function(expr, ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n ids = convertIds(reg, ids)\n if (missing(expr))\n return(ids %??% allIds(reg))\n\n expr = substitute(expr)\n ee = parent.frame()\n fun = function(pars) eval(expr, pars, enclos = ee)\n job.pars = NULL\n setkeyv(mergedJobs(reg, ids, c(\"job.id\", \"job.pars\"))[vlapply(job.pars, fun), \"job.id\"], \"job.id\")\n}\n\n#' @export\n#' @rdname findJobs\n#' @param prob.name [\\code{character}]\\cr\n#' Exact name of the problem (no substring matching).\n#' If not provided, all problems are matched.\n#' @param prob.pattern [\\code{character}]\\cr\n#' Regular expression pattern to match problem names.\n#' If not provided, all problems are matched.\n#' @param algo.name [\\code{character}]\\cr\n#' Exact name of the problem (no substring matching).\n#' If not provided, all algorithms are matched.\n#' @param algo.pattern [\\code{character}]\\cr\n#' Regular expression pattern to match algorithm names.\n#' If not provided, all algorithms are matched.\n#' @param prob.pars [\\code{expression}]\\cr\n#' Predicate expression evaluated in the problem parameters.\n#' @param algo.pars [\\code{expression}]\\cr\n#' Predicate expression evaluated in the algorithm parameters.\n#' @param repls [\\code{integer}]\\cr\n#' Whitelist of replication numbers. If not provided, all replications are matched.\nfindExperiments = function(ids = NULL, prob.name = NA_character_, prob.pattern = NA_character_, algo.name = NA_character_, algo.pattern = NA_character_, prob.pars, algo.pars, repls = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", sync = TRUE)\n assertString(prob.name, na.ok = TRUE, min.chars = 1L)\n assertString(prob.pattern, na.ok = TRUE, min.chars = 1L)\n assertString(algo.name, na.ok = TRUE, min.chars = 1L)\n assertString(algo.pattern, na.ok = TRUE, min.chars = 1L)\n ee = parent.frame()\n tab = mergedJobs(reg, convertIds(reg, ids), c(\"job.id\", \"problem\", \"algorithm\", \"prob.pars\", \"algo.pars\", \"repl\"))\n\n if (!is.na(prob.name)) {\n problem = NULL\n tab = tab[problem == prob.name]\n }\n\n if (!is.na(prob.pattern)) {\n problem = NULL\n tab = tab[stri_detect_regex(problem, prob.pattern)]\n }\n\n if (!is.na(algo.name)) {\n algorithm = NULL\n tab = tab[algorithm == algo.name]\n }\n\n if (!is.na(algo.pattern)) {\n algorithm = NULL\n tab = tab[stri_detect_regex(algorithm, algo.pattern)]\n }\n\n if (!is.null(repls)) {\n repls = asInteger(repls, any.missing = FALSE)\n repl = NULL\n tab = tab[repl %in% repls]\n }\n\n if (!missing(prob.pars)) {\n expr = substitute(prob.pars)\n fun = function(pars) eval(expr, pars, enclos = ee)\n prob.pars = NULL\n tab = tab[vlapply(prob.pars, fun)]\n }\n\n if (!missing(algo.pars)) {\n expr = substitute(algo.pars)\n fun = function(pars) eval(expr, pars, enclos = ee)\n algo.pars = NULL\n tab = tab[vlapply(algo.pars, fun)]\n }\n\n setkeyv(tab[, \"job.id\"], \"job.id\")[]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindSubmitted = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findSubmitted(reg, convertIds(reg, ids))\n}\n\n.findSubmitted = function(reg, ids = NULL) {\n submitted = NULL\n filter(reg$status, ids, c(\"job.id\", \"submitted\"))[!is.na(submitted), \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindNotSubmitted = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findNotSubmitted(reg, convertIds(reg, ids))\n}\n\n.findNotSubmitted = function(reg, ids = NULL) {\n submitted = NULL\n filter(reg$status, ids, c(\"job.id\", \"submitted\"))[is.na(submitted), \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindStarted = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findStarted(reg, convertIds(reg, ids))\n}\n\n.findStarted = function(reg, ids = NULL, batch.ids = getBatchIds(reg, status = \"running\")) {\n started = batch.id = status = NULL\n bids = batch.ids[status == \"running\"]$batch.id\n filter(reg$status, ids, c(\"job.id\", \"started\", \"batch.id\"))[!is.na(started) | batch.id %in% bids, \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindNotStarted = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findNotStarted(reg, convertIds(reg, ids))\n}\n\n.findNotStarted = function(reg, ids = NULL, batch.ids = getBatchIds(reg, status = \"running\")) {\n started = batch.id = status = NULL\n bids = batch.ids[status == \"running\"]$batch.id\n filter(reg$status, ids, c(\"job.id\", \"started\", \"batch.id\"))[is.na(started) & ! batch.id %chin% bids, \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindDone = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findDone(reg, convertIds(reg, ids))\n}\n\n.findDone = function(reg, ids = NULL) {\n done = error = NULL\n filter(reg$status, ids, c(\"job.id\", \"done\", \"error\"))[!is.na(done) & is.na(error), \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindNotDone = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findNotDone(reg, convertIds(reg, ids))\n}\n\n.findNotDone = function(reg, ids = NULL) {\n done = error = NULL\n filter(reg$status, ids, c(\"job.id\", \"done\", \"error\"))[is.na(done) | !is.na(error), \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindErrors = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findErrors(reg, convertIds(reg, ids))\n}\n\n.findErrors = function(reg, ids = NULL) {\n error = NULL\n filter(reg$status, ids, c(\"job.id\", \"error\"))[!is.na(error), \"job.id\"]\n}\n\n\n# used in waitForJobs: find jobs which are done or error\n.findTerminated = function(reg, ids = NULL) {\n done = NULL\n filter(reg$status, ids, c(\"job.id\", \"done\"))[!is.na(done), \"job.id\"]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindOnSystem = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findOnSystem(reg, convertIds(reg, ids))\n}\n\n.findOnSystem = function(reg, ids = NULL, cols = \"job.id\", batch.ids = getBatchIds(reg, status = \"all\")) {\n if (length(batch.ids) == 0L)\n return(noIds())\n submitted = done = batch.id = NULL\n filter(reg$status, ids, c(\"job.id\", \"submitted\", \"done\", \"batch.id\"))[!is.na(submitted) & is.na(done) & batch.id %in% batch.ids$batch.id, cols, with = FALSE]\n}\n\n\n#' @export\n#' @rdname findJobs\nfindRunning = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findOnSystem(reg, convertIds(reg, ids), batch.ids = getBatchIds(reg, status = \"running\"))\n}\n\n#' @export\n#' @rdname findJobs\nfindQueued = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findOnSystem(reg, convertIds(reg, ids), batch.ids = getBatchIds(reg, status = \"queued\"))\n}\n\n#' @export\n#' @rdname findJobs\nfindExpired = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n .findExpired(reg, convertIds(reg, ids))\n}\n\n.findExpired = function(reg, ids = NULL, batch.ids = getBatchIds(reg)) {\n submitted = done = batch.id = NULL\n filter(reg$status, ids, c(\"job.id\", \"submitted\", \"done\", \"batch.id\"))[!is.na(submitted) & is.na(done) & batch.id %chnin% batch.ids$batch.id, \"job.id\"]\n}\n\n#' @export\n#' @rdname findJobs\n#' @param tags [\\code{character}]\\cr\n#' Return jobs which are tagged with any of the tags provided.\nfindTagged = function(tags = character(0L), ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n ids = convertIds(reg, ids, default = allIds(reg))\n assertCharacter(tags, any.missing = FALSE, pattern = \"^[[:alnum:]_.]+$\", min.len = 1L)\n tag = NULL\n\n ids[unique(reg$tags[tag %chin% tags, \"job.id\"], by = \"job.id\")]\n}\n" }, { "path": "R/getDefaultRegistry.R", "content": "#' @title Get and Set the Default Registry\n#' @description\n#' \\code{getDefaultRegistry} returns the registry currently set as default (or\n#' stops with an exception if none is set). \\code{setDefaultRegistry} sets\n#' a registry as default.\n#'\n#' @template reg\n#' @family Registry\n#' @export\ngetDefaultRegistry = function() {\n if (is.null(batchtools$default.registry))\n stop(\"No default registry defined\")\n batchtools$default.registry\n}\n\n#' @export\n#' @rdname getDefaultRegistry\nsetDefaultRegistry = function(reg) {\n if (!is.null(reg))\n assertRegistry(reg)\n batchtools$default.registry = reg\n}\n" }, { "path": "R/getErrorMessages.R", "content": "#' @title Retrieve Error Messages\n#'\n#' @description\n#' Extracts error messages from the internal data base and returns them in a table.\n#'\n#' @templateVar ids.default findErrors\n#' @template ids\n#' @param missing.as.error [\\code{logical(1)}]\\cr\n#' Treat missing results as errors? If \\code{TRUE}, the error message \\dQuote{[not terminated]} is imputed\n#' for jobs which have not terminated. Default is \\code{FALSE}\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id}, \\dQuote{terminated} (logical),\n#' \\dQuote{error} (logical) and \\dQuote{message} (string).\n#' @family debug\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' fun = function(i) if (i == 3) stop(i) else i\n#' ids = batchMap(fun, i = 1:5, reg = tmp)\n#' submitJobs(1:4, reg = tmp)\n#' waitForJobs(1:4, reg = tmp)\n#' getErrorMessages(ids, reg = tmp)\n#' getErrorMessages(ids, missing.as.error = TRUE, reg = tmp)\ngetErrorMessages = function(ids = NULL, missing.as.error = FALSE, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n assertFlag(missing.as.error)\n ids = convertIds(reg, ids, default = .findErrors(reg = reg))\n\n job.id = done = error = NULL\n tab = reg$status[ids, list(job.id, terminated = !is.na(done), error = !is.na(error), message = error)]\n\n if (missing.as.error)\n tab[!tab$terminated, c(\"error\", \"message\") := list(TRUE, \"[not terminated]\")]\n tab[]\n}\n" }, { "path": "R/getStatus.R", "content": "#' @title Summarize the Computational Status\n#'\n#' @description\n#' This function gives an encompassing overview over the computational status on your system.\n#' The status can be one or many of the following:\n#' \\itemize{\n#' \\item \\dQuote{defined}: Jobs which are defined via \\code{\\link{batchMap}} or \\code{\\link{addExperiments}}, but are not yet submitted.\n#' \\item \\dQuote{submitted}: Jobs which are submitted to the batch system via \\code{\\link{submitJobs}}, scheduled for execution.\n#' \\item \\dQuote{started}: Jobs which have been started.\n#' \\item \\dQuote{done}: Jobs which terminated successfully.\n#' \\item \\dQuote{error}: Jobs which terminated with an exception.\n#' \\item \\dQuote{running}: Jobs which are listed by the cluster functions to be running on the live system. Not supported for all cluster functions.\n#' \\item \\dQuote{queued}: Jobs which are listed by the cluster functions to be queued on the live system. Not supported for all cluster functions.\n#' \\item \\dQuote{system}: Jobs which are listed by the cluster functions to be queued or running. Not supported for all cluster functions.\n#' \\item \\dQuote{expired}: Jobs which have been submitted, but vanished from the live system. Note that this is determined heuristically and may include some false positives.\n#' }\n#' Here, a job which terminated successfully counts towards the jobs which are submitted, started and done.\n#' To retrieve the corresponding job ids, see \\code{\\link{findJobs}}.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] (with class \\dQuote{Status} for printing).\n#' @seealso \\code{\\link{findJobs}}\n#' @export\n#' @family debug\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' fun = function(i) if (i == 3) stop(i) else i\n#' ids = batchMap(fun, i = 1:5, reg = tmp)\n#' submitJobs(ids = 1:4, reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' tab = getStatus(reg = tmp)\n#' print(tab)\n#' str(tab)\ngetStatus = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n stats = getStatusTable(convertIds(reg, ids), reg = reg)\n setClasses(stats, c(\"Status\", class(stats)))\n}\n\ngetStatusTable = function(ids = NULL, batch.ids = getBatchIds(reg = reg), reg = getDefaultRegistry()) {\n submitted = started = done = error = status = NULL\n stats = merge(filter(reg$status, ids), batch.ids, by = \"batch.id\", all.x = TRUE, all.y = FALSE, sort = FALSE)[, list(\n defined = .N,\n submitted = count(submitted),\n started = sum(!is.na(started) | !is.na(status) & status == \"running\"),\n done = count(done),\n error = count(error),\n queued = sum(status == \"queued\", na.rm = TRUE),\n running = sum(status == \"running\", na.rm = TRUE),\n expired = sum(!is.na(submitted) & is.na(done) & is.na(status))\n )]\n stats$done = stats$done - stats$error\n stats$system = stats$queued + stats$running\n return(stats)\n}\n\n#' @export\nprint.Status = function(x, ...) {\n fmt = sprintf(\" %%-13s: %%%ii (%%5.1f%%%%)\", stri_width(x$defined))\n pr = function(label, h) catf(fmt, label, h, h / x$defined * 100)\n\n catf(\"Status for %i jobs at %s:\", x$defined, strftime(Sys.time()))\n pr(\"Submitted\", x$submitted)\n pr(\"-- Queued\", x$queued)\n pr(\"-- Started\", x$started)\n pr(\"---- Running\", x$running)\n pr(\"---- Done\", x$done)\n pr(\"---- Error\", x$error)\n pr(\"---- Expired\", x$expired)\n}\n" }, { "path": "R/helpers.R", "content": "mergedJobs = function(reg, ids, cols) {\n if (is.null(ids))\n reg$defs[reg$status, cols, on = \"def.id\", nomatch = 0L, with = missing(cols)]\n else\n reg$defs[reg$status[ids, nomatch = 0L, on = \"job.id\"], cols, on = \"def.id\", nomatch = 0L, with = missing(cols)]\n}\n\nauto_increment = function(ids, n = 1L) {\n if (length(ids) == 0L) seq_len(n) else max(ids) + seq_len(n)\n}\n\nustamp = function() {\n round(as.numeric(Sys.time()), 4L)\n}\n\nnames2 = function (x, missing.val = NA_character_) {\n n = names(x)\n if (is.null(n))\n return(rep.int(missing.val, length(x)))\n replace(n, is.na(n) | !nzchar(n), missing.val)\n}\n\ninsert = function(x, y) {\n x[names2(y)] = y\n x[order(names2(x))]\n}\n\nmakeProgressBar = function(...) {\n if (!batchtools$debug && getOption(\"batchtools.verbose\", TRUE) && getOption(\"batchtools.progress\", TRUE) && getOption(\"width\") >= 20L) {\n progress_bar$new(...)\n } else {\n list(tick = function(len = 1, tokens = list()) NULL, update = function(ratio, tokens) NULL)\n }\n}\n\nseq_row = function(x) {\n seq_len(nrow(x))\n}\n\nvlapply = function (x, fun, ..., use.names = TRUE) {\n vapply(X = x, FUN = fun, ..., FUN.VALUE = NA, USE.NAMES = use.names)\n}\n\nviapply = function (x, fun, ..., use.names = TRUE) {\n vapply(X = x, FUN = fun, ..., FUN.VALUE = NA_integer_, USE.NAMES = use.names)\n}\n\nvnapply = function (x, fun, ..., use.names = TRUE) {\n vapply(X = x, FUN = fun, ..., FUN.VALUE = NA_real_, USE.NAMES = use.names)\n}\n\nvcapply = function (x, fun, ..., use.names = TRUE) {\n vapply(X = x, FUN = fun, ..., FUN.VALUE = NA_character_, USE.NAMES = use.names)\n}\n\nis.error = function(x) {\n inherits(x, \"try-error\")\n}\n\n# formating info message\ninfo = function(...) {\n if (getOption(\"batchtools.verbose\", TRUE))\n message(sprintf(...))\n}\n\n# formating cat()\ncatf = function(..., con = \"\") {\n cat(stri_flatten(sprintf(...), \"\\n\"), \"\\n\", sep = \"\", file = con)\n}\n\n# formating message()\nmessagef = function(..., con = \"\") {\n message(sprintf(...))\n}\n\n# formating waring()\nwarningf = function(...) {\n warning(simpleWarning(sprintf(...), call = sys.call(sys.parent())))\n}\n\n# formating stop()\nstopf = function(...) {\n stop(simpleError(sprintf(...), call = NULL))\n}\n\n`%nin%` = function(x, y) {\n !match(x, y, nomatch = 0L)\n}\n\n`%chnin%` = function(x, y) {\n !chmatch(x, y, nomatch = 0L)\n}\n\nsetClasses = function(x, cl) {\n setattr(x, \"class\", cl)\n x\n}\n\n#' @useDynLib batchtools count_not_missing\ncount = function(x) {\n .Call(count_not_missing, x)\n}\n\nfilterNull = function(x) {\n x[!vlapply(x, is.null)]\n}\n\nstri_trunc = function(str, length, append = \"\") {\n if (is.na(str))\n return(str)\n if (stri_length(str) > length) {\n if (is.na(append) || !nzchar(append))\n return(stri_sub(str, 1L, length))\n return(stri_join(stri_sub(str, 1L, length - stri_length(append)), append))\n }\n return(str)\n}\n\nRscript = function() {\n fs::path(R.home(\"bin\"), ifelse(testOS(\"windows\"), \"Rscript.exe\", \"Rscript\"))\n}\n\ngetSeed = function(start.seed, id) {\n if (id > .Machine$integer.max - start.seed)\n start.seed - .Machine$integer.max + id\n else\n start.seed + id\n}\n\nchsetdiff = function(x, y) {\n # Note: assumes that x has no duplicates\n x[chmatch(x, y, 0L) == 0L]\n}\n\nchintersect = function(x, y) {\n # Note: assumes that x has no duplicates\n x[chmatch(y, x, 0L)]\n}\n\nrnd_hash = function(prefix = \"\") {\n stri_join(prefix, digest(list(runif(1L), as.numeric(Sys.time()))))\n}\n\nnow = function() {\n if (isTRUE(getOption(\"batchtools.timestamps\", FALSE)))\n sprintf(\" %s\", strftime(Sys.time()))\n else\n \"\"\n}\n\nexample_push_temp = function(i = 1L) {\n if (identical(Sys.getenv(\"IN_PKGDOWN\"), \"true\")) {\n base = fs::path(dirname(tempdir()), \"batchtools-example\")\n dirs = if (i == 1L) fs::path(base, \"reg\") else fs::path(base, sprintf(\"reg%i\", seq_len(i)))\n fs::dir_delete(dirs[fs::dir_exists(dirs)])\n fs::file_temp_push(dirs)\n }\n}\n" }, { "path": "R/ids.R", "content": "allIds = function(reg) {\n reg$status[, \"job.id\"]\n}\n\nnoIds = function() {\n data.table(job.id = integer(0L), key = \"job.id\")\n}\n\ncastIds = function(ids, setkey = TRUE) {\n if (is.data.table(ids)) {\n qassert(ids$job.id, \"X\", .var.name = \"column 'job.id'\")\n\n if (!is.integer(ids$job.id)) {\n \"!DEBUG [castIds]: Casting ids in data.table to integer\"\n ids = copy(ids)\n ids$job.id = as.integer(ids$job.id)\n }\n\n if (setkey && !identical(key(ids), \"job.id\")) {\n \"!DEBUG [castIds]: Setting missing key for ids table\"\n ids = copy(ids)\n setkeyv(ids, \"job.id\")\n }\n\n return(ids)\n }\n\n if (is.data.frame(ids)) {\n \"!DEBUG [castIds]: Casting ids from data.frame to data.table\"\n ids$job.id = asInteger(ids$job.id, .var.name = \"column 'job.id'\")\n ids = as.data.table(ids)\n if (setkey)\n setkeyv(ids, \"job.id\")\n return(ids)\n }\n\n if (qtest(ids, \"X\")) {\n \"!DEBUG [castIds]: Casting ids from vector to data.table\"\n return(data.table(job.id = as.integer(ids), key = if (setkey) \"job.id\" else NULL))\n }\n\n stop(\"Format of 'ids' not recognized. Must be a data.frame with column 'job.id' or an integerish vector\")\n}\n\nconvertIds = function(reg, ids, default = NULL, keep.extra = character(0L), keep.order = FALSE) {\n if (is.null(ids))\n return(default)\n\n ids = castIds(ids, setkey = !keep.order)\n if (anyDuplicated(ids, by = \"job.id\"))\n stop(\"Duplicated ids provided\")\n\n invalid = ids[!reg$status, on = \"job.id\", which = TRUE]\n if (length(invalid) > 0L) {\n info(\"Ignoring %i invalid job id%s\", length(invalid), if (length(ids) > 1L) \"s\" else \"\")\n ids = ids[-invalid]\n }\n\n cols = if (length(keep.extra)) union(\"job.id\", chintersect(keep.extra, names(ids))) else \"job.id\"\n ids[, cols, with = FALSE]\n}\n\nconvertId = function(reg, id) {\n id = convertIds(reg, id)\n if (nrow(id) != 1L)\n stopf(\"You must provide exactly one valid id (%i provided)\", nrow(id))\n return(id)\n}\n\nfilter = function(x, y, cols) {\n if (is.null(y)) {\n if (missing(cols))\n return(x)\n return(x[, cols, with = FALSE])\n }\n return(x[y, cols, on = key(x), nomatch = 0L, with = missing(cols)])\n}\n" }, { "path": "R/killJobs.R", "content": "#' @title Kill Jobs\n#'\n#' @description\n#' Kill jobs which are currently running on the batch system.\n#'\n#' In case of an error when killing, the function tries - after a short sleep - to kill the remaining\n#' batch jobs again. If this fails three times for some jobs, the function gives up. Jobs that could be\n#' successfully killed are reset in the \\link{Registry}.\n#'\n#' @templateVar ids.default findOnSystem\n#' @template ids\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id}, the corresponding \\dQuote{batch.id} and\n#' the logical flag \\dQuote{killed} indicating success.\n#' @family debug\n#' @export\nkillJobs = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE)\n\n kill = reg$cluster.functions$killJob\n if (is.null(kill))\n stop(\"ClusterFunctions implementation does not support the killing of jobs\")\n\n ids = convertIds(reg, ids, default = .findSubmitted(reg = reg))\n tab = reg$status[.findOnSystem(ids = ids, reg = reg), c(\"job.id\", \"started\", \"batch.id\")]\n\n if (nrow(tab) == 0L)\n return(data.table(job.id = integer(0L), batch.id = character(0L), killed = logical(0L)))\n\n runHook(reg, \"pre.kill\", tab)\n info(\"Trying to kill %i jobs ...\", nrow(tab))\n\n # kill queued jobs first, otherwise they might get started while killing running jobs\n setorderv(tab, \"started\", na.last = FALSE)\n tab[, \"killed\" := FALSE]\n\n batch.ids = unique(tab$batch.id)\n info(\"Killing %i real batch jobs ...\", length(batch.ids))\n\n for (i in seq_len(3L)) {\n tab[!tab$killed, \"killed\" := !is.error(try(kill(reg, .BY$batch.id), silent = TRUE)), by = \"batch.id\"]\n if (all(tab$killed))\n break\n Sys.sleep(2)\n }\n\n if (!all(tab$killed))\n warningf(\"Could not kill %i jobs\", sum(!tab$killed))\n\n # reset killed jobs\n merged = sync(reg = reg)\n cols = c(\"submitted\", \"started\", \"done\", \"error\", \"mem.used\", \"resource.id\", \"batch.id\", \"log.file\", \"job.hash\")\n reg$status[tab[tab$killed], (cols) := list(NA_real_, NA_real_, NA_real_, NA_character_, NA_real_, NA_integer_, NA_character_, NA_character_, NA_character_)]\n saveRegistry(reg)\n file_remove(merged)\n\n tab = setkeyv(tab[, c(\"job.id\", \"batch.id\", \"killed\")], \"job.id\")\n Sys.sleep(reg$cluster.functions$scheduler.latency)\n runHook(reg, \"post.kill\", tab)\n return(tab)\n}\n" }, { "path": "R/loadRegistry.R", "content": "#' @title Load a Registry from the File System\n#'\n#' @description\n#' Loads a registry from its \\code{file.dir}.\n#'\n#' Multiple R sessions accessing the same registry simultaneously can lead to database inconsistencies.\n#' This is especially dangerous if the same \\code{file.dir} is accessed from multiple machines, e.g. via a mount.\n#'\n#' If you just need to check on the status or peek into some preliminary results while another process is still submitting or waiting\n#' for pending results, you can load the registry in a read-only mode.\n#' All operations that need to change the registry will raise an exception in this mode.\n#' Files communicated back by the computational nodes are parsed to update the registry in memory while the registry on the file system remains unchanged.\n#'\n#' A heuristic tries to detect if the registry has been altered in the background by an other process and in this case automatically restricts the current registry to read-only mode.\n#' However, you should rely on this heuristic to work flawlessly.\n#' Thus, set to \\code{writeable} to \\code{TRUE} if and only if you are absolutely sure that other state-changing processes are terminated.\n#'\n#' If you need write access, load the registry with \\code{writeable} set to \\code{TRUE}.\n#'\n#' @param writeable [\\code{logical(1)}]\\cr\n#' Loads the registry in read-write mode. Default is \\code{FALSE}.\n#' @inheritParams makeRegistry\n#' @family Registry\n#' @return [\\code{\\link{Registry}}].\n#' @export\nloadRegistry = function(file.dir, work.dir = NULL, conf.file = findConfFile(), make.default = TRUE, writeable = FALSE) {\n assertString(file.dir)\n assertDirectory(file.dir)\n assertString(work.dir, null.ok = TRUE)\n assertString(conf.file, na.ok = TRUE)\n assertFlag(make.default)\n assertFlag(writeable)\n\n # read registry\n if (writeable)\n info(\"Reading registry in read-write mode\")\n else\n info(stri_paste(\n \"Reading registry in read-only mode.\",\n \"You can inspect results and errors, but cannot add, remove, submit or alter jobs in any way.\",\n \"If you need write-access, re-load the registry with `loadRegistry([...], writeable = TRUE)`.\"\n ))\n file.dir = fs::path_abs(file.dir)\n reg = readRegistry(file.dir)\n\n # re-allocate stuff which has not been serialized\n reg$file.dir = file.dir\n reg$writeable = writeable\n reg$mtime = file_mtime(fs::path(reg$file.dir, \"registry.rds\"))\n alloc.col(reg$status, ncol(reg$status))\n alloc.col(reg$defs, ncol(reg$defs))\n alloc.col(reg$resources, ncol(reg$resources))\n alloc.col(reg$tags, ncol(reg$tags))\n if (!is.null(work.dir))\n reg$work.dir = fs::path_abs(work.dir)\n updated = updateRegistry(reg = reg)\n\n # try to load dependencies relative to work.dir\n if (fs::dir_exists(reg$work.dir)) {\n with_dir(reg$work.dir, loadRegistryDependencies(reg))\n } else {\n warningf(\"The work.dir '%s' does not exist, jobs might fail to run on this system.\", reg$work.dir)\n loadRegistryDependencies(reg)\n }\n\n # source system config\n setSystemConf(reg, conf.file)\n if (make.default)\n batchtools$default.registry = reg\n\n merged = sync(reg = reg)\n if (length(merged) || updated) {\n saveRegistry(reg)\n file_remove(merged)\n }\n return(reg)\n}\n\nreadRegistry = function(file.dir) {\n fn.old = fs::path(file.dir, \"registry.rds\")\n fn.new = fs::path(file.dir, \"registry.new.rds\")\n\n if (fs::file_exists(fn.new)) {\n reg = try(readRDS(fn.new), silent = TRUE)\n if (!is.error(reg)) {\n fs::file_move(fn.new, fn.old)\n return(reg)\n } else {\n warning(\"Latest version of registry seems to be corrupted, trying backup ...\")\n }\n }\n\n if (fs::file_exists(fn.old)) {\n reg = try(readRDS(fn.old), silent = TRUE)\n if (!is.error(reg))\n return(reg)\n stop(\"Could not load the registry, files seem to be corrupt\")\n }\n\n stopf(\"No registry found in '%s'\", file.dir)\n}\n" }, { "path": "R/loadResult.R", "content": "#' @title Load the Result of a Single Job\n#'\n#' @description\n#' Loads the result of a single job.\n#'\n#' @template id\n#' @template reg\n#' @return [\\code{ANY}]. The stored result.\n#' @family Results\n#' @export\nloadResult = function(id, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n id = convertId(reg, id)\n if (nrow(.findDone(reg, id)) == 0L)\n stopf(\"Job with id %i not terminated\", id$job.id)\n fn = getResultFiles(reg, id)\n return(readRDS(fn))\n}\n" }, { "path": "R/mergeRegistries.R", "content": "# @title Merge the computational status of two registries\n#\n# @description\n# Merges the computational status of jobs found in the registries located at\n# \\code{file.dir} into the registry \\code{reg}.\n# Both registries must have the same jobs defined and may only differ w.r.t.\n# the computational status of the jobs.\n# This function is intended to be applied in the following context:\n# \\enumerate{\n# \\item Define all jobs locally (and ensure they work as intended by testing them).\n# \\item Copy the \\code{file.dir} to remote systems.\n# \\item Submit a subset of jobs on each system,\n# \\item After all jobs are terminated, copy both registries back to the local file system. Remember to keep backups.\n# \\item Load one registry with \\code{\\link{loadRegistry}}, merge the second with this function.\n# }\n#\n# @param file.dir [\\code{character(1)}]\\cr\n# Path to first registry.\n# @template reg\n# @return [\\code{\\link{Registry}}].\n# @export\n# @examples\n# \\dontshow{ batchtools:::example_push_temp(2) }\n# target = makeRegistry(NA, make.default = FALSE)\n# batchMap(identity, 1:10, reg = target)\n# td = tempdir()\n# file.copy(target$file.dir, td, recursive = TRUE)\n# file.dir = file.path(td, basename(target$file.dir))\n# source = loadRegistry(file.dir, update.paths = TRUE)\n#\n# submitJobs(1:5, reg = target)\n# submitJobs(6:10, reg = source)\n#\n# new = mergeRegistries(source, target)\nmergeRegistries = function(source, target = getDefaultRegistry()) {\n assertRegistry(source, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n assertRegistry(target, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n if (fs::path_real(source$file.dir) == fs::path_real(target$file.dir))\n stop(\"You must provide two different registries (using different file directories\")\n hash = function(x) unlist(.mapply(function(...) digest(list(...)), x[, !\"def.id\"], list()))\n\n # update only jobs which are not already computed and only those which are terminated\n status = source$status[.findNotDone(target), ][.findSubmitted(source)]\n\n # create a hash of parameters to match on\n status$hash = hash(sjoin(source$defs, status))\n\n # create temp table for target with the same hashes\n tmp = data.table(def.id = status$def.id, hash = hash(sjoin(target$defs, status)))\n\n # filter status to keep only jobs with matching ids and hashes\n # in status there are now only jobs which have an exact match in target$status\n # perform an updating join\n status = status[tmp, nomatch = 0L, on = c(\"def.id\", \"hash\")]\n info(\"Merging %i jobs ...\", nrow(status))\n\n src = getResultFiles(source, status)\n dst = fs::path(dir(target, \"results\"), fs::path_file(src))\n info(\"Copying %i result files ...\", length(src))\n fs::file_copy(src, dst, overwrite = TRUE)\n\n src = getLogFiles(source, status)\n dst = fs::path(dir(target, \"logs\"), fs::path_file(src))\n info(\"Copying %i log files ...\", length(src))\n fs::file_copy(src, dst, overwrite = TRUE)\n\n ext.dirs = as.integer(chintersect(list.files(dir(source, \"external\")), as.character(status$job.id)))\n if (length(ext.dirs) > 0L) {\n src = getExternalDirs(source, ext.dirs)\n dst = getExternalDirs(target, ext.dirs)\n info(\"Copying %i external directories ...\", length(ext.dirs))\n fs::dir_delete(dst[fs::dir_exists(dst)])\n fs::dir_copy(src, fs::path_dir(dst))\n }\n\n target$status = ujoin(target$status, status, by = \"job.id\")\n saveRegistry(reg = target)\n}\n" }, { "path": "R/reduceResults.R", "content": "#' @title Reduce Results\n#'\n#' @description\n#' A version of \\code{\\link[base]{Reduce}} for \\code{\\link{Registry}} objects\n#' which iterates over finished jobs and aggregates them.\n#' All jobs must have terminated, an error is raised otherwise.\n#'\n#' @note\n#' If you have thousands of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\n#' can significantly increase the performance.\n#'\n#' @templateVar ids.default findDone\n#' @template ids\n#' @param fun [\\code{function}]\\cr\n#' A function to reduce the results. The result of previous iterations (or\n#' the \\code{init}) will be passed as first argument, the result of of the\n#' i-th iteration as second. See \\code{\\link[base]{Reduce}} for some\n#' examples.\n#' If the function has the formal argument \\dQuote{job}, the \\code{\\link{Job}}/\\code{\\link{Experiment}}\n#' is also passed to the function (named).\n#' @param init [\\code{ANY}]\\cr\n#' Initial element, as used in \\code{\\link[base]{Reduce}}.\n#' If missing, the reduction uses the result of the first job as \\code{init} and the reduction starts\n#' with the second job.\n#' @param ... [\\code{ANY}]\\cr\n#' Additional arguments passed to function \\code{fun}.\n#' @return Aggregated results in the same order as provided ids.\n#' Return type depends on the user function. If \\code{ids}\n#' is empty, \\code{reduceResults} returns \\code{init} (if available) or \\code{NULL} otherwise.\n#' @template reg\n#' @family Results\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(function(a, b) list(sum = a+b, prod = a*b), a = 1:3, b = 1:3, reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # Extract element sum from each result\n#' reduceResults(function(aggr, res) c(aggr, res$sum), init = list(), reg = tmp)\n#'\n#' # Aggregate element sum via '+'\n#' reduceResults(function(aggr, res) aggr + res$sum, init = 0, reg = tmp)\n#'\n#' # Aggregate element prod via '*' where parameter b < 3\n#' reduce = function(aggr, res, job) {\n#' if (job$pars$b >= 3)\n#' return(aggr)\n#' aggr * res$prod\n#' }\n#' reduceResults(reduce, init = 1, reg = tmp)\n#'\n#' # Reduce to data.frame() (inefficient, use reduceResultsDataTable() instead)\n#' reduceResults(rbind, init = data.frame(), reg = tmp)\n#'\n#' # Reduce to data.frame by collecting results first, then utilize vectorization of rbind:\n#' res = reduceResultsList(fun = as.data.frame, reg = tmp)\n#' do.call(rbind, res)\n#'\n#' # Reduce with custom combine function:\n#' comb = function(x, y) list(sum = x$sum + y$sum, prod = x$prod * y$prod)\n#' reduceResults(comb, reg = tmp)\n#'\n#' # The same with neutral element NULL\n#' comb = function(x, y) if (is.null(x)) y else list(sum = x$sum + y$sum, prod = x$prod * y$prod)\n#' reduceResults(comb, init = NULL, reg = tmp)\n#'\n#' # Alternative: Reduce in list, reduce manually in a 2nd step\n#' res = reduceResultsList(reg = tmp)\n#' Reduce(comb, res)\nreduceResults = function(fun, ids = NULL, init, ..., reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n ids = convertIds(reg, ids, default = .findDone(reg = reg), keep.order = TRUE)\n fun = match.fun(fun)\n if (nrow(.findNotDone(reg, ids)))\n stop(\"All jobs must be have been successfully computed\")\n\n if (nrow(ids) == 0L)\n return(if (missing(init)) NULL else init)\n\n fns = getResultFiles(reg, ids)\n if (missing(init)) {\n init = readRDS(fns[1L])\n fns = fns[-1L]\n if (length(fns) == 0L)\n return(init)\n }\n\n pb = makeProgressBar(total = length(fns), format = \"Reduce [:bar] :percent eta: :eta\")\n if (\"job\" %chin% names(formals(fun))) {\n for (i in seq_along(fns)) {\n init = fun(init, readRDS(fns[i]), job = makeJob(ids[i], reg = reg), ...)\n pb$tick()\n }\n } else {\n for (i in seq_along(fns)) {\n init = fun(init, readRDS(fns[i]), ...)\n pb$tick()\n }\n }\n return(init)\n}\n\n\n#' @title Apply Functions on Results\n#'\n#' @description\n#' Applies a function on the results of your finished jobs and thereby collects\n#' them in a \\code{\\link[base]{list}} or \\code{\\link[data.table]{data.table}}.\n#' The later requires the provided function to return a list (or \\code{data.frame}) of scalar values.\n#' See \\code{\\link[data.table]{rbindlist}} for features and limitations of the aggregation.\n#'\n#' If not all jobs are terminated, the respective result will be \\code{NULL}.\n#'\n#' @note\n#' If you have thousands of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\n#' can significantly increase the performance.\n#'\n#' @templateVar ids.default findDone\n#' @template ids\n#' @param fun [\\code{function}]\\cr\n#' Function to apply to each result. The result is passed unnamed as first argument. If \\code{NULL}, the identity is used.\n#' If the function has the formal argument \\dQuote{job}, the \\code{\\link{Job}}/\\code{\\link{Experiment}} is also passed to the function.\n#' @param ... [\\code{ANY}]\\cr\n#' Additional arguments passed to to function \\code{fun}.\n#' @template missing.val\n#' @template reg\n#' @return \\code{reduceResultsList} returns a list of the results in the same order as the provided ids.\n#' \\code{reduceResultsDataTable} returns a \\code{\\link[data.table]{data.table}} with columns \\dQuote{job.id} and additional result columns\n#' created via \\code{\\link[data.table]{rbindlist}}, sorted by \\dQuote{job.id}.\n#' @seealso \\code{\\link{reduceResults}}\n#' @family Results\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(2) }\n#' ### Example 1 - reduceResultsList\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(function(x) x^2, x = 1:10, reg = tmp)\n#' submitJobs(reg = tmp)\n#' waitForJobs(reg = tmp)\n#' reduceResultsList(fun = sqrt, reg = tmp)\n#'\n#' ### Example 2 - reduceResultsDataTable\n#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # add first problem\n#' fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\n#' addProblem(\"rnorm\", fun = fun, reg = tmp)\n#'\n#' # add second problem\n#' fun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\n#' addProblem(\"rexp\", fun = fun, reg = tmp)\n#'\n#' # add first algorithm\n#' fun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\n#' addAlgorithm(\"average\", fun = fun, reg = tmp)\n#'\n#' # add second algorithm\n#' fun = function(instance, ...) sd(instance)\n#' addAlgorithm(\"deviation\", fun = fun, reg = tmp)\n#'\n#' # define problem and algorithm designs\n#' library(data.table)\n#' prob.designs = algo.designs = list()\n#' prob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\n#' prob.designs$rexp = data.table(n = 100, lambda = 1:5)\n#' algo.designs$average = data.table(method = c(\"mean\", \"median\"))\n#' algo.designs$deviation = data.table()\n#'\n#' # add experiments and submit\n#' addExperiments(prob.designs, algo.designs, reg = tmp)\n#' submitJobs(reg = tmp)\n#'\n#' # collect results and join them with problem and algorithm paramters\n#' res = ijoin(\n#' getJobPars(reg = tmp),\n#' reduceResultsDataTable(reg = tmp, fun = function(x) list(res = x))\n#' )\n#' unwrap(res, sep = \".\")\nreduceResultsList = function(ids = NULL, fun = NULL, ..., missing.val, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n assertFunction(fun, null.ok = TRUE)\n ids = convertIds(reg, ids, default = .findDone(reg = reg), keep.order = TRUE)\n .reduceResultsList(ids, fun, ..., missing.val = missing.val, reg = reg)\n}\n\n#' @export\n#' @rdname reduceResultsList\nreduceResultsDataTable = function(ids = NULL, fun = NULL, ..., missing.val, reg = getDefaultRegistry()) {\n assertRegistry(reg, sync = TRUE)\n ids = convertIds(reg, ids, default = .findDone(reg = reg))\n assertFunction(fun, null.ok = TRUE)\n\n results = .reduceResultsList(ids = ids, fun = fun, ..., missing.val = missing.val, reg = reg)\n if (length(results) == 0L)\n return(noIds())\n ids[, \"result\" := results][]\n}\n\n.reduceResultsList = function(ids, fun = NULL, ..., missing.val, reg = getDefaultRegistry()) {\n if (is.null(fun)) {\n worker = function(.res, .job, ...) .res\n } else {\n fun = match.fun(fun)\n if (\"job\" %chin% names(formals(fun)))\n worker = function(.res, .job, ...) fun(.res, job = .job, ...)\n else\n worker = function(.res, .job, ...) fun(.res, ...)\n }\n\n results = vector(\"list\", nrow(ids))\n done = ids[.findDone(reg, ids), nomatch = 0L, which = TRUE, on = \"job.id\"]\n\n if (missing(missing.val)) {\n if (length(done) != nrow(ids))\n stop(\"All jobs must be have been successfully computed\")\n } else {\n results[setdiff(seq_row(ids), done)] = list(missing.val)\n }\n\n if (length(done) > 0L) {\n fns = getResultFiles(reg, ids)\n pb = makeProgressBar(total = length(fns), format = \"Reducing [:bar] :percent eta: :eta\")\n reader = RDSReader$new(TRUE)\n\n for (i in done) {\n res = worker(readRDS(fns[i]), makeJob(ids$job.id[i], reader = reader, reg = reg), ...)\n if (!is.null(res))\n results[[i]] = res\n rm(res)\n pb$tick()\n }\n }\n return(results)\n}\n" }, { "path": "R/removeExperiments.R", "content": "#' @title Remove Experiments\n#'\n#' @description\n#' Remove Experiments from an \\code{\\link{ExperimentRegistry}}.\n#' This function automatically checks if any of the jobs to reset is either pending or running.\n#' However, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\n#' Use with care while jobs are running.\n#'\n#' @templateVar ids.default none\n#' @template ids\n#' @template expreg\n#' @return [\\code{\\link[data.table]{data.table}}] of removed job ids, invisibly.\n#' @export\n#' @family Experiment\nremoveExperiments = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\", writeable = TRUE, running.ok = FALSE)\n ids = convertIds(reg, ids, default = noIds())\n\n info(\"Removing %i Experiments ...\", nrow(ids))\n reg$status = reg$status[!ids]\n i = reg$defs[!reg$status, on = \"def.id\", which = TRUE]\n if (length(i) > 0L) {\n info(\"Removing %i job definitions ...\", length(i))\n reg$defs = reg$defs[-i]\n }\n fns = getResultFiles(reg, ids)\n file_remove(fns)\n\n sweepRegistry(reg)\n invisible(ids)\n}\n" }, { "path": "R/removeRegistry.R", "content": "#' @title Remove a Registry from the File System\n#'\n#' @description\n#' All files will be erased from the file system, including all results.\n#' If you wish to remove only intermediate files, use \\code{\\link{sweepRegistry}}.\n#'\n#' @param wait [\\code{numeric(1)}]\\cr\n#' Seconds to wait before proceeding. This is a safety measure to not\n#' accidentally remove your precious files. Set to 0 in\n#' non-interactive scripts to disable this precaution.\n#' @template reg\n#'\n#' @return [\\code{character(1)}]: Path of the deleted file directory.\n#' @export\n#' @family Registry\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' removeRegistry(0, tmp)\nremoveRegistry = function(wait = 5, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n assertNumber(wait, lower = 0)\n\n if (wait > 0) {\n info(\"This deletes all files in '%s'. Proceeding in %g seconds ...\", reg$file.dir, wait)\n Sys.sleep(wait)\n }\n\n if (identical(batchtools$default.registry$file.dir, reg$file.dir)) {\n info(\"Unsetting registry as default\")\n setDefaultRegistry(NULL)\n }\n\n info(\"Recursively removing files in '%s' ...\", reg$file.dir)\n fs::dir_delete(reg$file.dir)\n}\n" }, { "path": "R/resetJobs.R", "content": "#' @title Reset the Computational State of Jobs\n#'\n#' @description\n#' Resets the computational state of jobs in the \\code{\\link{Registry}}.\n#' This function automatically checks if any of the jobs to reset is either pending or running.\n#' However, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\n#' Use with care while jobs are running.\n#'\n#' @templateVar ids.default none\n#' @template ids\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] of job ids which have been reset.\n#' See \\code{\\link{JoinTables}} for examples on working with job tables.\n#' @family debug\n#' @export\nresetJobs = function(ids = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n ids = convertIds(reg, ids, default = noIds())\n\n info(\"Resetting %i jobs in DB ...\", nrow(ids))\n cols = c(\"submitted\", \"started\", \"done\", \"error\", \"mem.used\", \"resource.id\", \"batch.id\", \"log.file\", \"job.hash\")\n reg$status[ids, (cols) := list(NA_real_, NA_real_, NA_real_, NA_character_, NA_real_, NA_integer_, NA_character_, NA_character_, NA_character_), on = \"job.id\"]\n fns = getResultFiles(reg, ids)\n file_remove(fns)\n\n sweepRegistry(reg)\n invisible(ids)\n}\n" }, { "path": "R/runOSCommand.R", "content": "#' @title Run OS Commands on Local or Remote Machines\n#'\n#' @description\n#' This is a helper function to run arbitrary OS commands on local or remote machines.\n#' The interface is similar to \\code{\\link[base]{system2}}, but it always returns the exit status\n#' \\emph{and} the output.\n#'\n#' @param sys.cmd [\\code{character(1)}]\\cr\n#' Command to run.\n#' @param sys.args [\\code{character}]\\cr\n#' Arguments for \\code{sys.cmd}.\n#' @param stdin [\\code{character(1)}]\\cr\n#' Argument passed to \\code{\\link[base]{system2}}.\n#' @param nodename [\\code{character(1)}]\\cr\n#' Name of the SSH node to run the command on. If set to \\dQuote{localhost} (default), the command\n#' is not piped through SSH.\n#' @return [\\code{named list}] with \\dQuote{sys.cmd}, \\dQuote{sys.args}, \\dQuote{exit.code} (integer), \\dQuote{output} (character).\n#' @export\n#' @family ClusterFunctionsHelper\n#' @examples\n#' \\dontrun{\n#' runOSCommand(\"ls\")\n#' runOSCommand(\"ls\", \"-al\")\n#' runOSCommand(\"notfound\")\n#' }\nrunOSCommand = function(sys.cmd, sys.args = character(0L), stdin = \"\", nodename = \"localhost\") {\n assertCharacter(sys.cmd, any.missing = FALSE, len = 1L)\n assertCharacter(sys.args, any.missing = FALSE)\n assertString(nodename, min.chars = 1L)\n\n if (!isLocalHost(nodename)) {\n command = sprintf(\"%s %s\", sys.cmd, stri_flatten(sys.args, \" \"))\n if (getRversion() < \"4.0.0\") {\n command = shQuote(command)\n }\n command = stri_replace_all_fixed(command, \"\\\\$\", \"$\")\n sys.args = c(\"-q\", nodename, command)\n sys.cmd = \"ssh\"\n }\n\n \"!DEBUG [runOSCommand]: cmd: `sys.cmd` `stri_flatten(sys.args, ' ')`\"\n\n if (nzchar(Sys.which(sys.cmd))) {\n res = suppressWarnings(system2(command = sys.cmd, args = sys.args, stdin = stdin, stdout = TRUE, stderr = TRUE, wait = TRUE))\n output = as.character(res)\n exit.code = attr(res, \"status\") %??% 0L\n } else {\n output = \"command not found\"\n exit.code = 127L\n }\n\n \"!DEBUG [runOSCommand]: OS result (stdin '`stdin`', exit code `exit.code`):\"\n \"!DEBUG [runOSCommand]: `paste0(output, sep = '\\n')`\"\n\n return(list(sys.cmd = sys.cmd, sys.args = sys.args, exit.code = exit.code, output = output))\n}\n\nisLocalHost = function(nodename) {\n is.null(nodename) || nodename %chin% c(\"localhost\", \"127.0.0.1\", \"::1\")\n}\n\nOSError = function(msg, res) {\n cmd = stri_flatten(c(res$sys.cmd, res$sys.args), collapse = \" \") %??% NA_character_\n exit.code = res$exit.code %??% NA_integer_\n output = stri_flatten(res$output, \"\\n\") %??% \"\"\n\n stopf(\"%s (exit code %i);\\ncmd: '%s'\\noutput:\\n%s\", msg, exit.code, cmd, output)\n}\n" }, { "path": "R/saveRegistry.R", "content": "#' @title Store the Registy to the File System\n#'\n#' @description\n#' Stores the registry on the file system in its \\dQuote{file.dir} (specified\n#' for construction in \\code{\\link{makeRegistry}}, can be accessed via\n#' \\code{reg$file.dir}).\n#' This function is usually called internally whenever needed.\n#'\n#' @template reg\n#' @return [\\code{logical(1)}]: \\code{TRUE} if the registry was saved,\n#' \\code{FALSE} otherwise (if the registry is read-only).\n#' @family Registry\n#' @export\nsaveRegistry = function(reg = getDefaultRegistry()) {\n if (!reg$writeable) {\n \"!DEBUG [saveRegistry]: Skipping saveRegistry (read-only)\"\n return(FALSE)\n }\n\n \"!DEBUG [saveRegistry]: Saving Registry\"\n reg$hash = rnd_hash()\n fn = fs::path(reg$file.dir, c(\"registry.new.rds\", \"registry.rds\"))\n ee = new.env(parent = asNamespace(\"batchtools\"))\n exclude = c(\"cluster.functions\", \"default.resources\", \"temp.dir\", \"mtime\", \"writeable\")\n list2env(mget(chsetdiff(ls(reg), exclude), reg), ee)\n class(ee) = class(reg)\n writeRDS(ee, file = fn[1L], compress = reg$compress)\n fs::file_move(fn[1L], fn[2L])\n reg$mtime = file_mtime(fn[2L])\n return(TRUE)\n}\n" }, { "path": "R/sleep.R", "content": "getSleepFunction = function(reg, sleep) {\n if (is.null(sleep)) {\n if (is.null(reg$sleep))\n return(function(i) { Sys.sleep(5 + 115 * pexp(i - 1, rate = 0.01)) })\n sleep = reg$sleep\n }\n\n if (is.numeric(sleep)) {\n assertNumber(sleep, lower = 0)\n return(function(i) Sys.sleep(sleep))\n }\n\n if (is.function(sleep)) {\n return(function(i) Sys.sleep(sleep(i)))\n }\n\n stop(\"Argument 'sleep' must be either a numeric value or function(i)\")\n}\n" }, { "path": "R/submitJobs.R", "content": "#' @title Submit Jobs to the Batch Systems\n#'\n#' @description\n#' Submits defined jobs to the batch system.\n#'\n#' After submitting the jobs, you can use \\code{\\link{waitForJobs}} to wait for the\n#' termination of jobs or call \\code{\\link{reduceResultsList}}/\\code{\\link{reduceResults}}\n#' to collect partial results.\n#' The progress can be monitored with \\code{\\link{getStatus}}.\n#'\n#' @section Resources:\n#' You can pass arbitrary resources to \\code{submitJobs()} which then are available in the cluster function template.\n#' Some resources' names are standardized and it is good practice to stick to the following nomenclature to avoid confusion:\n#' \\describe{\n#' \\item{walltime:}{Upper time limit in seconds for jobs before they get killed by the scheduler. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n#' \\item{memory:}{Memory limit in Mb. If jobs exceed this limit, they are usually killed by the scheduler. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n#' \\item{ncpus:}{Number of (physical) CPUs to use on the slave. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n#' \\item{omp.threads:}{Number of threads to use via OpenMP. Used to set environment variable \\dQuote{OMP_NUM_THREADS}. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n#' \\item{pp.size:}{Maximum size of the pointer protection stack, see \\code{\\link[base]{Memory}}.}\n#' \\item{blas.threads:}{Number of threads to use for the BLAS backend. Used to set environment variables \\dQuote{MKL_NUM_THREADS} and \\dQuote{OPENBLAS_NUM_THREADS}. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n#' \\item{measure.memory:}{Enable memory measurement for jobs. Comes with a small runtime overhead.}\n#' \\item{chunks.as.arrayjobs:}{Execute chunks as array jobs.}\n#' \\item{pm.backend:}{Start a \\pkg{parallelMap} backend on the slave.}\n#' \\item{foreach.backend:}{Start a \\pkg{foreach} backend on the slave.}\n#' \\item{clusters:}{Resource used for Slurm to select the set of clusters to run \\code{sbatch}/\\code{squeue}/\\code{scancel} on.}\n#' }\n#'\n#' @section Chunking of Jobs:\n#' Multiple jobs can be grouped (chunked) together to be executed sequentially on the batch system as a single batch job.\n#' This is especially useful to avoid overburding the scheduler by submitting thousands of jobs simultaneously.\n#' To chunk jobs together, job ids must be provided as \\code{data.frame} with columns \\dQuote{job.id} and \\dQuote{chunk} (integer).\n#' All jobs with the same chunk number will be executed sequentially inside the same batch job.\n#' The utility functions \\code{\\link{chunk}}, \\code{\\link{binpack}} and \\code{\\link{lpt}}\n#' can assist in grouping jobs.\n#'\n#' @section Array Jobs:\n#' If your cluster supports array jobs, you can set the resource \\code{chunks.as.arrayjobs} to \\code{TRUE} in order\n#' to execute chunks as job arrays on the cluster.\n#' For each chunk of size \\code{n}, \\pkg{batchtools} creates a \\code{\\link{JobCollection}} of (possibly heterogeneous) jobs which is\n#' submitted to the scheduler as a single array job with \\code{n} repetitions.\n#' For each repetition, the \\code{JobCollection} is first read from the file system, then subsetted to the \\code{i}-th job using\n#' the environment variable \\code{reg$cluster.functions$array.var} (depending on the cluster backend, defined automatically) and finally\n#' executed.\n#'\n#' @section Order of Submission:\n#' Jobs are submitted in the order of chunks, i.e. jobs which have chunk number\n#' \\code{sort(unique(ids$chunk))[1]} first, then jobs with chunk number \\code{sort(unique(ids$chunk))[2]}\n#' and so on. If no chunks are provided, jobs are submitted in the order of \\code{ids$job.id}.\n#'\n#' @section Limiting the Number of Jobs:\n#' If requested, \\code{submitJobs} tries to limit the number of concurrent jobs of the user by waiting until jobs terminate\n#' before submitting new ones.\n#' This can be controlled by setting \\dQuote{max.concurrent.jobs} in the configuration file (see \\code{\\link{Registry}})\n#' or by setting the resource \\dQuote{max.concurrent.jobs} to the maximum number of jobs to run simultaneously.\n#' If both are set, the setting via the resource takes precedence over the setting in the configuration.\n#'\n#' @section Measuring Memory:\n#' Setting the resource \\code{measure.memory} to \\code{TRUE} turns on memory measurement:\n#' \\code{\\link[base]{gc}} is called directly before and after the job and the difference is\n#' stored in the internal database. Note that this is just a rough estimate and does\n#' neither work reliably for external code like C/C++ nor in combination with threading.\n#'\n#' @section Inner Parallelization:\n#' Inner parallelization is typically done via threading, sockets or MPI.\n#' Two backends are supported to assist in setting up inner parallelization.\n#'\n#' The first package is \\pkg{parallelMap}.\n#' If you set the resource \\dQuote{pm.backend} to \\dQuote{multicore}, \\dQuote{socket} or \\dQuote{mpi},\n#' \\code{\\link[parallelMap]{parallelStart}} is called on the slave before the first job in the chunk is started\n#' and \\code{\\link[parallelMap]{parallelStop}} is called after the last job terminated.\n#' This way, the resources for inner parallelization can be set and get automatically stored just like other computational resources.\n#' The function provided by the user just has to call \\code{\\link[parallelMap]{parallelMap}} to start parallelization using the preconfigured backend.\n#'\n#' To control the number of CPUs, you have to set the resource \\code{ncpus}.\n#' Otherwise \\code{ncpus} defaults to the number of available CPUs (as reported by (see \\code{\\link[parallel]{detectCores}}))\n#' on the executing machine for multicore and socket mode and defaults to the return value of \\code{\\link[Rmpi]{mpi.universe.size}-1} for MPI.\n#' Your template must be set up to handle the parallelization, e.g. request the right number of CPUs or start R with \\code{mpirun}.\n#' You may pass further options like \\code{level} to \\code{\\link[parallelMap]{parallelStart}} via the named list \\dQuote{pm.opts}.\n#'\n#' The second supported parallelization backend is \\pkg{foreach}.\n#' If you set the resource \\dQuote{foreach.backend} to \\dQuote{seq} (sequential mode), \\dQuote{parallel} (\\pkg{doParallel}) or\n#' \\dQuote{mpi} (\\pkg{doMPI}), the requested \\pkg{foreach} backend is automatically registered on the slave.\n#' Again, the resource \\code{ncpus} is used to determine the number of CPUs.\n#'\n#' Neither the namespace of \\pkg{parallelMap} nor the namespace \\pkg{foreach} are attached.\n#' You have to do this manually via \\code{\\link[base]{library}} or let the registry load the packages for you.\n#'\n#' @note\n#' If you a large number of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\n#' can significantly increase the performance of \\code{submitJobs}.\n#'\n#' @templateVar ids.default findNotSubmitted\n#' @template ids\n#' @param resources [\\code{named list}]\\cr\n#' Computational resources for the jobs to submit. The actual elements of this list\n#' (e.g. something like \\dQuote{walltime} or \\dQuote{nodes}) depend on your template file, exceptions are outlined in the section 'Resources'.\n#' Default settings for a system can be set in the configuration file by defining the named list \\code{default.resources}.\n#' Note that these settings are merged by name, e.g. merging \\code{list(walltime = 300)} into \\code{list(walltime = 400, memory = 512)}\n#' will result in \\code{list(walltime = 300, memory = 512)}.\n#' Same holds for individual job resources passed as additional column of \\code{ids} (c.f. section 'Resources').\n#' @param sleep [\\code{function(i)} | \\code{numeric(1)}]\\cr\n#' Parameter to control the duration to sleep between temporary errors.\n#' You can pass an absolute numeric value in seconds or a \\code{function(i)} which returns the number of seconds to sleep in the \\code{i}-th\n#' iteration between temporary errors.\n#' If not provided (\\code{NULL}), tries to read the value (number/function) from the configuration file (stored in \\code{reg$sleep}) or defaults to\n#' a function with exponential backoff between 5 and 120 seconds.\n#' @template reg\n#' @return [\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id} and \\dQuote{chunk}.\n#' @export\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(3) }\n#' ### Example 1: Submit subsets of jobs\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # toy function which fails if x is even and an input file does not exists\n#' fun = function(x, fn) if (x %% 2 == 0 && !file.exists(fn)) stop(\"file not found\") else x\n#'\n#' # define jobs via batchMap\n#' fn = tempfile()\n#' ids = batchMap(fun, 1:20, reg = tmp, fn = fn)\n#'\n#' # submit some jobs\n#' ids = 1:10\n#' submitJobs(ids, reg = tmp)\n#' waitForJobs(ids, reg = tmp)\n#' getStatus(reg = tmp)\n#'\n#' # create the required file and re-submit failed jobs\n#' file.create(fn)\n#' submitJobs(findErrors(ids, reg = tmp), reg = tmp)\n#' getStatus(reg = tmp)\n#'\n#' # submit remaining jobs which have not yet been submitted\n#' ids = findNotSubmitted(reg = tmp)\n#' submitJobs(ids, reg = tmp)\n#' getStatus(reg = tmp)\n#'\n#' # collect results\n#' reduceResultsList(reg = tmp)\n#'\n#' ### Example 2: Using memory measurement\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # Toy function which creates a large matrix and returns the column sums\n#' fun = function(n, p) colMeans(matrix(runif(n*p), n, p))\n#'\n#' # Arguments to fun:\n#' args = data.table::CJ(n = c(1e4, 1e5), p = c(10, 50)) # like expand.grid()\n#' print(args)\n#'\n#' # Map function to create jobs\n#' ids = batchMap(fun, args = args, reg = tmp)\n#'\n#' # Set resources: enable memory measurement\n#' res = list(measure.memory = TRUE)\n#'\n#' # Submit jobs using the currently configured cluster functions\n#' submitJobs(ids, resources = res, reg = tmp)\n#'\n#' # Retrive information about memory, combine with parameters\n#' info = ijoin(getJobStatus(reg = tmp)[, .(job.id, mem.used)], getJobPars(reg = tmp))\n#' print(unwrap(info))\n#'\n#' # Combine job info with results -> each job is aggregated using mean()\n#' unwrap(ijoin(info, reduceResultsDataTable(fun = function(res) list(res = mean(res)), reg = tmp)))\n#'\n#' ### Example 3: Multicore execution on the slave\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#'\n#' # Function which sleeps 10 seconds, i-times\n#' f = function(i) {\n#' parallelMap::parallelMap(Sys.sleep, rep(10, i))\n#' }\n#'\n#' # Create one job with parameter i=4\n#' ids = batchMap(f, i = 4, reg = tmp)\n#'\n#' # Set resources: Use parallelMap in multicore mode with 4 CPUs\n#' # batchtools internally loads the namespace of parallelMap and then\n#' # calls parallelStart() before the job and parallelStop() right\n#' # after the job last job in the chunk terminated.\n#' res = list(pm.backend = \"multicore\", ncpus = 4)\n#'\n#' \\dontrun{\n#' # Submit both jobs and wait for them\n#' submitJobs(resources = res, reg = tmp)\n#' waitForJobs(reg = tmp)\n#'\n#' # If successfull, the running time should be ~10s\n#' getJobTable(reg = tmp)[, .(job.id, time.running)]\n#'\n#' # There should also be a note in the log:\n#' grepLogs(pattern = \"parallelMap\", reg = tmp)\n#' }\nsubmitJobs = function(ids = NULL, resources = list(), sleep = NULL, reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE)\n assertList(resources, names = \"strict\")\n resources = insert(reg$default.resources, resources)\n if (hasName(resources, \"pm.backend\"))\n assertChoice(resources$pm.backend, c(\"local\", \"multicore\", \"socket\", \"mpi\"))\n if (hasName(resources, \"foreach.backend\"))\n assertChoice(resources$foreach.backend, c(\"seq\", \"parallel\", \"mpi\"))\n if (hasName(resources, \"pm.opts\"))\n assertList(resources$pm.opts, names = \"unique\")\n if (hasName(resources, \"ncpus\"))\n assertCount(resources$ncpus, positive = TRUE)\n if (hasName(resources, \"measure.memory\"))\n assertFlag(resources$measure.memory)\n sleep = getSleepFunction(reg, sleep)\n\n ids = convertIds(reg, ids, default = .findNotSubmitted(reg = reg), keep.extra = c(\"chunk\", batchtools$resources$per.job))\n if (nrow(ids) == 0L)\n return(noIds())\n\n # handle chunks\n use.chunking = hasName(ids, \"chunk\") && anyDuplicated(ids, by = \"chunk\") > 0L\n if (use.chunking) {\n ids$chunk = asInteger(ids$chunk, any.missing = FALSE)\n chunks = sort(unique(ids$chunk))\n } else {\n chunks = ids$chunk = seq_row(ids)\n }\n\n # check for jobs already on system\n on.sys = .findOnSystem(reg = reg, cols = c(\"job.id\", \"batch.id\"))\n ids.on.sys = on.sys[ids, nomatch = 0L, on = \"job.id\"]\n if (nrow(ids.on.sys) > 0L)\n stopf(\"Some jobs are already on the system, e.g. %i\", ids.on.sys[1L, ]$job.id)\n\n # handle max.concurrent.jobs\n max.concurrent.jobs = assertCount(resources$max.concurrent.jobs, null.ok = TRUE) %??%\n assertCount(reg$max.concurrent.jobs, null.ok = TRUE) %??% NA_integer_\n\n # handle chunks.as.arrayjobs\n chunks.as.arrayjobs = FALSE\n if (hasName(resources, \"chunks.as.arrayjobs\")) {\n assertFlag(resources$chunks.as.arrayjobs)\n if (resources$chunks.as.arrayjobs) {\n if (is.na(reg$cluster.functions$array.var)) {\n info(\"Ignoring resource 'chunks.as.arrayjobs', not supported by cluster functions '%s'\", reg$cluster.functions$name)\n } else {\n chunks.as.arrayjobs = TRUE\n }\n }\n }\n\n if (!is.na(max.concurrent.jobs)) {\n if (uniqueN(on.sys, by = \"batch.id\") + (!chunks.as.arrayjobs) * length(chunks) + chunks.as.arrayjobs * nrow(ids) > max.concurrent.jobs) {\n \"!DEBUG [submitJobs]: Limiting the number of concurrent jobs to `max.concurrent.jobs`\"\n } else {\n max.concurrent.jobs = NA_integer_\n }\n }\n\n # handle job resources\n per.job.resources = chintersect(names(ids), batchtools$resources$per.job)\n if (length(per.job.resources) > 0L) {\n if (use.chunking)\n stopf(\"Combining per-job resources with chunking is not supported\")\n ids$resource.id = addResources(reg, .mapply(function(...) insert(resources, list(...)), ids[, per.job.resources, with = FALSE], MoreArgs = list()))\n ids[, (per.job.resources) := NULL]\n } else {\n ids$resource.id = addResources(reg, list(resources))\n }\n\n info(\"Submitting %i jobs in %i chunks using cluster functions '%s' ...\", nrow(ids), length(chunks), reg$cluster.functions$name)\n on.exit(saveRegistry(reg))\n\n chunk = NULL\n runHook(reg, \"pre.submit\")\n\n pb = makeProgressBar(total = length(chunks), format = \":status [:bar] :percent eta: :eta\")\n pb$tick(0, tokens = list(status = \"Submitting\"))\n\n for (ch in chunks) {\n ids.chunk = ids[chunk == ch, c(\"job.id\", \"resource.id\")]\n jc = makeJobCollection(ids.chunk, resources = reg$resources[ids.chunk, on = \"resource.id\"]$resources[[1L]], reg = reg)\n if (reg$cluster.functions$store.job.collection)\n writeRDS(jc, file = jc$uri, compress = jc$compress)\n\n # do we have to wait for jobs to get terminated before proceeding?\n if (!is.na(max.concurrent.jobs)) {\n # count chunks or job.id\n i = 1L\n repeat {\n n.on.sys = uniqueN(getBatchIds(reg), by = \"batch.id\")\n \"!DEBUG [submitJobs]: Detected `n.on.sys` batch jobs on system (`max.concurrent.jobs` allowed concurrently)\"\n\n if (n.on.sys < max.concurrent.jobs)\n break\n pb$tick(0, tokens = list(status = \"Waiting \"))\n sleep(i)\n i = i + 1L\n }\n }\n\n # remove old result files\n fns = getResultFiles(reg, ids.chunk)\n file_remove(fns)\n\n i = 1L\n repeat {\n runHook(reg, \"pre.submit.job\")\n now = ustamp()\n submit = reg$cluster.functions$submitJob(reg = reg, jc = jc)\n\n if (submit$status == 0L) {\n if (!testCharacter(submit$batch.id, any.missing = FALSE, min.len = 1L)) {\n stopf(\"Cluster function did not return valid batch ids:\\n%s\", stri_flatten(capture.output(str(submit$batch.id)), \"\\n\"))\n }\n reg$status[ids.chunk,\n c(\"submitted\", \"started\", \"done\", \"error\", \"mem.used\", \"resource.id\", \"batch.id\", \"log.file\", \"job.hash\") :=\n list(now, NA_real_, NA_real_, NA_character_, NA_real_, ids.chunk$resource.id, submit$batch.id, submit$log.file, jc$job.hash)]\n runHook(reg, \"post.submit.job\")\n break\n } else if (submit$status > 0L && submit$status < 100L) {\n # temp error\n pb$tick(0, tokens = list(status = submit$msg))\n sleep(i)\n i = i + 1L\n } else if (submit$status > 100L && submit$status <= 200L) {\n # fatal error\n stopf(\"Fatal error occurred: %i. %s\", submit$status, submit$msg)\n }\n }\n pb$tick(len = 1, tokens = list(status = \"Submitting\"))\n }\n\n Sys.sleep(reg$cluster.functions$scheduler.latency)\n runHook(reg, \"post.submit\")\n\n # return ids, registry is saved via on.exit()\n return(invisible(ids))\n}\n\naddResources = function(reg, resources) {\n ai = function(tab, col) { # auto increment by reference\n i = tab[is.na(get(col)), which = TRUE]\n if (length(i) > 0L) {\n ids = seq_along(i)\n if (length(i) < nrow(tab))\n ids = ids + max(tab[, max(col, na.rm = TRUE), with = FALSE][[1L]], na.rm = TRUE)\n tab[i, (col) := ids]\n setkeyv(tab, col)[]\n }\n }\n\n tab = data.table(resources = resources, resource.hash = vcapply(resources, digest))\n new.tab = unique(tab, by = \"resource.hash\")[!reg$resources, on = \"resource.hash\"]\n if (nrow(new.tab)) {\n reg$resources = rbindlist(list(reg$resources, new.tab), fill = TRUE, use.names = TRUE)\n ai(reg$resources, \"resource.id\")\n }\n reg$resources[tab, \"resource.id\", on = \"resource.hash\"][[1L]]\n}\n" }, { "path": "R/summarizeExperiments.R", "content": "#' @title Quick Summary over Experiments\n#'\n#' @description\n#' Returns a frequency table of defined experiments.\n#' See \\code{\\link{ExperimentRegistry}} for an example.\n#'\n#' @templateVar ids.default all\n#' @template ids\n#' @param by [\\code{character}]\\cr\n#' Split the resulting table by columns of \\code{\\link{getJobPars}}.\n#' @template expreg\n#' @return [\\code{\\link[data.table]{data.table}}] of frequencies.\n#' @export\n#' @family Experiment\nsummarizeExperiments = function(ids = NULL, by = c(\"problem\", \"algorithm\"), reg = getDefaultRegistry()) {\n assertRegistry(reg, class = \"ExperimentRegistry\")\n assertCharacter(by, any.missing = FALSE, min.chars = 1L, min.len = 1L, unique = TRUE)\n tab = getJobPars(ids = ids, reg = reg)\n if (!setequal(by, c(\"problem\", \"algorithm\")))\n tab = unwrap(tab)\n tab[, list(.count = .N), by = by]\n}\n" }, { "path": "R/sweepRegistry.R", "content": "#' @title Check Consistency and Remove Obsolete Information\n#'\n#' @description\n#' Canceled jobs and jobs submitted multiple times may leave stray files behind.\n#' This function checks the registry for consistency and removes obsolete files\n#' and redundant data base entries.\n#'\n#' @template reg\n#' @family Registry\n#' @export\nsweepRegistry = function(reg = getDefaultRegistry()) {\n assertRegistry(reg, writeable = TRUE, sync = TRUE, running.ok = FALSE)\n \"!DEBUG [sweepRegistry]: Running sweepRegistry\"\n\n submitted = reg$status[.findSubmitted(reg = reg), c(\"job.id\", \"job.hash\")]\n obsolete = chsetdiff(\n list.files(dir(reg, \"results\"), full.names = TRUE),\n getResultFiles(reg, submitted)\n )\n if (length(obsolete)) {\n info(\"Removing %i obsolete result files ...\", length(obsolete))\n fs::file_delete(obsolete)\n }\n\n obsolete = chsetdiff(\n list.files(dir(reg, \"logs\"), full.names = TRUE),\n getLogFiles(reg, submitted)\n )\n if (length(obsolete)) {\n info(\"Removing %i obsolete log files ...\", length(obsolete))\n fs::file_delete(obsolete)\n }\n\n obsolete = list.files(dir(reg, \"jobs\"), pattern = \"\\\\.rds\", full.names = TRUE)\n if (length(obsolete)) {\n info(\"Removing %i obsolete job collection files ...\", length(obsolete))\n fs::file_delete(obsolete)\n }\n\n obsolete = list.files(dir(reg, \"jobs\"), pattern = \"\\\\.job$\", full.names = TRUE)\n if (length(obsolete)) {\n info(\"Removing %i job description files ...\", length(obsolete))\n fs::file_delete(obsolete)\n }\n\n obsolete = chsetdiff(\n list.files(dir(reg, \"external\"), pattern = \"^[0-9]+$\", full.names = TRUE),\n getExternalDirs(reg, submitted)\n )\n if (length(obsolete)) {\n info(\"Removing %i external directories of unsubmitted jobs ...\", length(obsolete))\n fs::dir_delete(obsolete)\n }\n\n obsolete = reg$resources[!reg$status, on = \"resource.id\", which = TRUE]\n if (length(obsolete)) {\n info(\"Removing %i resource specifications ...\", length(obsolete))\n reg$resources = reg$resources[-obsolete]\n }\n\n obsolete = reg$tags[!reg$status, on = \"job.id\", which = TRUE]\n if (length(obsolete)) {\n info(\"Removing %i tags ...\", length(obsolete))\n reg$tags = reg$tags[-obsolete]\n }\n\n saveRegistry(reg)\n}\n" }, { "path": "R/syncRegistry.R", "content": "#' @title Synchronize the Registry\n#'\n#' @description\n#' Parses update files written by the slaves to the file system and updates the\n#' internal data base.\n#'\n#' @template reg\n#' @return [\\code{logical(1)}]: \\code{TRUE} if the state has changed, \\code{FALSE} otherwise.\n#' @family Registry\n#' @export\nsyncRegistry = function(reg = getDefaultRegistry()) {\n assertRegistry(reg)\n merged = sync(reg)\n if (length(merged)) {\n saveRegistry(reg)\n file_remove(merged)\n }\n length(merged) > 0L\n}\n\n\nsync = function(reg) {\n \"!DEBUG [syncRegistry]: Triggered syncRegistry\"\n fns = list.files(dir(reg, \"updates\"), full.names = TRUE)\n if (length(fns) == 0L)\n return(character())\n\n runHook(reg, \"pre.sync\", fns = fns)\n\n updates = lapply(fns, function(fn) {\n x = try(readRDS(fn), silent = TRUE)\n if (is.error(x)) {\n if (reg$writeable && difftime(Sys.time(), fs::file_info(fn)$modification_time, units = \"mins\") > 60) {\n info(\"Removing unreadable update file '%s'\", fn)\n file_remove(fn)\n } else {\n info(\"Skipping unreadable update file '%s'\", fn)\n }\n return(NULL)\n }\n return(x)\n })\n\n failed = vlapply(updates, is.null)\n updates = rbindlist(updates, fill = TRUE, use.names = TRUE) # -> fill = TRUE for #135\n\n if (nrow(updates) > 0L) {\n expr = quote(`:=`(started = i.started, done = i.done, error = i.error, mem.used = i.mem.used))\n reg$status[updates, eval(expr), on = \"job.id\"]\n }\n\n runHook(reg, \"post.sync\", updates = updates)\n if (reg$writeable) fns[!failed] else character()\n}\n" }, { "path": "R/testJob.R", "content": "#' @title Run Jobs Interactively\n#'\n#' @description\n#' Starts a single job on the local machine.\n#'\n#' @template id\n#' @param external [\\code{logical(1)}]\\cr\n#' Run the job in an external R session? If \\code{TRUE}, starts a fresh R\n#' session on the local machine to execute the with \\code{\\link{execJob}}.\n#' You will not be able to use debug tools like \\code{\\link[base]{traceback}}\n#' or \\code{\\link[base]{browser}}.\n#'\n#' If \\code{external} is set to \\code{FALSE} (default) on the other hand,\n#' \\code{testJob} will execute the job in the current R session and the usual\n#' debugging tools work. However, spotting missing variable declarations (as they\n#' are possibly resolved in the global environment) is impossible.\n#' Same holds for missing package dependency declarations.\n#'\n#' @template reg\n#' @return Returns the result of the job if successful.\n#' @export\n#' @family debug\n#' @examples\n#' \\dontshow{ batchtools:::example_push_temp(1) }\n#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)\n#' batchMap(function(x) if (x == 2) xxx else x, 1:2, reg = tmp)\n#' testJob(1, reg = tmp)\n#' \\dontrun{\n#' testJob(2, reg = tmp)\n#' }\ntestJob = function(id, external = FALSE, reg = getDefaultRegistry()) {\n assertRegistry(reg)\n assertFlag(external)\n id = convertId(reg, id)\n\n if (external) {\n td = fs::path_abs(fs::path_temp())\n fn.r = fs::path(td, \"batchtools-testJob.R\")\n fn.jc = fs::path(td, \"batchtools-testJob.jc\")\n fn.res = fs::path(td, \"batchtools-testJob.rds\")\n\n writeRDS(makeJobCollection(id, reg = reg), file = fn.jc, compress = reg$compress)\n brew(file = system.file(fs::path(\"templates\", \"testJob.tmpl\"), package = \"batchtools\", mustWork = TRUE),\n output = fn.r, envir = list2env(list(jc = fn.jc, result = fn.res)))\n file_remove(fn.res)\n\n res = runOSCommand(Rscript(), fn.r)\n\n writeLines(res$output)\n if (res$exit.code == 0L && file.exists(fn.res))\n return(readRDS(fn.res))\n stopf(\"testJob() failed for job with id=%i. To properly debug, re-run with external=FALSE\", id$job.id)\n } else {\n with_dir(reg$work.dir, {\n loadRegistryDependencies(reg, must.work = TRUE)\n execJob(job = makeJob(id, reg = reg))\n })\n }\n}\n" }, { "path": "R/unwrap.R", "content": "#' @title Unwrap Nested Data Frames\n#'\n#' @description\n#' Some functions (e.g., \\code{\\link{getJobPars}}, \\code{\\link{getJobResources}} or \\code{\\link{reduceResultsDataTable}}\n#' return a \\code{data.table} with columns of type \\code{list}.\n#' These columns can be unnested/unwrapped with this function.\n#' The contents of these columns will be transformed to a \\code{data.table} and \\code{\\link[base]{cbind}}-ed\n#' to the input data.frame \\code{x}, replacing the original nested column.\n#'\n#' @note\n#' There is a name clash with function \\code{flatten} in package \\pkg{purrr}.\n#' The function \\code{flatten} is discouraged to use for this reason in favor of \\code{unwrap}.\n#'\n#' @param x [\\code{\\link{data.frame}} | \\code{\\link[data.table]{data.table}}]\\cr\n#' Data frame to flatten.\n#' @param cols [\\code{character}]\\cr\n#' Columns to consider for this operation. If set to \\code{NULL} (default),\n#' will operate on all columns of type \\dQuote{list}.\n#' @param sep [\\code{character(1)}]\\cr\n#' If \\code{NULL} (default), the column names of the additional columns will re-use the names\n#' of the nested \\code{list}/\\code{data.frame}.\n#' This may lead to name clashes.\n#' If you provide \\code{sep}, the variable column name will be constructed as\n#' \\dQuote{[column name of x][sep][inner name]}.\n#' @return [\\code{\\link[data.table]{data.table}}].\n#' @export\n#' @examples\n#' x = data.table::data.table(\n#' id = 1:3,\n#' values = list(list(a = 1, b = 3), list(a = 2, b = 2), list(a = 3))\n#' )\n#' unwrap(x)\n#' unwrap(x, sep = \".\")\nunwrap = function(x, cols = NULL, sep = NULL) {\n assertDataFrame(x)\n if (!is.data.table(x))\n x = as.data.table(x)\n\n if (is.null(cols)) {\n cols = names(x)[vlapply(x, is.list)]\n } else {\n assertNames(cols, \"unique\", subset.of = names(x))\n qassertr(x[, cols, with = FALSE], \"l\")\n }\n assertString(sep, null.ok = TRUE)\n\n res = data.table(.row = seq_row(x), key = \".row\")\n extra.cols = chsetdiff(names(x), cols)\n if (length(extra.cols))\n res = cbind(res, x[, extra.cols, with = FALSE])\n\n for (col in cols) {\n xc = x[[col]]\n\n new.cols = lapply(xc, function(x) {\n if (!is.null(x)) {\n ii = !vlapply(x, qtest, c(\"l\", \"d\", \"v1\")) # FIXME: add parameter `which` to qtestr\n x[ii] = lapply(x[ii], list)\n na = which(is.na(names2(x)))\n if (length(na) > 0L)\n names(x)[na] = sprintf(\"%s.%i\", col, seq_along(na))\n }\n x\n })\n new.cols = rbindlist(new.cols, fill = TRUE, idcol = \".row\", use.names = TRUE)\n\n if (ncol(new.cols) > 1L) {\n if (nrow(new.cols) > nrow(x) || anyDuplicated(new.cols, by = \".row\") > 0L)\n stopf(\"Some rows are unsuitable for unnesting. Unwrapping row in column '%s' leads to multiple rows\", col)\n if (!is.null(sep)) {\n nn = setdiff(names(new.cols), \".row\")\n setnames(new.cols, nn, stri_paste(col, nn, sep = sep))\n }\n clash = chsetdiff(chintersect(names(res), names(new.cols)), \".row\")\n if (length(clash) > 0L)\n stopf(\"Name clash while unwrapping data.table: Duplicated column names: %s\", stri_flatten(clash, \", \"))\n res = merge(res, new.cols, all.x = TRUE, by = \".row\")\n }\n }\n\n res[, \".row\" := NULL]\n kx = key(x)\n if (!is.null(kx) && all(kx %chin% names(res)))\n setkeyv(res, kx)\n res[]\n}\n\n#' @rdname unwrap\n#' @export\nflatten = function(x, cols = NULL, sep = NULL) { #nocov start\n \"!DEBUG Call of soon-to-be deprecated function flatten. Use unwrap() instead!\"\n unwrap(x, cols, sep)\n} #nocov end\n" }, { "path": "R/updateRegisty.R", "content": "# returns TRUE if the state possibly changed\nupdateRegistry = function(reg = getDefaultRegistry()) { # nocov start\n \"!DEBUG [updateRegistry]: Running updateRegistry\"\n pv = packageVersion(\"batchtools\")\n if (identical(pv, reg$version))\n return(FALSE)\n\n if (is.null(reg$version) || reg$version < \"0.9.0\")\n stop(\"Your registry is too old.\")\n\n if (reg$version < \"0.9.1-9000\") {\n ### hotfix for timestamps\n if (is.integer(reg$status$submitted)) {\n info(\"Converting timestamps to numeric\")\n for (x in c(\"submitted\", \"started\", \"done\"))\n reg$status[[x]] = as.numeric(reg$status[[x]])\n }\n\n ### hotfix for log.file column\n if (\"log.file\" %chnin% names(reg$status)) {\n info(\"Adding column 'log.file'\")\n reg$status[, (\"log.file\") := rep(NA_character_, .N)]\n }\n }\n\n if (reg$version < \"0.9.1-9001\") {\n ### hotfix for base32 encoding of exports\n fns = list.files(fs::path(reg$file.dir, \"exports\"), pattern = \"\\\\.rds$\", all.files = TRUE, no.. = TRUE)\n if (length(fns)) {\n info(\"Renaming export files\")\n fs::file_move(\n fs::path(reg$file.dir, fns),\n fs::path(reg$file.dir, mangle(stri_sub(fns, to = -5L)))\n )\n }\n }\n\n if (reg$version < \"0.9.1-9002\" && inherits(reg, \"ExperimentRegistry\")) {\n info(\"Renaming problems and algorithm files\")\n getProblemIds = function(reg) levels(reg$defs$problem)\n getAlgorithmIds = function(reg) levels(reg$defs$algorithm)\n\n for (prob in getProblemIds(reg))\n fs::file_move(fs::path(reg$file.dir, \"problems\", sprintf(\"%s.rds\", digest(prob))), getProblemURI(reg, prob))\n for (algo in getAlgorithmIds(reg))\n fs::file_move(fs::path(reg$file.dir, \"algorithms\", sprintf(\"%s.rds\", digest(algo))), getAlgorithmURI(reg, algo))\n }\n\n if (reg$version < \"0.9.4-9001\") {\n if (\"job.name\" %chnin% names(reg$status)) {\n info(\"Adding column 'job.name'\")\n reg$status[, (\"job.name\") := rep(NA_character_, .N)]\n }\n }\n\n if (reg$version < \"0.9.6-9001\") {\n info(\"Updating registry internals\")\n if (!inherits(reg, \"ExperimentRegistry\")) {\n setnames(reg$defs, \"pars\", \"job.pars\")\n } else {\n alloc.col(reg$defs, ncol(reg$defs) + 1L)\n reg$problems = levels(reg$defs$problem)\n reg$algorithms = levels(reg$defs$algorithm)\n reg$defs$problem = as.character(reg$defs$problem)\n reg$defs$algorithm = as.character(reg$defs$algorithm)\n reg$defs$prob.pars = lapply(reg$defs$pars, `[[`, \"prob.pars\")\n reg$defs$algo.pars = lapply(reg$defs$pars, `[[`, \"algo.pars\")\n reg$defs$pars = NULL\n info(\"Recalculating job hashes\")\n reg$defs$pars.hash = calculateHash(reg$defs)\n }\n }\n\n if (reg$version < \"0.9.7-9001\") {\n if (inherits(reg, \"ExperimentRegistry\")) {\n info(\"Updating problems\")\n for (id in reg$problems) {\n uri = getProblemURI(reg, id)\n p = readRDS(uri)\n p$cache = FALSE\n saveRDS(p, file = uri, version = 2L)\n }\n }\n }\n\n if (reg$version < \"0.9.7-9002\") {\n if (hasName(reg$status, \"memory\")) {\n info(\"Renaming memory column in data base\")\n setnames(reg$status, \"memory\", \"mem.used\")\n }\n\n fns = list.files(dir(reg, \"updates\"), full.names = TRUE)\n if (length(fns) > 0L) {\n info(\"Renaming memory column in update files\")\n updates = lapply(fns, function(fn) {\n x = try(readRDS(fn), silent = TRUE)\n if (is.error(x)) {\n fs::file_delete(x)\n } else {\n if (hasName(x, \"memory\")) {\n setnames(x, \"memory\", \"mem.used\")\n saveRDS(x, file = fn, version = 2L)\n }\n }\n })\n }\n }\n\n if (is.null(reg$compress)) {\n reg$compress = \"gzip\"\n }\n\n reg$version = pv\n return(TRUE)\n} # nocov end\n" }, { "path": "R/waitForFiles.R", "content": "# use list.files() here as this seems to trick the nfs cache\n# see https://github.com/mlr-org/batchtools/issues/85\nwaitForFiles = function(path, fns, timeout = 0) {\n if (timeout == 0)\n return(TRUE)\n\n fns = fns[!fs::file_exists(fns)]\n if (length(fns) == 0L)\n return(TRUE)\n\n \"!DEBUG [waitForFiles]: `length(fns)` files not found via 'file.exists()'\"\n fns = chsetdiff(fns, list.files(path, all.files = TRUE))\n if (length(fns) == 0L)\n return(TRUE)\n\n timeout = timeout + Sys.time()\n repeat {\n Sys.sleep(0.5)\n fns = chsetdiff(fns, list.files(path, all.files = TRUE))\n if (length(fns) == 0L)\n return(TRUE)\n if (Sys.time() > timeout)\n stopf(\"Timeout while waiting for %i files, e.g. '%s'\", length(fns), fns[1L])\n }\n}\n\nwaitForFile = function(fn, timeout = 0, must.work = TRUE) {\n if (timeout == 0 || fs::file_exists(fn))\n return(TRUE)\n\n \"!DEBUG [waitForFile]: `fn` not found via 'file.exists()'\"\n timeout = timeout + Sys.time()\n path = fs::path_dir(fn)\n repeat {\n Sys.sleep(0.5)\n if (basename(fn) %chin% list.files(path, all.files = TRUE))\n return(TRUE)\n if (Sys.time() > timeout) {\n if (must.work)\n stopf(\"Timeout while waiting for file '%s'\", fn)\n return(FALSE)\n }\n }\n}\n\n" }, { "path": "R/waitForJobs.R", "content": "#' @title Wait for Termination of Jobs\n#'\n#' @description\n#' This function simply waits until all jobs are terminated.\n#'\n#' @templateVar ids.default findSubmitted\n#' @template ids\n#' @param sleep [\\code{function(i)} | \\code{numeric(1)}]\\cr\n#' Parameter to control the duration to sleep between queries.\n#' You can pass an absolute numeric value in seconds or a \\code{function(i)} which returns\n#' the number of seconds to sleep in the \\code{i}-th iteration.\n#' If not provided (\\code{NULL}), tries to read the value (number/function) from the configuration file\n#' (stored in \\code{reg$sleep}) or defaults to a function with exponential backoff between\n#' 5 and 120 seconds.\n#' @param timeout [\\code{numeric(1)}]\\cr\n#' After waiting \\code{timeout} seconds, show a message and return\n#' \\code{FALSE}. This argument may be required on some systems where, e.g.,\n#' expired jobs or jobs on hold are problematic to detect. If you don't want\n#' a timeout, set this to \\code{Inf}. Default is \\code{604800} (one week).\n#' @param expire.after [\\code{integer(1)}]\\cr\n#' Jobs count as \\dQuote{expired} if they are not found on the system but have not communicated back\n#' their results (or error message). This frequently happens on managed system if the scheduler kills\n#' a job because the job has hit the walltime or request more memory than reserved.\n#' On the other hand, network file systems often require several seconds for new files to be found,\n#' which can lead to false positives in the detection heuristic.\n#' \\code{waitForJobs} treats such jobs as expired after they have not been detected on the system\n#' for \\code{expire.after} iterations.\n#' If not provided (\\code{NULL}), tries to read the value from the configuration file (stored in \\code{reg$expire.after}),\n#' and finally defaults to \\code{3}.\n#' @param stop.on.error [\\code{logical(1)}]\\cr\n#' Immediately cancel if a job terminates with an error? Default is\n#' \\code{FALSE}.\n#' @param stop.on.expire [\\code{logical(1)}]\\cr\n#' Immediately cancel if jobs are detected to be expired? Default is \\code{FALSE}.\n#' Expired jobs will then be ignored for the remainder of \\code{waitForJobs()}.\n#' @template reg\n#' @return [\\code{logical(1)}]. Returns \\code{TRUE} if all jobs terminated\n#' successfully and \\code{FALSE} if either the timeout is reached or at least\n#' one job terminated with an exception or expired.\n#' @export\nwaitForJobs = function(ids = NULL, sleep = NULL, timeout = 604800, expire.after = NULL, stop.on.error = FALSE, stop.on.expire = FALSE, reg = getDefaultRegistry()) {\n waitForResults = function(ids) {\n waitForFiles(\n fs::path(reg$file.dir, \"results\"),\n sprintf(\"%i.rds\", .findDone(reg, ids)$job.id),\n reg$cluster.functions$fs.latency\n )\n }\n\n assertRegistry(reg, sync = TRUE)\n assertNumber(timeout, lower = 0)\n assertFlag(stop.on.error)\n assertFlag(stop.on.expire)\n expire.after = assertCount(expire.after, positive = TRUE, null.ok = TRUE) %??% reg$expire.after %??% 3L\n sleep = getSleepFunction(reg, sleep)\n ids = convertIds(reg, ids, default = .findSubmitted(reg = reg))\n\n if (nrow(.findNotSubmitted(ids = ids, reg = reg)) > 0L) {\n warning(\"Cannot wait for unsubmitted jobs. Removing from ids.\")\n ids = ids[.findSubmitted(ids = ids, reg = reg), nomatch = 0L]\n }\n\n if (nrow(ids) == 0L) {\n return(TRUE)\n }\n\n terminated = on.sys = expire.counter = NULL\n ids$terminated = FALSE\n ids$on.sys = FALSE\n ids$expire.counter = 0L\n\n timeout = Sys.time() + timeout\n pb = makeProgressBar(total = nrow(ids), format = \"Waiting (Q::queued R::running D::done E::error ?::expired) [:bar] :percent eta: :eta\")\n i = 0L\n\n repeat {\n ### case 1: all jobs terminated or expired -> nothing on system\n ids[.findTerminated(reg, ids), \"terminated\" := TRUE]\n if (ids[!(terminated) & expire.counter <= expire.after, .N] == 0L) {\n \"!DEBUG [waitForJobs]: All jobs terminated\"\n pb$update(1)\n waitForResults(ids)\n return(nrow(.findDone(reg, ids)) == nrow(ids))\n }\n\n ### case 2: there are errors and stop.on.error is TRUE\n if (stop.on.error && nrow(.findErrors(reg, ids)) > 0L) {\n \"!DEBUG [waitForJobs]: Errors found and stop.on.error is TRUE\"\n pb$update(1)\n return(FALSE)\n }\n\n batch.ids = getBatchIds(reg)\n ids[, \"on.sys\" := FALSE][.findOnSystem(reg, ids, batch.ids = batch.ids), \"on.sys\" := TRUE]\n ids[(on.sys), \"expire.counter\" := 0L]\n ids[!(on.sys) & !(terminated), \"expire.counter\" := expire.counter + 1L]\n stats = getStatusTable(ids = ids, batch.ids = batch.ids, reg = reg)\n pb$update(mean(ids$terminated), tokens = as.list(stats))\n \"!DEBUG [waitForJobs]: batch.ids: `stri_flatten(batch.ids$batch.id, ',')`\"\n\n ### case 3: jobs disappeared, we cannot find them on the system after [expire.after] iterations\n if (stop.on.expire && ids[!(terminated) & expire.counter > expire.after, .N] > 0L) {\n warning(\"Jobs disappeared from the system\")\n pb$update(1)\n waitForResults(ids)\n return(FALSE)\n }\n\n # case 4: we reach a timeout\n sleep(i)\n i = i + 1L\n if (Sys.time() > timeout) {\n pb$update(1)\n warning(\"Timeout reached\")\n return(FALSE)\n }\n\n merged = suppressMessages(sync(reg = reg))\n if (length(merged)) {\n saveRegistry(reg)\n file_remove(merged)\n }\n }\n}\n" }, { "path": "R/zzz.R", "content": "#' @description\n#' For bug reports and feature requests please use the tracker:\n#' \\url{https://github.com/mlr-org/batchtools}.\n#'\n#' @section Package options:\n#' \\describe{\n#' \\item{\\code{batchtools.verbose}}{\n#' Verbosity. Set to \\code{FALSE} to suppress info messages and progress bars.\n#' }\n#' \\item{\\code{batchtools.progress}}{\n#' Progress bars. Set to \\code{FALSE} to disable them.\n#' }\n#' \\item{\\code{batchtools.timestamps}}{\n#' Add time stamps to log output. Set to \\code{FALSE} to disable them.\n#' }\n#' }\n#' Furthermore, you may enable a debug mode using the \\pkg{debugme} package by\n#' setting the environment variable \\dQuote{DEBUGME} to \\dQuote{batchtools} before\n#' loading \\pkg{batchtools}.\n#' @import utils\n#' @import checkmate\n#' @import stringi\n#' @import data.table\n#' @importFrom R6 R6Class\n#' @importFrom digest digest\n#' @importFrom brew brew\n#' @importFrom progress progress_bar\n#' @importFrom rappdirs user_config_dir site_config_dir\n#' @importFrom stats runif predict pexp\n#' @importFrom base64url base32_encode base32_decode\n#' @importFrom withr with_dir with_seed local_options local_dir\n\"_PACKAGE\"\n\nbatchtools = new.env(parent = emptyenv())\nbatchtools$debug = FALSE\nbatchtools$hooks = list(\n remote = c(\"pre.do.collection\", \"post.do.collection\"),\n local = c(\"pre.sync\", \"post.sync\", \"pre.submit.job\", \"post.submit.job\", \"pre.submit\", \"post.submit\", \"pre.kill\", \"post.kill\")\n)\n\nbatchtools$resources = list(\n per.job = c(\"walltime\", \"memory\", \"ncpus\", \"omp.threads\", \"blas.threads\"),\n per.chunk = c(\"measure.memory\", \"chunks.as.arrayjobs\", \"pm.backend\", \"foreach.backend\")\n)\n\n.onLoad = function(libname, pkgname) { # nocov start\n if (requireNamespace(\"debugme\", quietly = TRUE) && \"batchtools\" %in% strsplit(Sys.getenv(\"DEBUGME\"), \",\", fixed = TRUE)[[1L]]) {\n debugme::debugme()\n batchtools$debug = TRUE\n }\n backports::import(pkgname, \"...length\")\n backports::import(pkgname, \"hasName\", force = TRUE)\n} # nocov end\n\n.onUnload = function (libpath) { # nocov start\n library.dynam.unload(\"batchtools\", libpath)\n} # nocov end\n" }, { "path": "README.Rmd", "content": "---\noutput: github_document\n---\n\n# batchtools\n\nPackage website: [release](https://batchtools.mlr-org.com/) | [dev](https://batchtools.mlr-org.com/dev/)\n\n\n\n[![JOSS Publication](https://joss.theoj.org/papers/10.21105/joss.00135/status.svg)](https://doi.org/10.21105/joss.00135)\n[![r-cmd-check](https://github.com/mlr-org/batchtools/actions/workflows/r-cmd-check.yml/badge.svg)](https://github.com/mlr-org/batchtools/actions/workflows/r-cmd-check.yml)\n[![CRAN Status](https://www.r-pkg.org/badges/version-ago/batchtools)](https://cran.r-project.org/package=batchtools)\n[![Mattermost](https://img.shields.io/badge/chat-mattermost-orange.svg)](https://lmmisld-lmu-stats-slds.srv.mwn.de/mlr_invite/)\n\n\nAs a successor of the packages [BatchJobs](https://github.com/tudo-r/BatchJobs) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments), batchtools provides a parallel implementation of Map for high performance computing systems managed by schedulers like Slurm, Sun Grid Engine, OpenLava, TORQUE/OpenPBS, Load Sharing Facility (LSF) or Docker Swarm (see the setup section in the [vignette](https://batchtools.mlr-org.com/articles/batchtools.html)).\n\nMain features:\n\n* Convenience: All relevant batch system operations (submitting, listing, killing) are either handled internally or abstracted via simple R functions\n* Portability: With a well-defined interface, the source is independent from the underlying batch system - prototype locally, deploy on any high performance cluster\n* Reproducibility: Every computational part has an associated seed stored in a data base which ensures reproducibility even when the underlying batch system changes\n* Abstraction: The code layers for algorithms, experiment definitions and execution are cleanly separated and allow to write readable and maintainable code to manage large scale computer experiments\n\n\n## Installation\n\nInstall the stable version from CRAN:\n```{R, eval = FALSE}\ninstall.packages(\"batchtools\")\n```\nFor the development version, use [devtools](https://cran.r-project.org/package=devtools):\n\n```{R, eval = FALSE}\ndevtools::install_github(\"mlr-org/batchtools\")\n```\n\nNext, you need to setup `batchtools` for your HPC (it will run sequentially otherwise).\nSee the [vignette](https://batchtools.mlr-org.com/articles/batchtools.html) for instructions.\n\n## Why batchtools?\nThe development of [BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) is discontinued for the following reasons:\n\n* Maintainability: The packages [BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) are tightly connected which makes maintenance difficult. Changes have to be synchronized and tested against the current CRAN versions for compatibility. Furthermore, BatchExperiments violates CRAN policies by calling internal functions of BatchJobs.\n* Data base issues: Although we invested weeks to mitigate issues with locks of the SQLite data base or file system (staged queries, file system timeouts, ...), `BatchJobs` kept working unreliable on some systems with high latency under certain conditions. This made `BatchJobs` unusable for many users.\n\n[BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) will remain on CRAN, but new features are unlikely to be ported back.\nThe [vignette](https://batchtools.mlr-org.com/articles/batchtools.html) contains a section comparing the packages.\n\n\n## Resources\n* [Function reference](https://batchtools.mlr-org.com/reference/)\n* [Vignette](https://batchtools.mlr-org.com/articles/batchtools.html)\n* [JOSS Paper](https://doi.org/10.21105/joss.00135): Short paper on batchtools. Please cite this if you use batchtools.\n* [Paper on BatchJobs/BatchExperiments](https://www.jstatsoft.org/v64/i11): The described concept still holds for batchtools and most examples work analogously (see the [vignette](https://batchtools.mlr-org.com/articles/batchtools.html) for differences between the packages).\n\n## Citation\nPlease cite the [JOSS paper](https://doi.org/10.21105/joss.00135) using the following BibTeX entry:\n```\n@article{,\n doi = {10.21105/joss.00135},\n url = {https://doi.org/10.21105/joss.00135},\n year = {2017},\n month = {feb},\n publisher = {The Open Journal},\n volume = {2},\n number = {10},\n author = {Michel Lang and Bernd Bischl and Dirk Surmann},\n title = {batchtools: Tools for R to work on batch systems},\n journal = {The Journal of Open Source Software}\n}\n```\n\n## Related Software\n* The [High Performance Computing Task View](https://cran.r-project.org/view=HighPerformanceComputing) lists the most relevant packages for scientific computing with R.\n* [clustermq](https://cran.r-project.org/package=clustermq) is a similar approach which also supports multiple schedulers. Uses the ZeroMQ network protocol for communication, and shines if you have millions of fast jobs.\n* [batch](https://cran.r-project.org/package=batch) assists in splitting and submitting jobs to LSF and MOSIX clusters.\n* [flowr](https://cran.r-project.org/package=flowr) supports LSF, Slurm, TORQUE and Moab and provides a scatter-gather approach to define computational jobs.\n* [future.batchtools](https://cran.r-project.org/package=future.batchtools) implements `batchtools` as backend for [future](https://cran.r-project.org/package=future.batchtools).\n* [doFuture](https://cran.r-project.org/package=doFuture) together with [future.batchtools](https://cran.r-project.org/package=future.batchtools) connects `batchtools` to [foreach](https://cran.r-project.org/package=foreach).\n* [drake](https://cran.r-project.org/package=drake) uses graphs to define computational jobs. `batchtools` is used as a backend via [future.batchtools](https://cran.r-project.org/package=future.batchtools).\n\n## Contributing to batchtools\nThis R package is licensed under the [LGPL-3](https://www.gnu.org/licenses/lgpl-3.0.en.html).\nIf you encounter problems using this software (lack of documentation, misleading or wrong documentation, unexpected behaviour, bugs, ...) or just want to suggest features, please open an issue in the [issue tracker](https://github.com/mlr-org/batchtools/issues).\nPull requests are welcome and will be included at the discretion of the author.\nIf you have customized a template file for your (larger) computing site, please share it: fork the repository, place your template in `inst/templates` and send a pull request.\n" }, { "path": "README.md", "content": "\n# batchtools\n\nPackage website: [release](https://batchtools.mlr-org.com/) \\|\n[dev](https://batchtools.mlr-org.com/dev/)\n\n\n\n[![JOSS\nPublication](https://joss.theoj.org/papers/10.21105/joss.00135/status.svg)](https://doi.org/10.21105/joss.00135)\n[![r-cmd-check](https://github.com/mlr-org/batchtools/actions/workflows/r-cmd-check.yml/badge.svg)](https://github.com/mlr-org/batchtools/actions/workflows/r-cmd-check.yml)\n[![CRAN\nStatus](https://www.r-pkg.org/badges/version-ago/batchtools)](https://cran.r-project.org/package=batchtools)\n[![Mattermost](https://img.shields.io/badge/chat-mattermost-orange.svg)](https://lmmisld-lmu-stats-slds.srv.mwn.de/mlr_invite/)\n\n\nAs a successor of the packages\n[BatchJobs](https://github.com/tudo-r/BatchJobs) and\n[BatchExperiments](https://github.com/tudo-r/Batchexperiments),\nbatchtools provides a parallel implementation of Map for high\nperformance computing systems managed by schedulers like Slurm, Sun Grid\nEngine, OpenLava, TORQUE/OpenPBS, Load Sharing Facility (LSF) or Docker\nSwarm (see the setup section in the\n[vignette](https://batchtools.mlr-org.com/articles/batchtools.html)).\n\nMain features:\n\n- Convenience: All relevant batch system operations (submitting,\n listing, killing) are either handled internally or abstracted via\n simple R functions\n- Portability: With a well-defined interface, the source is independent\n from the underlying batch system - prototype locally, deploy on any\n high performance cluster\n- Reproducibility: Every computational part has an associated seed\n stored in a data base which ensures reproducibility even when the\n underlying batch system changes\n- Abstraction: The code layers for algorithms, experiment definitions\n and execution are cleanly separated and allow to write readable and\n maintainable code to manage large scale computer experiments\n\n## Installation\n\nInstall the stable version from CRAN:\n\n``` r\ninstall.packages(\"batchtools\")\n```\n\nFor the development version, use\n[devtools](https://cran.r-project.org/package=devtools):\n\n``` r\ndevtools::install_github(\"mlr-org/batchtools\")\n```\n\nNext, you need to setup `batchtools` for your HPC (it will run\nsequentially otherwise). See the\n[vignette](https://batchtools.mlr-org.com/articles/batchtools.html) for\ninstructions.\n\n## Why batchtools?\n\nThe development of [BatchJobs](https://github.com/tudo-r/BatchJobs/) and\n[BatchExperiments](https://github.com/tudo-r/Batchexperiments) is\ndiscontinued for the following reasons:\n\n- Maintainability: The packages\n [BatchJobs](https://github.com/tudo-r/BatchJobs/) and\n [BatchExperiments](https://github.com/tudo-r/Batchexperiments) are\n tightly connected which makes maintenance difficult. Changes have to\n be synchronized and tested against the current CRAN versions for\n compatibility. Furthermore, BatchExperiments violates CRAN policies by\n calling internal functions of BatchJobs.\n- Data base issues: Although we invested weeks to mitigate issues with\n locks of the SQLite data base or file system (staged queries, file\n system timeouts, …), `BatchJobs` kept working unreliable on some\n systems with high latency under certain conditions. This made\n `BatchJobs` unusable for many users.\n\n[BatchJobs](https://github.com/tudo-r/BatchJobs/) and\n[BatchExperiments](https://github.com/tudo-r/Batchexperiments) will\nremain on CRAN, but new features are unlikely to be ported back. The\n[vignette](https://batchtools.mlr-org.com/articles/batchtools.html)\ncontains a section comparing the packages.\n\n## Resources\n\n- [Function reference](https://batchtools.mlr-org.com/reference/)\n- [Vignette](https://batchtools.mlr-org.com/articles/batchtools.html)\n- [JOSS Paper](https://doi.org/10.21105/joss.00135): Short paper on\n batchtools. Please cite this if you use batchtools.\n- [Paper on\n BatchJobs/BatchExperiments](https://www.jstatsoft.org/v64/i11): The\n described concept still holds for batchtools and most examples work\n analogously (see the\n [vignette](https://batchtools.mlr-org.com/articles/batchtools.html)\n for differences between the packages).\n\n## Citation\n\nPlease cite the [JOSS paper](https://doi.org/10.21105/joss.00135) using\nthe following BibTeX entry:\n\n @article{,\n doi = {10.21105/joss.00135},\n url = {https://doi.org/10.21105/joss.00135},\n year = {2017},\n month = {feb},\n publisher = {The Open Journal},\n volume = {2},\n number = {10},\n author = {Michel Lang and Bernd Bischl and Dirk Surmann},\n title = {batchtools: Tools for R to work on batch systems},\n journal = {The Journal of Open Source Software}\n }\n\n## Related Software\n\n- The [High Performance Computing Task\n View](https://cran.r-project.org/view=HighPerformanceComputing) lists\n the most relevant packages for scientific computing with R.\n- [clustermq](https://cran.r-project.org/package=clustermq) is a similar\n approach which also supports multiple schedulers. Uses the ZeroMQ\n network protocol for communication, and shines if you have millions of\n fast jobs.\n- [batch](https://cran.r-project.org/package=batch) assists in splitting\n and submitting jobs to LSF and MOSIX clusters.\n- [flowr](https://cran.r-project.org/package=flowr) supports LSF, Slurm,\n TORQUE and Moab and provides a scatter-gather approach to define\n computational jobs.\n- [future.batchtools](https://cran.r-project.org/package=future.batchtools)\n implements `batchtools` as backend for\n [future](https://cran.r-project.org/package=future.batchtools).\n- [doFuture](https://cran.r-project.org/package=doFuture) together with\n [future.batchtools](https://cran.r-project.org/package=future.batchtools)\n connects `batchtools` to\n [foreach](https://cran.r-project.org/package=foreach).\n- [drake](https://cran.r-project.org/package=drake) uses graphs to\n define computational jobs. `batchtools` is used as a backend via\n [future.batchtools](https://cran.r-project.org/package=future.batchtools).\n\n## Contributing to batchtools\n\nThis R package is licensed under the\n[LGPL-3](https://www.gnu.org/licenses/lgpl-3.0.en.html). If you\nencounter problems using this software (lack of documentation,\nmisleading or wrong documentation, unexpected behaviour, bugs, …) or\njust want to suggest features, please open an issue in the [issue\ntracker](https://github.com/mlr-org/batchtools/issues). Pull requests\nare welcome and will be included at the discretion of the author. If you\nhave customized a template file for your (larger) computing site, please\nshare it: fork the repository, place your template in `inst/templates`\nand send a pull request.\n" }, { "path": "_pkgdown.yml", "content": "url: https://batchtools.mlr-org.com\n\ntemplate:\n bootstrap: 5\n light-switch: true\n math-rendering: mathjax\n package: mlr3pkgdowntemplate\n\ndevelopment:\n mode: auto\n version_label: default\n version_tooltip: \"Version\"\n\ntoc:\n depth: 3\n\nnavbar:\n structure:\n left: [reference, news, articles, book]\n right: [search, github, mattermost, stackoverflow, rss, lightswitch]\n components:\n home: ~\n reference:\n icon: fa fa-file-alt\n text: Reference\n href: reference/index.html\n mattermost:\n icon: fa fa-comments\n href: https://lmmisld-lmu-stats-slds.srv.mwn.de/mlr_invite/\n book:\n text: mlr3book\n icon: fa fa-link\n href: https://mlr3book.mlr-org.com\n stackoverflow:\n icon: fab fa-stack-overflow\n href: https://stackoverflow.com/questions/tagged/mlr3\n rss:\n icon: fa-rss\n href: https://mlr-org.com/\n\nreference:\n - title: Overview\n contents:\n - batchtools\n - title: Registry\n contents:\n - Registry\n - ExperimentRegistry\n - assertRegistry\n - loadRegistry\n - saveRegistry\n - syncRegistry\n - sweepRegistry\n - removeRegistry\n - getDefaultRegistry\n - title: Define Jobs\n contents:\n - batchMap\n - batchReduce\n - batchMapResults\n - Problem\n - Algorithm\n - addExperiments\n - title: Submit Jobs\n contents:\n - submitJobs\n - batchExport\n - waitForJobs\n - chunk\n - lpt\n - binpack\n - setJobNames\n - title: Query Job Information\n contents:\n - getStatus\n - findJobs\n - getJobPars\n - summarizeExperiments\n - title: Retrieve Results\n contents:\n - reduceResults\n - reduceResultsDataTable\n - loadResult\n - unwrap\n - title: Debugging\n contents:\n - resetJobs\n - testJob\n - getLog\n - getErrorMessages\n - grepLogs\n - title: Remove Jobs\n contents:\n - killJobs\n - clearRegistry\n - removeExperiments\n - title: Additional objects\n contents:\n - Job\n - Experiment\n - JobCollection\n - title: Cluster Functions\n contents:\n - cfKillJob\n - cfBrewTemplate\n - cfReadBrewTemplate\n - cfHandleUnknownSubmitError\n - ClusterFunctions\n - starts_with(\"makeCluster\")\n - SubmitJobResult\n - Hook\n - Worker\n - title: Miscellaneous\n contents:\n - Tags\n - btlapply\n - JoinTables\n - runOSCommand\n - execJob\n - doJobCollection\n - estimateRuntimes\n" }, { "path": "docs/404.html", "content": "\n\n\n \n \n\n\n\nPage not found (404) • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
\n
\n
\n
\n
\n \n \n batchtools\n 0.9.14\n \n
\n\n
\n \n \n \n
\n
\n
\n\n \n\n
\n\n
\n
\n
\n

Page not found (404)

\n
\n\nContent not found. Please use links in the navbar.\n\n
\n\n
\n \n
\n\n
\n\n\n\n
\n
\n

Developed by Michel Lang, Bernd Bischl.

\n
\n\n
\n

Site built with pkgdown 1.6.1.

\n
\n\n
\n
\n\n \n\n\n \n\n\n\n" }, { "path": "docs/CNAME", "content": "batchtools.mlr-org.com" }, { "path": "docs/LICENSE-text.html", "content": "\n\n\n \n \n\n\n\nLicense • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
\n
\n
\n
\n
\n \n \n batchtools\n 0.9.14\n \n
\n\n
\n \n \n \n
\n
\n
\n\n \n\n
\n\n
\n
\n
\n

License

\n
\n\n
                   GNU LESSER GENERAL PUBLIC LICENSE\n                       Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>\n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n\n  This version of the GNU Lesser General Public License incorporates\nthe terms and conditions of version 3 of the GNU General Public\nLicense, supplemented by the additional permissions listed below.\n\n  0. Additional Definitions.\n\n  As used herein, \"this License\" refers to version 3 of the GNU Lesser\nGeneral Public License, and the \"GNU GPL\" refers to version 3 of the GNU\nGeneral Public License.\n\n  \"The Library\" refers to a covered work governed by this License,\nother than an Application or a Combined Work as defined below.\n\n  An \"Application\" is any work that makes use of an interface provided\nby the Library, but which is not otherwise based on the Library.\nDefining a subclass of a class defined by the Library is deemed a mode\nof using an interface provided by the Library.\n\n  A \"Combined Work\" is a work produced by combining or linking an\nApplication with the Library.  The particular version of the Library\nwith which the Combined Work was made is also called the \"Linked\nVersion\".\n\n  The \"Minimal Corresponding Source\" for a Combined Work means the\nCorresponding Source for the Combined Work, excluding any source code\nfor portions of the Combined Work that, considered in isolation, are\nbased on the Application, and not on the Linked Version.\n\n  The \"Corresponding Application Code\" for a Combined Work means the\nobject code and/or source code for the Application, including any data\nand utility programs needed for reproducing the Combined Work from the\nApplication, but excluding the System Libraries of the Combined Work.\n\n  1. Exception to Section 3 of the GNU GPL.\n\n  You may convey a covered work under sections 3 and 4 of this License\nwithout being bound by section 3 of the GNU GPL.\n\n  2. Conveying Modified Versions.\n\n  If you modify a copy of the Library, and, in your modifications, a\nfacility refers to a function or data to be supplied by an Application\nthat uses the facility (other than as an argument passed when the\nfacility is invoked), then you may convey a copy of the modified\nversion:\n\n   a) under this License, provided that you make a good faith effort to\n   ensure that, in the event an Application does not supply the\n   function or data, the facility still operates, and performs\n   whatever part of its purpose remains meaningful, or\n\n   b) under the GNU GPL, with none of the additional permissions of\n   this License applicable to that copy.\n\n  3. Object Code Incorporating Material from Library Header Files.\n\n  The object code form of an Application may incorporate material from\na header file that is part of the Library.  You may convey such object\ncode under terms of your choice, provided that, if the incorporated\nmaterial is not limited to numerical parameters, data structure\nlayouts and accessors, or small macros, inline functions and templates\n(ten or fewer lines in length), you do both of the following:\n\n   a) Give prominent notice with each copy of the object code that the\n   Library is used in it and that the Library and its use are\n   covered by this License.\n\n   b) Accompany the object code with a copy of the GNU GPL and this license\n   document.\n\n  4. Combined Works.\n\n  You may convey a Combined Work under terms of your choice that,\ntaken together, effectively do not restrict modification of the\nportions of the Library contained in the Combined Work and reverse\nengineering for debugging such modifications, if you also do each of\nthe following:\n\n   a) Give prominent notice with each copy of the Combined Work that\n   the Library is used in it and that the Library and its use are\n   covered by this License.\n\n   b) Accompany the Combined Work with a copy of the GNU GPL and this license\n   document.\n\n   c) For a Combined Work that displays copyright notices during\n   execution, include the copyright notice for the Library among\n   these notices, as well as a reference directing the user to the\n   copies of the GNU GPL and this license document.\n\n   d) Do one of the following:\n\n       0) Convey the Minimal Corresponding Source under the terms of this\n       License, and the Corresponding Application Code in a form\n       suitable for, and under terms that permit, the user to\n       recombine or relink the Application with a modified version of\n       the Linked Version to produce a modified Combined Work, in the\n       manner specified by section 6 of the GNU GPL for conveying\n       Corresponding Source.\n\n       1) Use a suitable shared library mechanism for linking with the\n       Library.  A suitable mechanism is one that (a) uses at run time\n       a copy of the Library already present on the user's computer\n       system, and (b) will operate properly with a modified version\n       of the Library that is interface-compatible with the Linked\n       Version.\n\n   e) Provide Installation Information, but only if you would otherwise\n   be required to provide such information under section 6 of the\n   GNU GPL, and only to the extent that such information is\n   necessary to install and execute a modified version of the\n   Combined Work produced by recombining or relinking the\n   Application with a modified version of the Linked Version. (If\n   you use option 4d0, the Installation Information must accompany\n   the Minimal Corresponding Source and Corresponding Application\n   Code. If you use option 4d1, you must provide the Installation\n   Information in the manner specified by section 6 of the GNU GPL\n   for conveying Corresponding Source.)\n\n  5. Combined Libraries.\n\n  You may place library facilities that are a work based on the\nLibrary side by side in a single library together with other library\nfacilities that are not Applications and are not covered by this\nLicense, and convey such a combined library under terms of your\nchoice, if you do both of the following:\n\n   a) Accompany the combined library with a copy of the same work based\n   on the Library, uncombined with any other library facilities,\n   conveyed under the terms of this License.\n\n   b) Give prominent notice with the combined library that part of it\n   is a work based on the Library, and explaining where to find the\n   accompanying uncombined form of the same work.\n\n  6. Revised Versions of the GNU Lesser General Public License.\n\n  The Free Software Foundation may publish revised and/or new versions\nof the GNU Lesser General Public License from time to time. Such new\nversions will be similar in spirit to the present version, but may\ndiffer in detail to address new problems or concerns.\n\n  Each version is given a distinguishing version number. If the\nLibrary as you received it specifies that a certain numbered version\nof the GNU Lesser General Public License \"or any later version\"\napplies to it, you have the option of following the terms and\nconditions either of that published version or of any later version\npublished by the Free Software Foundation. If the Library as you\nreceived it does not specify a version number of the GNU Lesser\nGeneral Public License, you may choose any version of the GNU Lesser\nGeneral Public License ever published by the Free Software Foundation.\n\n  If the Library as you received it specifies that a proxy can decide\nwhether future versions of the GNU Lesser General Public License shall\napply, that proxy's public statement of acceptance of any version is\npermanent authorization for you to choose that version for the\nLibrary.\n
\n\n
\n\n
\n \n
\n\n
\n\n\n\n
\n
\n

Developed by Michel Lang, Bernd Bischl.

\n
\n\n
\n

Site built with pkgdown 1.6.1.

\n
\n\n
\n
\n\n \n\n\n \n\n\n\n" }, { "path": "docs/articles/batchtools.html", "content": "\n\n\n\n\n\n\nbatchtools • batchtools\n\n\n\n\n\n\n\n\n\n\n
\n
\n
\n
\n \n \n batchtools\n 0.9.14\n \n
\n\n
\n \n\n
\n\n
\n\n
\n\n\n \n\n
\n
\n
\n

batchtools

\n \n \n Source: vignettes/batchtools.Rmd\n
batchtools.Rmd
\n\n
\n\n \n \n
\n

\nSetup

\n
\n

\nCluster Functions

\n

The communication with the batch system is managed via so-called cluster functions. They are created with the constructor makeClusterFunctions which defines how jobs are submitted on your system. Furthermore, you may provide functions to list queued/running jobs and to kill jobs.

\n

Usually you do not have to start from scratch but can just use one of the cluster functions which ship with the package:

\n\n

To use the package with the socket cluster functions, you would call the respective constructor makeClusterFunctionsSocket():

\n
\nreg = makeRegistry(NA)\nreg$cluster.functions = makeClusterFunctionsSocket(2)
\n

To make this selection permanent for this registry, save the Registry with saveRegistry(). To make your cluster function selection permanent for a specific system across R sessions for all new Registries, you can set up a configuration file (see below).

\n

If you have trouble debugging your cluster functions, you can enable the debug mode for extra output. To do so, install the debugme package and set the environment variable DEBUGME to batchtools before you load the batchtools package:

\n
\nSys.setenv(DEBUGME = \"batchtools\")\nlibrary(batchtools)
\n
\n
\n

\nTemplate Files

\n

Many cluster functions require a template file as argument. These templates are used to communicate with the scheduler and contain placeholders to evaluate arbitrary R expressions. Internally, the brew package is used for this purpose. Some exemplary template files can be found here. It would be great if you would help expand this collection to cover more exotic configurations. To do so, please send your template via mail or open a new pull request.

\n

Note that all variables defined in a JobCollection can be used inside the template. If you need to pass extra variables, you can set them via the argument resources of submitJobs().

\n

If the flexibility which comes with templating is not sufficient, you can still construct a custom cluster function implementation yourself using the provided constructor.

\n
\n
\n

\nConfiguration File

\n

The configuration file can be used to set system specific options. Its default location depends on the operating system (see Registry), but for the first time setup you can put one in the current working directory (as reported by getwd()). In order to set the cluster function implementation, you would generate a file with the following content:

\n
\ncluster.functions = makeClusterFunctionsInteractive()
\n

The configuration file is parsed whenever you create or load a Registry. It is sourced inside of your registry which has the advantage that you can (a) access all of the parameters which are passed to makeRegistry and (b) you can also directly change them. Lets say you always want your working directory in your home directory and you always want to load the checkmate package on the nodes, you can just append these lines:

\n
\nwork.dir = \"~\"\npackages = union(packages, \"checkmate\")
\n

See the documentation on Registry for a more complete list of supported configuration options.

\n
\n
\n
\n

\nMigration from BatchJobs/Batchexperiments\n

\n

The development of BatchJobs and BatchExperiments is discontinued because of the following reasons:

\n
    \n
  • Maintainability: The packages BatchJobs and BatchExperiments are tightly connected which makes maintaining difficult. Changes have to be synchronized and tested against the current CRAN versions for compatibility. Furthermore, BatchExperiments violates CRAN policies by calling internal functions of BatchJobs.
    • \n
  • Data base issues: Although we invested weeks to mitigate issues with locks of the SQLite data base or file system (staged queries, file system timeouts, …), BatchJobs kept working unreliable on some systems with high latency or specific file systems. This made BatchJobs unusable for many users.
    • \n
\n

BatchJobs and BatchExperiments will remain on CRAN, but new features are unlikely to be ported back.

\n
\n

\nInternal Changes

\n
    \n
  • batchtools does not use SQLite anymore. Instead, all the information is stored directly in the registry using data.tables acting as an in-memory database. As a side effect, many operations are much faster.
    • \n
  • Nodes do not have to access the registry. submitJobs() stores a temporary object of type JobCollection on the file system which holds all the information necessary to execute a chunk of jobs via doJobCollection() on the node. This avoids file system locks because each job accesses only one file exclusively.
    • \n
  • \nClusterFunctionsMulticore now uses the parallel package for multicore execution.
    • \n
  • \nClusterFunctionsSSH can still be used to emulate a scheduler-like system which respects the work load on the local machine. Setting the hostname to \"localhost\" just strips out ssh of the command issued.
    • \n
\n
\n
\n

\nInterface Changes

\n
    \n
  • batchtools remembers the last created or loaded Registry and sets it as default registry. This way, you do not need to pass the registry around anymore. If you need to work with multiple registries simultaneously on the other hand, you can still do so by explicitly passing registries to the functions.
    • \n
  • Most functions now return a data.table which is keyed with the job.id. This way, return values can be joined together easily and efficient (see this help page for some examples).
    • \n
  • The building blocks of a problem has been renamed from static and dynamic to the more intuitive data and fun. Thus, algorithm function should have the formal arguments job, data and instance.
    • \n
  • The function makeDesign has been removed. Parameters can be defined by just passing a data.frame or data.table to addExperiments. For exhaustive designs, use data.table::CJ().
    • \n
\n
\n
\n

\nTemplate changes

\n
    \n
  • The scheduler should directly execute the command:
    • \n
\n
Rscript -e 'batchtools::doJobCollection(<filename>)'
\n

There is no intermediate R source file like there was in BatchJobs. * All information stored in the object JobCollection can be accessed while brewing the template. * Extra variables may be passed via the argument resoures of submitJobs.

\n
\n
\n

\nNew features

\n
    \n
  • Support for Docker Swarm via ClusterFunctionsDocker.
    • \n
  • Jobs can now be tagged and untagged to provide an easy way to group them.
    • \n
  • Some resources like the number of CPUs are now optionally passed to parallelMap. This eases nested parallelization, e.g. to use multicore parallelization on the slave by just setting a resource on the master. See submitJobs() for an example.
    • \n
  • \nClusterFunctions are now more flexible in general as they can define hook functions which will be called at certain events. ClusterFunctionsDocker is an example use case which implements a housekeeping routine. This routine is called every time before a job is about to get submitted to the scheduler (in the case: the Docker Swarm) via the hook pre.submit and every time directly after the registry synchronized jobs stored on the file system via the hook post.sync.
    • \n
  • More new features are covered in the NEWS.
    • \n
\n
\n
\n

\nPorting to batchtools\n

\n

The following table assists in porting to batchtools by mapping BatchJobs/BatchExperiments functions to their counterparts in batchtools. The table does not cover functions which are (a) used only internally in BatchJobs and (b) functions which have not been renamed.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
BatchJobsbatchtools
addRegistryPackagesSet reg$packages or reg$namespaces, call saveRegistry()\n
addRegistrySourceDirs-
addRegistrySourceFilesSet reg$source, call saveRegistry()\n
batchExpandGrid\nbatchMap: batchMap(..., args = CJ(x = 1:3, y = 1:10))\n
batchMapQuickbtmapply
batchReduceResults-
batchUnexportbatchExport
filterResults-
getJobIdsfindJobs
getJobInfogetJobStatus
getJobmakeJob
getJobParamDfgetJobPars
loadResultsreduceResultsList
reduceResultsDataFramereduceResultsDataTable
reduceResultsMatrix\nreduceResultsList + do.call(rbind, res)\n
reduceResultsVectorreduceResultsDataTable
setJobFunction-
setJobNames-
showStatusgetStatus
\n
\n
\n
\n

\nExample 1: Approximation of \\(\\pi\\)\n

\n

To get a first insight into the usage of batchtools, we start with an exemplary Monte Carlo simulation to approximate \\(\\pi\\). For background information, see Wikipedia.

\n

First, a so-called registry object has to be created, which defines a directory where all relevant information, files and results of the computational jobs will be stored. There are two different types of registry objects: First, a regular Registry which we will use in this example. Second, an ExperimentRegistry which provides an alternative way to define computational jobs and thereby is tailored for a broad range of large scale computer experiments. Here, we use a temporary registry which is stored in the temp directory of the system and gets automatically deleted if you close the R session.

\n
\nreg = makeRegistry(file.dir = NA, seed = 1)
\n

For a permanent registry, set the file.dir to a valid path. It can then be reused later, e.g., when you login to the system again, by calling the function loadRegistry(file.dir).

\n

When a registry object is created or loaded, it is stored for the active R session as the default. Therefore the argument reg will be ignored in functions calls of this example, assuming the correct registry is set as default. To get the current default registry, getDefaultRegistry can be used. To switch to another registry, use setDefaultRegistry().

\n

First, we create a function which samples \\(n\\) points \\((x_i, y_i)\\) whereas \\(x_i\\) and \\(y_i\\) are distributed uniformly, i.e. \\(x_i, y_i \\sim \\mathcal{U}(0,1)\\). Next, the distance to the origin \\((0, 0)\\) is calculated and the fraction of points in the unit circle (\\(d \\leq 1\\)) is returned.

\n
\npiApprox = function(n) {\n  nums = matrix(runif(2 * n), ncol = 2)\n  d = sqrt(nums[, 1]^2 + nums[, 2]^2)\n  4 * mean(d <= 1)\n}\nset.seed(42)\npiApprox(1000)
\n
## [1] 3.156
\n

We now parallelize piApprox() with batchtools: We create 10 jobs, each doing a MC simulation with \\(10^5\\) jobs. We use batchMap() to define the jobs (note that this does not yet start the calculation):

\n
\nbatchMap(fun = piApprox, n = rep(1e5, 10))
\n
## Adding 10 jobs ...
\n

The length of the vector or list defines how many different jobs are created, while the elements itself are used as arguments for the function. The function batchMap(fun, ...) works analogously to Map(f, ...) of the base package. An overview over the jobs and their IDs can be retrieved with getJobTable() which returns a data.frame with all relevant information:

\n\n
##  [1] \"job.id\"       \"submitted\"    \"started\"      \"done\"         \"error\"       \n##  [6] \"mem.used\"     \"batch.id\"     \"log.file\"     \"job.hash\"     \"job.name\"    \n## [11] \"time.queued\"  \"time.running\" \"job.pars\"     \"resources\"    \"tags\"
\n

Note that a unique job ID is assigned to each job. These IDs can be used to restrict operations to subsets of jobs. To actually start the calculation, call submitJobs(). The registry and the selected job IDs can be taken as arguments as well as an arbitrary list of resource requirements, which are to be handled by the cluster back end.

\n
\nsubmitJobs(resources = list(walltime = 3600, memory = 1024))
\n
## Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
\n

In this example, a cap for the execution time (so-called walltime) and for the maximum memory requirements are set. The progress of the submitted jobs can be checked with getStatus().

\n\n
## Status for 10 jobs at 2020-10-21 09:39:36:\n##   Submitted    : 10 (100.0%)\n##   -- Queued    :  0 (  0.0%)\n##   -- Started   : 10 (100.0%)\n##   ---- Running :  0 (  0.0%)\n##   ---- Done    : 10 (100.0%)\n##   ---- Error   :  0 (  0.0%)\n##   ---- Expired :  0 (  0.0%)
\n

The resulting output includes the number of jobs in the registry, how many have been submitted, have started to execute on the batch system, are currently running, have successfully completed, and have terminated due to an R exception. After jobs have successfully terminated, we can load their results on the master. This can be done in a simple fashion by using either loadResult(), which returns a single result exactly in the form it was calculated during mapping, or by using reduceResults(), which is a version of Reduce() from the base package for registry objects.

\n\n
## [1] TRUE
\n
\nmean(sapply(1:10, loadResult))
\n
## [1] 3.140652
\n
\nreduceResults(function(x, y) x + y) / 10
\n
## [1] 3.140652
\n

If you are absolutely sure that your function works, you can take a shortcut and use batchtools in an lapply fashion using btlapply(). This function creates a temporary registry (but you may also pass one yourself), calls batchMap(), wait for the jobs to terminate with waitForJobs() and then uses reduceResultsList() to return the results.

\n
\nres = btlapply(rep(1e5, 10), piApprox)\nmean(unlist(res))
\n
## [1] 3.14124
\n
\n
\n

\nExample 2: Machine Learning

\n

We stick to a rather simple, but not unrealistic example to explain some further functionalities: Applying two classification learners to the famous iris data set (Anderson 1935), vary a few hyperparameters and evaluate the effect on the classification performance.

\n

First, we create a registry, the central meta-data object which records technical details and the setup of the experiments. We use an ExperimentRegistry where the job definition is split into creating problems and algorithms. See the paper on BatchJobs and BatchExperiments for a detailed explanation. Again, we use a temporary registry and make it the default registry.

\n
\nlibrary(batchtools)\nreg = makeExperimentRegistry(file.dir = NA, seed = 1)
\n
\n

\nProblems and Algorithms

\n

By adding a problem to the registry, we can define the data on which certain computational jobs shall work. This can be a matrix, data frame or array that always stays the same for all subsequent experiments. But it can also be of a more dynamic nature, e.g., subsamples of a dataset or random numbers drawn from a probability distribution . Therefore the function addProblem() accepts static parts in its data argument, which is passed to the argument fun which generates a (possibly stochastic) problem instance. For data, any R object can be used. If only data is given, the generated instance is data. The argument fun has to be a function with the arguments data and job (and optionally other arbitrary parameters). The argument job is an object of type Job which holds additional information about the job.

\n

We want to split the iris data set into a training set and test set. In this example we use use subsampling which just randomly takes a fraction of the observations as training set. We define a problem function which returns the indices of the respective training and test set for a split with 100 * ratio% of the observations being in the test set:

\n
\nsubsample = function(data, job, ratio, ...) {\n  n = nrow(data)\n  train = sample(n, floor(n * ratio))\n  test = setdiff(seq_len(n), train)\n  list(test = test, train = train)\n}
\n

addProblem() files the problem to the file system and the problem gets recorded in the registry.

\n
\ndata(\"iris\", package = \"datasets\")\naddProblem(name = \"iris\", data = iris, fun = subsample, seed = 42)
\n
## Adding problem 'iris'
\n

The function call will be evaluated at a later stage on the workers. In this process, the data part will be loaded and passed to the function. Note that we set a problem seed to synchronize the experiments in the sense that the same resampled training and test sets are used for the algorithm comparison in each distinct replication.

\n

The algorithms for the jobs are added to the registry in a similar manner. When using addAlgorithm(), an identifier as well as the algorithm to apply to are required arguments. The algorithm must be given as a function with arguments job, data and instance. Further arbitrary arguments (e.g., hyperparameters or strategy parameters) may be defined analogously as for the function in addProblem. The objects passed to the function via job and data are here the same as above, while via instance the return value of the evaluated problem function is passed. The algorithm can return any R object which will automatically be stored on the file system for later retrieval. Firstly, we create an algorithm which applies a support vector machine:

\n
\nsvm.wrapper = function(data, job, instance, ...) {\n  library(\"e1071\")\n  mod = svm(Species ~ ., data = data[instance$train, ], ...)\n  pred = predict(mod, newdata = data[instance$test, ], type = \"class\")\n  table(data$Species[instance$test], pred)\n}\naddAlgorithm(name = \"svm\", fun = svm.wrapper)
\n
## Adding algorithm 'svm'
\n

Secondly, a random forest of classification trees:

\n
\nforest.wrapper = function(data, job, instance, ...) {\n  library(\"ranger\")\n  mod = ranger(Species ~ ., data = data[instance$train, ], write.forest = TRUE)\n  pred = predict(mod, data = data[instance$test, ])\n  table(data$Species[instance$test], pred$predictions)\n}\naddAlgorithm(name = \"forest\", fun = forest.wrapper)
\n
## Adding algorithm 'forest'
\n

Both algorithms return a confusion matrix for the predictions on the test set, which will later be used to calculate the misclassification rate.

\n

Note that using the ... argument in the wrapper definitions allows us to circumvent naming specific design parameters for now. This is an advantage if we later want to extend the set of algorithm parameters in the experiment. The algorithms get recorded in the registry and the corresponding functions are stored on the file system.

\n

Defined problems and algorithms can be queried with:

\n
\nreg$problems
\n
## [1] \"iris\"
\n
\nreg$algorithms
\n
## [1] \"svm\"    \"forest\"
\n

The flow to define experiments is summarized in the following figure:

\n

\n
\n
\n

\nCreating jobs

\n

addExperiments() is used to parametrize the jobs and thereby define computational jobs. To do so, you have to pass named lists of parameters to addExperiments(). The elements of the respective list (one for problems and one for algorithms) must be named after the problem or algorithm they refer to. The data frames contain parameter constellations for the problem or algorithm function where columns must have the same names as the target arguments. When the problem design and the algorithm design are combined in addExperiments(), each combination of the parameter sets of the two designs defines a distinct job. How often each of these jobs should be computed can be determined with the argument repls.

\n
\n# problem design: try two values for the ratio parameter\npdes = list(iris = data.table(ratio = c(0.67, 0.9)))\n\n# algorithm design: try combinations of kernel and epsilon exhaustively,\n# try different number of trees for the forest\nades = list(\n  svm = CJ(kernel = c(\"linear\", \"polynomial\", \"radial\"), epsilon = c(0.01, 0.1)),\n  forest = data.table(ntree = c(100, 500, 1000))\n)\n\naddExperiments(pdes, ades, repls = 5)
\n
## Adding 60 experiments ('iris'[2] x 'svm'[6] x repls[5]) ...
\n
## Adding 30 experiments ('iris'[2] x 'forest'[3] x repls[5]) ...
\n

The jobs are now available in the registry with an individual job ID for each. The function summarizeExperiments() returns a table which gives a quick overview over all defined experiments.

\n\n
##    problem algorithm .count\n##     <char>    <char>  <int>\n## 1:    iris       svm     60\n## 2:    iris    forest     30
\n
\nsummarizeExperiments(by = c(\"problem\", \"algorithm\", \"ratio\"))
\n
##    problem algorithm ratio .count\n##     <char>    <char> <num>  <int>\n## 1:    iris       svm  0.67     30\n## 2:    iris       svm  0.90     30\n## 3:    iris    forest  0.67     15\n## 4:    iris    forest  0.90     15
\n
\n
\n

\nBefore Submitting

\n

Before submitting all jobs to the batch system, we encourage you to test each algorithm individually. Or sometimes you want to submit only a subset of experiments because the jobs vastly differ in runtime. Another reoccurring task is the collection of results for only a subset of experiments. For all these use cases, findExperiments() can be employed to conveniently select a particular subset of jobs. It returns the IDs of all experiments that match the given criteria. Your selection can depend on substring matches of problem or algorithm IDs using prob.name or algo.name, respectively. You can also pass R expressions, which will be evaluated in your problem parameter setting (prob.pars) or algorithm parameter setting (algo.pars). The expression is then expected to evaluate to a Boolean value. Furthermore, you can restrict the experiments to specific replication numbers.

\n

To illustrate findExperiments(), we will select two experiments, one with a support vector machine and the other with a random forest and the parameter ntree = 1000. The selected experiment IDs are then passed to testJob.

\n
\nid1 = head(findExperiments(algo.name = \"svm\"), 1)\nprint(id1)
\n
##    job.id\n##     <int>\n## 1:      1
\n
\nid2 = head(findExperiments(algo.name = \"forest\", algo.pars = (ntree == 1000)), 1)\nprint(id2)
\n
##    job.id\n##     <int>\n## 1:     71
\n
\ntestJob(id = id1)
\n
## ### [bt]: Generating problem instance for problem 'iris' ...\n## ### [bt]: Applying algorithm 'svm' on problem 'iris' for job 1 (seed = 2) ...
\n
##             pred\n##              setosa versicolor virginica\n##   setosa         13          0         0\n##   versicolor      0         17         0\n##   virginica       0          1        19
\n
\ntestJob(id = id2)
\n
## ### [bt]: Generating problem instance for problem 'iris' ...\n## ### [bt]: Applying algorithm 'forest' on problem 'iris' for job 71 (seed = 72) ...
\n
##             \n##              setosa versicolor virginica\n##   setosa         13          0         0\n##   versicolor      0         16         1\n##   virginica       0          1        19
\n

If something goes wrong, batchtools comes with a bunch of useful debugging utilities (see separate vignette on error handling). If everything turns out fine, we can proceed with the calculation.

\n
\n
\n

\nSubmitting and Collecting Results

\n

To submit the jobs, we call submitJobs() and wait for all jobs to terminate using waitForJobs().

\n\n
## Submitting 90 jobs in 90 chunks using cluster functions 'Interactive' ...
\n\n
## [1] TRUE
\n

After jobs are finished, the results can be collected with reduceResultsDataTable() where we directly extract the mean misclassification error:

\n
\nreduce = function(res) list(mce = (sum(res) - sum(diag(res))) / sum(res))\nresults = unwrap(reduceResultsDataTable(fun = reduce))\nhead(results)
\n
##    job.id   mce\n##     <int> <num>\n## 1:      1  0.02\n## 2:      2  0.00\n## 3:      3  0.04\n## 4:      4  0.06\n## 5:      5  0.02\n## 6:      6  0.02
\n

Next, we merge the results table with the table of job parameters using one of the join helpers provided by batchtools (here, we use an inner join):

\n
\npars = unwrap(getJobPars())\ntab = ijoin(pars, results)\nhead(tab)
\n
##    job.id problem algorithm ratio kernel epsilon ntree   mce\n##     <int>  <char>    <char> <num> <char>   <num> <num> <num>\n## 1:      1    iris       svm  0.67 linear    0.01    NA  0.02\n## 2:      2    iris       svm  0.67 linear    0.01    NA  0.00\n## 3:      3    iris       svm  0.67 linear    0.01    NA  0.04\n## 4:      4    iris       svm  0.67 linear    0.01    NA  0.06\n## 5:      5    iris       svm  0.67 linear    0.01    NA  0.02\n## 6:      6    iris       svm  0.67 linear    0.10    NA  0.02
\n

We now aggregate the results group-wise. You can use data.table, base::aggregate(), or the dplyr package for this purpose. Here, we use data.table to subset the table to jobs where the ratio is 0.67 and group by algorithm the algorithm hyperparameters:

\n
\ntab[ratio == 0.67, list(mmce = mean(mce)),\n  by = c(\"algorithm\", \"kernel\", \"epsilon\", \"ntree\")]
\n
##    algorithm     kernel epsilon ntree  mmce\n##       <char>     <char>   <num> <num> <num>\n## 1:       svm     linear    0.01    NA 0.028\n## 2:       svm     linear    0.10    NA 0.028\n## 3:       svm polynomial    0.01    NA 0.096\n## 4:       svm polynomial    0.10    NA 0.096\n## 5:       svm     radial    0.01    NA 0.044\n## 6:       svm     radial    0.10    NA 0.044\n## 7:    forest       <NA>      NA   100 0.044\n## 8:    forest       <NA>      NA   500 0.048\n## 9:    forest       <NA>      NA  1000 0.044
\n
\n
\n
\n

\nExample: Error Handling

\n

In any large scale experiment many things can and will go wrong. The cluster might have an outage, jobs may run into resource limits or crash, subtle bugs in your code could be triggered or any other error condition might arise. In these situations it is important to quickly determine what went wrong and to recompute only the minimal number of required jobs.

\n

Therefore, before you submit anything you should use testJob() to catch errors that are easy to spot because they are raised in many or all jobs. If external is set, this function runs the job without side effects in an independent R process on your local machine via Rscript similar as on the slave, redirects the output of the process to your R console, loads the job result and returns it. If you do not set external, the job is executed is in the currently running R session, with the drawback that you might be unable to catch missing variable declarations or missing package dependencies.

\n

By way of illustration here is a small example. First, we create a temporary registry.

\n
\nlibrary(batchtools)\nreg = makeRegistry(file.dir = NA, seed = 1)
\n

Ten jobs are created, one will trow a warning and two of them will raise an exception.

\n
\nflakeyFunction <- function(value) {\n  if (value == 5) warning(\"Just a simple warning\")\n  if (value %in% c(2, 9)) stop(\"Ooops.\")\n  value^2\n}\nbatchMap(flakeyFunction, 1:10)
\n
## Adding 10 jobs ...
\n

Now that the jobs are defined, we can test jobs independently:

\n
\ntestJob(id = 1)
\n
## ### [bt]: Setting seed to 2 ...
\n
## [1] 1
\n

In this case, testing the job with ID = 1 provides the appropriate result but testing the job with ID = 2 leads to an error:

\n\n
## ### [bt]: Setting seed to 3 ...\n## Error in (function (value)  : Ooops.
\n
## [1] \"Error in (function (value)  : Ooops.\\n\"
\n

We ignore the error here, and just assume everything looks fine and submit all jobs.

\n\n
## Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
\n
## Error in (function (value)  : Ooops.
\n
## Warning in (function (value) : Just a simple warning
\n
## Error in (function (value)  : Ooops.
\n\n
## [1] FALSE
\n

After you have submitted jobs and suspect that something is going wrong, the first thing to do is to run getStatus() to display a summary of the current state of the system.

\n\n
## Status for 10 jobs at 2020-10-21 09:39:40:\n##   Submitted    : 10 (100.0%)\n##   -- Queued    :  0 (  0.0%)\n##   -- Started   : 10 (100.0%)\n##   ---- Running :  0 (  0.0%)\n##   ---- Done    :  8 ( 80.0%)\n##   ---- Error   :  2 ( 20.0%)\n##   ---- Expired :  0 (  0.0%)
\n

The status message shows that two of the jobs could not be executed successfully. To get the IDs of all jobs that failed due to an error we can use findErrors() and to retrieve the actual error message, we can use getErrorMessages().

\n\n
##    job.id\n##     <int>\n## 1:      2\n## 2:      9
\n\n
##    job.id terminated  error                              message\n##     <int>     <lgcl> <lgcl>                               <char>\n## 1:      2       TRUE   TRUE Error in (function (value)  : Ooops.\n## 2:      9       TRUE   TRUE Error in (function (value)  : Ooops.
\n

If we want to peek into the R log file of a job to see more context for the error we can use showLog() which opens a pager or use getLog() to get the log as character vector:

\n
\ntail(getLog(id = 9))
\n
## [1] \"### [bt]: Memory measurement disabled\"                           \n## [2] \"### [bt]: Starting job [batchtools job.id=9]\"                    \n## [3] \"### [bt]: Setting seed to 10 ...\"                                \n## [4] \"\"                                                                \n## [5] \"### [bt]: Job terminated with an exception [batchtools job.id=9]\"\n## [6] \"### [bt]: Calculation finished!\"
\n

You can also grep for messages (output suppressed in this vignette for technical reasons):

\n
\ngrepLogs(pattern = \"simple\", ignore.case = TRUE)
\n
\n
\n

\nWorkflow

\n
\n

\nOn the Local System

\n
    \n
  1. Create a Registry with makeRegistry() (or makeExperimentRegistry()) or load an existing from the file system with loadRegistry().
    2. \n
  2. Define computational jobs with batchMap() or batchReduce() if you used makeRegistry() or define with addAlgorithm(), addProblem() and addExperiments() if you started with makeExperimentRegistry(). It is advised to test some jobs with testJob() in the interactive session and with testJob(external = TRUE) in a separate R process. Note that you can add additional jobs if you are using an ExperimentRegistry.
    3. \n
  3. If required, query the data base for job ids depending on their status, parameters or tags (see findJobs()). The returned tables can easily be combined in a set-like fashion with data base verbs: union (ojoin() for outer join), intersect (ijoin() for inner join), difference (ajoin() for anti join).
    4. \n
  4. Submit jobs with submitJobs(). You can specify job resources here. If you have thousands of fast terminating jobs, you want to chunk() them first. If some jobs already terminated, you can estimate the runtimes with estimateRuntimes() and chunk jobs into heterogeneous groups with lpt() and binpack().
    5. \n
  5. Monitor jobs. getStatus() gives a summarizing overview. Use showLog() and grepLogs() to investigate log file. Run jobs in the currently running session with testJob() to get a traceback().
    6. \n
  6. Collect (partial) results. loadResult() retrieves a single result from the file system. reduceResults() mimics Reduce() and allows to apply a function to many files in an iterative fashion. reduceResultsList() and reduceResultsDataTable() collect results into a list or data.table, respectively.
    7. \n
\n

\n
\n
\n

\nOn Multiple Systems

\n

Most users develop and prototype their experiments on a desktop box in their preferred IDE and later deploy to a large computing cluster. This can be done by prototyping locally (testJob() or submit subsets via submitJobs()). To deploy to the cluster, just copy the file directory (as reported by reg$file.dir) to the remote system. Next, log in on the cluster (typically via ssh), cd to the copied directory and call loadRegistry(\"<file.dir.on.remote\">, \"<work.dir.on.remote>\", writeable = TRUE). This function will (a) source the local configuration file so that you can talk to the cluster (verify by checking the output of reg$cluster.functions) and (b) adjust the paths to the new system if argument update.paths is set. After loading the Registry, it is advised to test some jobs again with testJob() before submitting all of them with submitJobs(resources = list()) (remember you now need to set resources!). After some jobs are finished, the file.dir can be copied back (do not merge with the previous directory!) and loaded again with loadRegistry().

\n

This approach is totally viable as long as some general rules are followed:

\n
    \n
  1. Make sure you have all packages installed. Package versions can be synchronized across machines with checkpoint or packrat.
    2. \n
  2. Test jobs on the remote system prior to submit to ensure that paths are resolved correctly.
    3. \n
  3. Make sure you have set the cluster functions in a configuration file, and stick to one backend as long as jobs are running.
    4. \n
  4. The status can only be monitored on the remote system (for obvious reasons).
    5. \n
  5. Partial results can be inspected both on the remote system and on the local system. For the latter, you need to copy over the complete file.dir first. Overwriting/merging directories is not advised as this may lead to inconsistencies if you added or removed experiments on the remote. If you have to merge, use rsync with option --delete. Load the registry locally with loadRegistry() and collect results. Do not copy back and forth.
    6. \n
  6. Avoid accessing the file.dir with multiple sessions simultaneously. This includes accessing the registry via a mount! Simultaneous access may lead to inconsistencies and missing results.
    7. \n
\n
\n
\n
\n\n
\n\n \n
\n\n
\n\n\n\n
\n

Developed by Michel Lang, Bernd Bischl.

\n
\n\n
\n

Site built with pkgdown 1.6.1.

\n
\n\n
\n
\n\n \n\n\n \n\n" }, { "path": "docs/articles/batchtools_files/header-attrs-2.4/header-attrs.js", "content": "// Pandoc 2.9 adds attributes on both header and div. We remove the former (to\n// be compatible with the behavior of Pandoc < 2.8).\ndocument.addEventListener('DOMContentLoaded', function(e) {\n var hs = document.querySelectorAll(\"div.section[class*='level'] > :first-child\");\n var i, h, a;\n for (i = 0; i < hs.length; i++) {\n h = hs[i];\n if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6\n a = h.attributes;\n while (a.length > 0) h.removeAttribute(a[0].name);\n }\n});\n" }, { "path": "docs/articles/index.html", "content": "\n\n\n \n \n\n\n\nArticles • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
\n
\n
\n
\n
\n \n \n batchtools\n 0.9.14\n \n
\n\n
\n \n \n \n
\n
\n
\n\n \n\n
\n\n
\n
\n
\n

Articles

\n
\n\n
\n

All vignettes

\n

\n\n
\n
batchtools
\n
\n
\n
\n
\n
\n\n\n
\n
\n

Developed by Michel Lang, Bernd Bischl.

\n
\n\n
\n

Site built with pkgdown 1.6.1.

\n
\n\n
\n
\n\n \n\n\n \n\n\n\n" }, { "path": "docs/authors.html", "content": "\n\n\n \n \n\n\n\nCitation and Authors • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
\n
\n
\n
\n
\n \n \n batchtools\n 0.9.14\n \n
\n\n
\n \n \n \n
\n
\n
\n\n \n\n
\n\n
\n
\n
\n

Citation

\n Source: inst/CITATION\n
\n\n

Michel Lang, Bernd Bischl, Dirk Surmann (2017). batchtools: Tools for R to work on batch systems. The Journal of Open Source Software, 2(10). URL https://doi.org/10.21105/joss.00135.

\n 
@Article{,\n  title = {batchtools: Tools for R to work on batch systems},\n  author = {Michel Lang and Bernd Bischl and Dirk Surmann},\n  journal = {The Journal of Open Source Software},\n  year = {2017},\n  month = {feb},\n  number = {10},\n  doi = {10.21105/joss.00135},\n  url = {https://doi.org/10.21105/joss.00135},\n}
\n

Bernd Bischl, Michel Lang, Olaf Mersmann, Joerg Rahnenfuehrer, Claus Weihs (2015). BatchJobs and BatchExperiments: Abstraction Mechanisms for Using R in Batch Environments. Journal of Statistical Software, 64(11), 1-25. URL https://www.jstatsoft.org/v64/i11/.

\n 
@Article{,\n  title = {{BatchJobs} and {BatchExperiments}: Abstraction Mechanisms for Using {R} in Batch Environments},\n  author = {Bernd Bischl and Michel Lang and Olaf Mersmann and J{\\\"o}rg Rahnenf{\\\"u}hrer and Claus Weihs},\n  journal = {Journal of Statistical Software},\n  year = {2015},\n  volume = {64},\n  number = {11},\n  pages = {1--25},\n  url = {https://www.jstatsoft.org/v64/i11/},\n}
\n\n
\n

Authors

\n
\n\n
    \n
  • \n

    Michel Lang. Maintainer, author. \n

    \n
    • \n
  • \n

    Bernd Bischl. Author. \n

    \n
    • \n
  • \n

    Dirk Surmann. Contributor. \n

    \n
    • \n
\n\n
\n\n
\n\n\n\n
\n
\n

Developed by Michel Lang, Bernd Bischl.

\n
\n\n
\n

Site built with pkgdown 1.6.1.

\n
\n\n
\n
\n\n \n\n\n \n\n\n\n" }, { "path": "docs/bootstrap-toc.css", "content": "/*!\n * Bootstrap Table of Contents v0.4.1 (http://afeld.github.io/bootstrap-toc/)\n * Copyright 2015 Aidan Feldman\n * Licensed under MIT (https://github.com/afeld/bootstrap-toc/blob/gh-pages/LICENSE.md) */\n\n/* modified from https://github.com/twbs/bootstrap/blob/94b4076dd2efba9af71f0b18d4ee4b163aa9e0dd/docs/assets/css/src/docs.css#L548-L601 */\n\n/* All levels of nav */\nnav[data-toggle='toc'] .nav > li > a {\n display: block;\n padding: 4px 20px;\n font-size: 13px;\n font-weight: 500;\n color: #767676;\n}\nnav[data-toggle='toc'] .nav > li > a:hover,\nnav[data-toggle='toc'] .nav > li > a:focus {\n padding-left: 19px;\n color: #563d7c;\n text-decoration: none;\n background-color: transparent;\n border-left: 1px solid #563d7c;\n}\nnav[data-toggle='toc'] .nav > .active > a,\nnav[data-toggle='toc'] .nav > .active:hover > a,\nnav[data-toggle='toc'] .nav > .active:focus > a {\n padding-left: 18px;\n font-weight: bold;\n color: #563d7c;\n background-color: transparent;\n border-left: 2px solid #563d7c;\n}\n\n/* Nav: second level (shown on .active) */\nnav[data-toggle='toc'] .nav .nav {\n display: none; /* Hide by default, but at >768px, show it */\n padding-bottom: 10px;\n}\nnav[data-toggle='toc'] .nav .nav > li > a {\n padding-top: 1px;\n padding-bottom: 1px;\n padding-left: 30px;\n font-size: 12px;\n font-weight: normal;\n}\nnav[data-toggle='toc'] .nav .nav > li > a:hover,\nnav[data-toggle='toc'] .nav .nav > li > a:focus {\n padding-left: 29px;\n}\nnav[data-toggle='toc'] .nav .nav > .active > a,\nnav[data-toggle='toc'] .nav .nav > .active:hover > a,\nnav[data-toggle='toc'] .nav .nav > .active:focus > a {\n padding-left: 28px;\n font-weight: 500;\n}\n\n/* from https://github.com/twbs/bootstrap/blob/e38f066d8c203c3e032da0ff23cd2d6098ee2dd6/docs/assets/css/src/docs.css#L631-L634 */\nnav[data-toggle='toc'] .nav > .active > ul {\n display: block;\n}\n" }, { "path": "docs/bootstrap-toc.js", "content": "/*!\n * Bootstrap Table of Contents v0.4.1 (http://afeld.github.io/bootstrap-toc/)\n * Copyright 2015 Aidan Feldman\n * Licensed under MIT (https://github.com/afeld/bootstrap-toc/blob/gh-pages/LICENSE.md) */\n(function() {\n 'use strict';\n\n window.Toc = {\n helpers: {\n // return all matching elements in the set, or their descendants\n findOrFilter: function($el, selector) {\n // http://danielnouri.org/notes/2011/03/14/a-jquery-find-that-also-finds-the-root-element/\n // http://stackoverflow.com/a/12731439/358804\n var $descendants = $el.find(selector);\n return $el.filter(selector).add($descendants).filter(':not([data-toc-skip])');\n },\n\n generateUniqueIdBase: function(el) {\n var text = $(el).text();\n var anchor = text.trim().toLowerCase().replace(/[^A-Za-z0-9]+/g, '-');\n return anchor || el.tagName.toLowerCase();\n },\n\n generateUniqueId: function(el) {\n var anchorBase = this.generateUniqueIdBase(el);\n for (var i = 0; ; i++) {\n var anchor = anchorBase;\n if (i > 0) {\n // add suffix\n anchor += '-' + i;\n }\n // check if ID already exists\n if (!document.getElementById(anchor)) {\n return anchor;\n }\n }\n },\n\n generateAnchor: function(el) {\n if (el.id) {\n return el.id;\n } else {\n var anchor = this.generateUniqueId(el);\n el.id = anchor;\n return anchor;\n }\n },\n\n createNavList: function() {\n return $('
    ');\n },\n\n createChildNavList: function($parent) {\n var $childList = this.createNavList();\n $parent.append($childList);\n return $childList;\n },\n\n generateNavEl: function(anchor, text) {\n var $a = $('');\n $a.attr('href', '#' + anchor);\n $a.text(text);\n var $li = $('
  • ');\n $li.append($a);\n return $li;\n },\n\n generateNavItem: function(headingEl) {\n var anchor = this.generateAnchor(headingEl);\n var $heading = $(headingEl);\n var text = $heading.data('toc-text') || $heading.text();\n return this.generateNavEl(anchor, text);\n },\n\n // Find the first heading level (`

    `, then `

    `, etc.) that has more than one element. Defaults to 1 (for `

    `).\n getTopLevel: function($scope) {\n for (var i = 1; i <= 6; i++) {\n var $headings = this.findOrFilter($scope, 'h' + i);\n if ($headings.length > 1) {\n return i;\n }\n }\n\n return 1;\n },\n\n // returns the elements for the top level, and the next below it\n getHeadings: function($scope, topLevel) {\n var topSelector = 'h' + topLevel;\n\n var secondaryLevel = topLevel + 1;\n var secondarySelector = 'h' + secondaryLevel;\n\n return this.findOrFilter($scope, topSelector + ',' + secondarySelector);\n },\n\n getNavLevel: function(el) {\n return parseInt(el.tagName.charAt(1), 10);\n },\n\n populateNav: function($topContext, topLevel, $headings) {\n var $context = $topContext;\n var $prevNav;\n\n var helpers = this;\n $headings.each(function(i, el) {\n var $newNav = helpers.generateNavItem(el);\n var navLevel = helpers.getNavLevel(el);\n\n // determine the proper $context\n if (navLevel === topLevel) {\n // use top level\n $context = $topContext;\n } else if ($prevNav && $context === $topContext) {\n // create a new level of the tree and switch to it\n $context = helpers.createChildNavList($prevNav);\n } // else use the current $context\n\n $context.append($newNav);\n\n $prevNav = $newNav;\n });\n },\n\n parseOps: function(arg) {\n var opts;\n if (arg.jquery) {\n opts = {\n $nav: arg\n };\n } else {\n opts = arg;\n }\n opts.$scope = opts.$scope || $(document.body);\n return opts;\n }\n },\n\n // accepts a jQuery object, or an options object\n init: function(opts) {\n opts = this.helpers.parseOps(opts);\n\n // ensure that the data attribute is in place for styling\n opts.$nav.attr('data-toggle', 'toc');\n\n var $topContext = this.helpers.createChildNavList(opts.$nav);\n var topLevel = this.helpers.getTopLevel(opts.$scope);\n var $headings = this.helpers.getHeadings(opts.$scope, topLevel);\n this.helpers.populateNav($topContext, topLevel, $headings);\n }\n };\n\n $(function() {\n $('nav[data-toggle=\"toc\"]').each(function(i, el) {\n var $nav = $(el);\n Toc.init($nav);\n });\n });\n})();\n" }, { "path": "docs/docsearch.css", "content": "/* Docsearch -------------------------------------------------------------- */\n/*\n Source: https://github.com/algolia/docsearch/\n License: MIT\n*/\n\n.algolia-autocomplete {\n display: block;\n -webkit-box-flex: 1;\n -ms-flex: 1;\n flex: 1\n}\n\n.algolia-autocomplete .ds-dropdown-menu {\n width: 100%;\n min-width: none;\n max-width: none;\n padding: .75rem 0;\n background-color: #fff;\n background-clip: padding-box;\n border: 1px solid rgba(0, 0, 0, .1);\n box-shadow: 0 .5rem 1rem rgba(0, 0, 0, .175);\n}\n\n@media (min-width:768px) {\n .algolia-autocomplete .ds-dropdown-menu {\n width: 175%\n }\n}\n\n.algolia-autocomplete .ds-dropdown-menu::before {\n display: none\n}\n\n.algolia-autocomplete .ds-dropdown-menu [class^=ds-dataset-] {\n padding: 0;\n background-color: rgb(255,255,255);\n border: 0;\n max-height: 80vh;\n}\n\n.algolia-autocomplete .ds-dropdown-menu .ds-suggestions {\n margin-top: 0\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion {\n padding: 0;\n overflow: visible\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--category-header {\n padding: .125rem 1rem;\n margin-top: 0;\n font-size: 1.3em;\n font-weight: 500;\n color: #00008B;\n border-bottom: 0\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--wrapper {\n float: none;\n padding-top: 0\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column {\n float: none;\n width: auto;\n padding: 0;\n text-align: left\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--content {\n float: none;\n width: auto;\n padding: 0\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--content::before {\n display: none\n}\n\n.algolia-autocomplete .ds-suggestion:not(:first-child) .algolia-docsearch-suggestion--category-header {\n padding-top: .75rem;\n margin-top: .75rem;\n border-top: 1px solid rgba(0, 0, 0, .1)\n}\n\n.algolia-autocomplete .ds-suggestion .algolia-docsearch-suggestion--subcategory-column {\n display: block;\n padding: .1rem 1rem;\n margin-bottom: 0.1;\n font-size: 1.0em;\n font-weight: 400\n /* display: none */\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--title {\n display: block;\n padding: .25rem 1rem;\n margin-bottom: 0;\n font-size: 0.9em;\n font-weight: 400\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--text {\n padding: 0 1rem .5rem;\n margin-top: -.25rem;\n font-size: 0.8em;\n font-weight: 400;\n line-height: 1.25\n}\n\n.algolia-autocomplete .algolia-docsearch-footer {\n width: 110px;\n height: 20px;\n z-index: 3;\n margin-top: 10.66667px;\n float: right;\n font-size: 0;\n line-height: 0;\n}\n\n.algolia-autocomplete .algolia-docsearch-footer--logo {\n background-image: url(\"data:image/svg+xml;utf8,\");\n background-repeat: no-repeat;\n background-position: 50%;\n background-size: 100%;\n overflow: hidden;\n text-indent: -9000px;\n width: 100%;\n height: 100%;\n display: block;\n transform: translate(-8px);\n}\n\n.algolia-autocomplete .algolia-docsearch-suggestion--highlight {\n color: #FF8C00;\n background: rgba(232, 189, 54, 0.1)\n}\n\n\n.algolia-autocomplete .algolia-docsearch-suggestion--text .algolia-docsearch-suggestion--highlight {\n box-shadow: inset 0 -2px 0 0 rgba(105, 105, 105, .5)\n}\n\n.algolia-autocomplete .ds-suggestion.ds-cursor .algolia-docsearch-suggestion--content {\n background-color: rgba(192, 192, 192, .15)\n}\n" }, { "path": "docs/docsearch.js", "content": "$(function() {\n\n // register a handler to move the focus to the search bar\n // upon pressing shift + \"/\" (i.e. \"?\")\n $(document).on('keydown', function(e) {\n if (e.shiftKey && e.keyCode == 191) {\n e.preventDefault();\n $(\"#search-input\").focus();\n }\n });\n\n $(document).ready(function() {\n // do keyword highlighting\n /* modified from https://jsfiddle.net/julmot/bL6bb5oo/ */\n var mark = function() {\n\n var referrer = document.URL ;\n var paramKey = \"q\" ;\n\n if (referrer.indexOf(\"?\") !== -1) {\n var qs = referrer.substr(referrer.indexOf('?') + 1);\n var qs_noanchor = qs.split('#')[0];\n var qsa = qs_noanchor.split('&');\n var keyword = \"\";\n\n for (var i = 0; i < qsa.length; i++) {\n var currentParam = qsa[i].split('=');\n\n if (currentParam.length !== 2) {\n continue;\n }\n\n if (currentParam[0] == paramKey) {\n keyword = decodeURIComponent(currentParam[1].replace(/\\+/g, \"%20\"));\n }\n }\n\n if (keyword !== \"\") {\n $(\".contents\").unmark({\n done: function() {\n $(\".contents\").mark(keyword);\n }\n });\n }\n }\n };\n\n mark();\n });\n});\n\n/* Search term highlighting ------------------------------*/\n\nfunction matchedWords(hit) {\n var words = [];\n\n var hierarchy = hit._highlightResult.hierarchy;\n // loop to fetch from lvl0, lvl1, etc.\n for (var idx in hierarchy) {\n words = words.concat(hierarchy[idx].matchedWords);\n }\n\n var content = hit._highlightResult.content;\n if (content) {\n words = words.concat(content.matchedWords);\n }\n\n // return unique words\n var words_uniq = [...new Set(words)];\n return words_uniq;\n}\n\nfunction updateHitURL(hit) {\n\n var words = matchedWords(hit);\n var url = \"\";\n\n if (hit.anchor) {\n url = hit.url_without_anchor + '?q=' + escape(words.join(\" \")) + '#' + hit.anchor;\n } else {\n url = hit.url + '?q=' + escape(words.join(\" \"));\n }\n\n return url;\n}\n" }, { "path": "docs/index.html", "content": "\n\n\n\n\n\n\nTools for Computation on Batch Systems • batchtools\n\n\n\n\n\n\n\n\n\n\n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n\n
    \n\n
    \n\n
    \n\n\n \n\n
    \n
    \n
    \n

    \nbatchtools

    \n\n

    As a successor of the packages BatchJobs and BatchExperiments, batchtools provides a parallel implementation of Map for high performance computing systems managed by schedulers like Slurm, Sun Grid Engine, OpenLava, TORQUE/OpenPBS, Load Sharing Facility (LSF) or Docker Swarm (see the setup section in the vignette).

    \n

    Main features:

    \n
      \n
    • Convenience: All relevant batch system operations (submitting, listing, killing) are either handled internally or abstracted via simple R functions
      • \n
    • Portability: With a well-defined interface, the source is independent from the underlying batch system - prototype locally, deploy on any high performance cluster
      • \n
    • Reproducibility: Every computational part has an associated seed stored in a data base which ensures reproducibility even when the underlying batch system changes
      • \n
    • Abstraction: The code layers for algorithms, experiment definitions and execution are cleanly separated and allow to write readable and maintainable code to manage large scale computer experiments
      • \n
    \n
    \n

    \nInstallation

    \n

    Install the stable version from CRAN:

    \n
    install.packages(\"batchtools\")
    \n

    For the development version, use devtools:

    \n
    devtools::install_github(\"mllg/batchtools\")
    \n

    Next, you need to setup batchtools for your HPC (it will run sequentially otherwise). See the vignette for instructions.

    \n
    \n
    \n

    \nWhy batchtools?

    \n

    The development of BatchJobs and BatchExperiments is discontinued for the following reasons:

    \n
      \n
    • Maintainability: The packages BatchJobs and BatchExperiments are tightly connected which makes maintenance difficult. Changes have to be synchronized and tested against the current CRAN versions for compatibility. Furthermore, BatchExperiments violates CRAN policies by calling internal functions of BatchJobs.
      • \n
    • Data base issues: Although we invested weeks to mitigate issues with locks of the SQLite data base or file system (staged queries, file system timeouts, …), BatchJobs kept working unreliable on some systems with high latency under certain conditions. This made BatchJobs unusable for many users.
      • \n
    \n

    BatchJobs and BatchExperiments will remain on CRAN, but new features are unlikely to be ported back. The vignette contains a section comparing the packages.

    \n
    \n
    \n

    \nResources

    \n\n
    \n
    \n

    \nCitation

    \n

    Please cite the JOSS paper using the following BibTeX entry:

    \n
    @article{,\n  doi = {10.21105/joss.00135},\n  url = {https://doi.org/10.21105/joss.00135},\n  year  = {2017},\n  month = {feb},\n  publisher = {The Open Journal},\n  volume = {2},\n  number = {10},\n  author = {Michel Lang and Bernd Bischl and Dirk Surmann},\n  title = {batchtools: Tools for R to work on batch systems},\n  journal = {The Journal of Open Source Software}\n}
    \n
    \n
    \n

    \nRelated Software

    \n
      \n
    • The High Performance Computing Task View lists the most relevant packages for scientific computing with R.
      • \n
    • \nclustermq is a similar approach which also supports multiple schedulers. Uses the ZeroMQ network protocol for communication, and shines if you have millions of fast jobs.
      • \n
    • \nbatch assists in splitting and submitting jobs to LSF and MOSIX clusters.
      • \n
    • \nflowr supports LSF, Slurm, TORQUE and Moab and provides a scatter-gather approach to define computational jobs.
      • \n
    • \nfuture.batchtools implements batchtools as backend for future.
      • \n
    • \ndoFuture together with future.batchtools connects batchtools to foreach.
      • \n
    • \ndrake uses graphs to define computational jobs. batchtools is used as a backend via future.batchtools.
      • \n
    \n
    \n
    \n

    \nContributing to batchtools

    \n

    This R package is licensed under the LGPL-3. If you encounter problems using this software (lack of documentation, misleading or wrong documentation, unexpected behaviour, bugs, …) or just want to suggest features, please open an issue in the issue tracker. Pull requests are welcome and will be included at the discretion of the author. If you have customized a template file for your (larger) computing site, please share it: fork the repository, place your template in inst/templates and send a pull request.

    \n
    \n
    \n
    \n\n
    \n
    \n

    Links

    \n\n
    \n
    \n

    License

    \n\n
    \n
    \n

    Citation

    \n\n
    \n
    \n

    Developers

    \n
      \n
    • Michel Lang
      Maintainer, author
      • \n
    • Bernd Bischl
      Author
      • \n
    • All authors...
      • \n
    \n
    \n\n
    \n

    Dev status

    \n
      \n
    • \"JOSS
      • \n
    • \"CRAN
      • \n
    • \"Build
      • \n
    • \"Build
      • \n
    \n
    \n
    \n
    \n\n\n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n" }, { "path": "docs/news/index.html", "content": "\n\n\n \n \n\n\n\nChangelog • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Changelog

    \n Source: NEWS.md\n
    \n\n
    \n

    \nbatchtools 0.9.14 Unreleased \n

    \n
      \n
    • \nbatchMap() now supports unnamed more.args.
      • \n
    • Exports are now assigned with delayedAssign().
      • \n
    • Fix an option in the LSF template.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.13 2020-03-19 \n

    \n
      \n
    • Maintenance release for R-4.0.0.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.12 2020-01-10 \n

    \n
      \n
    • Moved data.table from Depends to Imports. User scripts might need to explicitly attach data.table via library() now.
      • \n
    • Fixes for ClusterFunctionsMulticore.
      • \n
    • Removed a workaround for system2() for R-devel (to be released as R-4.0.0).
      • \n
    • New configuration option compress to select the compression algorithm (passed down to saveRDS()).
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.11 2018-08-16 \n

    \n
      \n
    • Removed deprecated function chunkIds().
      • \n
    • New default for argument fs.timeout in the cluster function constructor is 0 (was NA before).
      • \n
    • Fixed a unit test for OSX.
      • \n
    • Improved stability and documentation.
      • \n
    • Fixed memory usage calculation.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.10 2018-05-19 \n

    \n
      \n
    • Exported functions findConfFile() and findTemplateFile().
      • \n
    • Dropped support for providing a template file directly as string. A valid file is now always required.
      • \n
    • Fixed writing to TMPDIR instead of the R session’s temporary directory.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.9 2018-05-14 \n

    \n
      \n
    • RDS files are explicitly stored in version 2 to ensure backward compatibility with R versions prior to 3.5.0.
      • \n
    • Package fs is now used internally for all file system operations.
      • \n
    • Support for per-site configuration files and templates to be set up by system administrators.
      • \n
    • The print of getStatus() now includes a time stamp.
      • \n
    • \nchunk() now optionally shuffles the ids before chunking.
      • \n
    • Support for setting per-job resources in submitJobs().
      • \n
    • Example templates now include resources for blas.threads and omp.threads.
      • \n
    • Some bug fixes regarding read-only registries.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.8 2017-12-15 \n

    \n
      \n
    • Renamed column “memory” in the status table to “mem.used” to avoid name clashes with the resource specification.
      • \n
    • Exported function assertRegistry().
      • \n
    • New function unwrap() as alias to flatten(). The latter causes a name clash with package purrr and will be deprecated in a future version.
      • \n
    • Registries now contain a unique hash which is updated each time the registry is altered. Can be utilized to invalidate caches, e.g. the cache of knitr.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.7 2017-11-15 \n

    \n
      \n
    • Added a workaround for a test to be compatible with testthat v2.0.0.
      • \n
    • Better and more customizable handling of expired jobs in waitForJobs().
      • \n
    • Package foreach is now supported for nested parallelization as an alternative to parallelMap.
      • \n
    • Depreciated argument flatten has been removed.
      • \n
    • New helper function flatten() to manually unnest/unwrap lists in data frames.
      • \n
    • Removed functions getProblemIds() and getAlgorithmIds(). Instead, you can just access reg$problems or reg$algorithms, respectively.
      • \n
    • The number of the maximum concurrent jobs can now also be controlled via setting resources.
      • \n
    • Internal data base changes to speed up some operations. Old registries are updated on first load by loadRegistry().
      • \n
    • Fixed a bug where the sleep mechanism between queries was not working.
      • \n
    • Fixed a bug where submit errors on SLURM and TORQUE were not detected as temporary.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.6 2017-09-06 \n

    \n
      \n
    • Fixed a bug where the wrong problem was retrieved from the cache. This was only triggered for chunked jobs in combination with an ExperimentRegistry.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.5 2017-08-18 \n

    \n
      \n
    • Added a missing routine to upgrade registries created with batchtools prior to v0.9.3.
      • \n
    • Fixed a bug where the registry could not be synced if jobs failed during initialization (#135).
      • \n
    • The sleep duration for waitForJobs() and submitJobs() can now be set via the configuration file.
      • \n
    • A new heuristic will try to detect if the registry has been altered by a simultaneously running R session. If this is detected, the registry in the current session will be set to a read-only state.
      • \n
    • \nwaitForJobs() has been reworked to allow control over the heuristic to detect expired jobs. Jobs are treated as expired if they have been submitted but are not detected on the system for expire.after iterations (default 3 iterations, before 1 iteration).
      • \n
    • New argument writeable for loadRegistry() to allow loading registries explicitly as read-only.
      • \n
    • Removed argument update.paths from loadRegistry(). Paths are always updated, but the registry on the file system remains unchanged unless loaded in read-write mode.
      • \n
    • \nClusterFunctionsSlurm now come with an experimental nodename argument. If set, all communication with the master is handled via SSH which effectively allows you to submit jobs from your local machine instead of the head node. Note that mounting the file system (e.g., via SSHFS) is mandatory.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.4 2017-08-07 \n

    \n
      \n
    • Fixed handling of file.dir with special chars like whitespace.
      • \n
    • All backward slashes will now be converted to forward slashes on windows.
      • \n
    • Fixed order of arguments in findExperiments() (argument ids is now first).
      • \n
    • Removed code to upgrade registries created with versions prior to v0.9.0 (first CRAN release).
      • \n
    • \naddExperiments() now warns if a design is passed as data.frame with factor columns and stringsAsFactors is TRUE.
      • \n
    • Added functions setJobNames() and getJobNames() to control the name of jobs on batch systems. Templates should be adapted to use job.name instead of job.hash for naming.
      • \n
    • Argument flatten of getJobResources(), getJobPars() and getJobTable() is deprecated and will be removed. Future versions of the functions will behave like flatten is set to FALSE explicitly. Single resources/parameters must be extracted manually (or with tidyr::unnest()).
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.3 2017-04-21 \n

    \n
      \n
    • Running jobs now are also included while querying for status “started”. This affects findStarted(), findNotStarted() and getStatus().
      • \n
    • \nfindExperiments() now performs an exact string match (instead of matching substrings) for patterns specified via prob.name and algo.name. For substring matching, use prob.pattern or algo.pattern, respectively.
      • \n
    • Changed arguments for reduceResultsDataTable()\n
        \n
      • Removed fill, now is always TRUE\n
        • \n
      • Introduced flatten to control if the result should be represented as a column of lists or flattened as separate columns. Defaults to a backward-compatible heuristic, similar to getJobPars.
        • \n
      \n
      • \n
    • Improved heuristic to lookup template files. Templates shipped with the package can now be used by providing just the file name (w/o extension).
      • \n
    • Updated CITATION
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.2 2017-02-20 \n

    \n
      \n
    • Full support for array jobs on Slurm and TORQUE.
      • \n
    • Array jobs have been disabled for SGE and LSF (due to missing information about the output format) but will be re-enable in a future release. Note that the variable n.array.jobs has been removed from JobCollection in favor of the new variable array.jobs (logical).
      • \n
    • \nfindExperiments() now has two additional arguments to match using regular expressions. The possibility to prefix a string with “~” to enable regular expression matching has been removed.
      • \n
    • New function batchReduce().
      • \n
    • New function estimateRuntimes().
      • \n
    • New function removeRegistry().
      • \n
    • Missing result files are now handled more consistently, raising an exception in its defaults if the result is not available. The argument missing.val has been added to reduceResultsList() and reduceResultsDataTable() and removed from loadResult() and batchMapResults().
      • \n
    • Timestamps are now stored with sub-second accuracy.
      • \n
    • Renamed Torque to TORQUE. This especially affects the constructor makeClusterFunctionsTorque which now must be called via makeClusterFunctionsTORQUE()\n
      • \n
    • \nchunkIds() has been deprecated. Use chunk(), lpt() or binpack() instead.
      • \n
    • Fixed listing of jobs for ClusterFunctionsLSF and ClusterFunctionsOpenLava (thanks to @phaverty).
      • \n
    • Job hashes are now prefixed with the literal string ‘job’ to ensure they start with a letter as required by some SGE systems.
      • \n
    • Fixed handling of NULL results in reduceResultsList()\n
      • \n
    • Fixed key lookup heuristic join functions.
      • \n
    • Fixed a bug where getJobTable() returned difftimes with the wrong unit (e.g., in minutes instead of seconds).
      • \n
    • Deactivated swap allocation for ClusterFunctionsDocker.
      • \n
    • The package is now more patient while communicating with the scheduler or file system by using a timeout-based approach. This should make the package more reliable and robust under heavy load.
      • \n
    \n
    \n
    \n

    \nbatchtools 0.9.0 2016-11-08 \n

    \n

    Initial CRAN release. See the vignette for a brief comparison with BatchJobs/BatchExperiments.

    \n
    \n
    \n\n
    \n \n
    \n\n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/pkgdown.css", "content": "/* Sticky footer */\n\n/**\n * Basic idea: https://philipwalton.github.io/solved-by-flexbox/demos/sticky-footer/\n * Details: https://github.com/philipwalton/solved-by-flexbox/blob/master/assets/css/components/site.css\n *\n * .Site -> body > .container\n * .Site-content -> body > .container .row\n * .footer -> footer\n *\n * Key idea seems to be to ensure that .container and __all its parents__\n * have height set to 100%\n *\n */\n\nhtml, body {\n height: 100%;\n}\n\nbody {\n position: relative;\n}\n\nbody > .container {\n display: flex;\n height: 100%;\n flex-direction: column;\n}\n\nbody > .container .row {\n flex: 1 0 auto;\n}\n\nfooter {\n margin-top: 45px;\n padding: 35px 0 36px;\n border-top: 1px solid #e5e5e5;\n color: #666;\n display: flex;\n flex-shrink: 0;\n}\nfooter p {\n margin-bottom: 0;\n}\nfooter div {\n flex: 1;\n}\nfooter .pkgdown {\n text-align: right;\n}\nfooter p {\n margin-bottom: 0;\n}\n\nimg.icon {\n float: right;\n}\n\nimg {\n max-width: 100%;\n}\n\n/* Fix bug in bootstrap (only seen in firefox) */\nsummary {\n display: list-item;\n}\n\n/* Typographic tweaking ---------------------------------*/\n\n.contents .page-header {\n margin-top: calc(-60px + 1em);\n}\n\ndd {\n margin-left: 3em;\n}\n\n/* Section anchors ---------------------------------*/\n\na.anchor {\n margin-left: -30px;\n display:inline-block;\n width: 30px;\n height: 30px;\n visibility: hidden;\n\n background-image: url(./link.svg);\n background-repeat: no-repeat;\n background-size: 20px 20px;\n background-position: center center;\n}\n\n.hasAnchor:hover a.anchor {\n visibility: visible;\n}\n\n@media (max-width: 767px) {\n .hasAnchor:hover a.anchor {\n visibility: hidden;\n }\n}\n\n\n/* Fixes for fixed navbar --------------------------*/\n\n.contents h1, .contents h2, .contents h3, .contents h4 {\n padding-top: 60px;\n margin-top: -40px;\n}\n\n/* Navbar submenu --------------------------*/\n\n.dropdown-submenu {\n position: relative;\n}\n\n.dropdown-submenu>.dropdown-menu {\n top: 0;\n left: 100%;\n margin-top: -6px;\n margin-left: -1px;\n border-radius: 0 6px 6px 6px;\n}\n\n.dropdown-submenu:hover>.dropdown-menu {\n display: block;\n}\n\n.dropdown-submenu>a:after {\n display: block;\n content: \" \";\n float: right;\n width: 0;\n height: 0;\n border-color: transparent;\n border-style: solid;\n border-width: 5px 0 5px 5px;\n border-left-color: #cccccc;\n margin-top: 5px;\n margin-right: -10px;\n}\n\n.dropdown-submenu:hover>a:after {\n border-left-color: #ffffff;\n}\n\n.dropdown-submenu.pull-left {\n float: none;\n}\n\n.dropdown-submenu.pull-left>.dropdown-menu {\n left: -100%;\n margin-left: 10px;\n border-radius: 6px 0 6px 6px;\n}\n\n/* Sidebar --------------------------*/\n\n#pkgdown-sidebar {\n margin-top: 30px;\n position: -webkit-sticky;\n position: sticky;\n top: 70px;\n}\n\n#pkgdown-sidebar h2 {\n font-size: 1.5em;\n margin-top: 1em;\n}\n\n#pkgdown-sidebar h2:first-child {\n margin-top: 0;\n}\n\n#pkgdown-sidebar .list-unstyled li {\n margin-bottom: 0.5em;\n}\n\n/* bootstrap-toc tweaks ------------------------------------------------------*/\n\n/* All levels of nav */\n\nnav[data-toggle='toc'] .nav > li > a {\n padding: 4px 20px 4px 6px;\n font-size: 1.5rem;\n font-weight: 400;\n color: inherit;\n}\n\nnav[data-toggle='toc'] .nav > li > a:hover,\nnav[data-toggle='toc'] .nav > li > a:focus {\n padding-left: 5px;\n color: inherit;\n border-left: 1px solid #878787;\n}\n\nnav[data-toggle='toc'] .nav > .active > a,\nnav[data-toggle='toc'] .nav > .active:hover > a,\nnav[data-toggle='toc'] .nav > .active:focus > a {\n padding-left: 5px;\n font-size: 1.5rem;\n font-weight: 400;\n color: inherit;\n border-left: 2px solid #878787;\n}\n\n/* Nav: second level (shown on .active) */\n\nnav[data-toggle='toc'] .nav .nav {\n display: none; /* Hide by default, but at >768px, show it */\n padding-bottom: 10px;\n}\n\nnav[data-toggle='toc'] .nav .nav > li > a {\n padding-left: 16px;\n font-size: 1.35rem;\n}\n\nnav[data-toggle='toc'] .nav .nav > li > a:hover,\nnav[data-toggle='toc'] .nav .nav > li > a:focus {\n padding-left: 15px;\n}\n\nnav[data-toggle='toc'] .nav .nav > .active > a,\nnav[data-toggle='toc'] .nav .nav > .active:hover > a,\nnav[data-toggle='toc'] .nav .nav > .active:focus > a {\n padding-left: 15px;\n font-weight: 500;\n font-size: 1.35rem;\n}\n\n/* orcid ------------------------------------------------------------------- */\n\n.orcid {\n font-size: 16px;\n color: #A6CE39;\n /* margins are required by official ORCID trademark and display guidelines */\n margin-left:4px;\n margin-right:4px;\n vertical-align: middle;\n}\n\n/* Reference index & topics ----------------------------------------------- */\n\n.ref-index th {font-weight: normal;}\n\n.ref-index td {vertical-align: top; min-width: 100px}\n.ref-index .icon {width: 40px;}\n.ref-index .alias {width: 40%;}\n.ref-index-icons .alias {width: calc(40% - 40px);}\n.ref-index .title {width: 60%;}\n\n.ref-arguments th {text-align: right; padding-right: 10px;}\n.ref-arguments th, .ref-arguments td {vertical-align: top; min-width: 100px}\n.ref-arguments .name {width: 20%;}\n.ref-arguments .desc {width: 80%;}\n\n/* Nice scrolling for wide elements --------------------------------------- */\n\ntable {\n display: block;\n overflow: auto;\n}\n\n/* Syntax highlighting ---------------------------------------------------- */\n\npre {\n word-wrap: normal;\n word-break: normal;\n border: 1px solid #eee;\n}\n\npre, code {\n background-color: #f8f8f8;\n color: #333;\n}\n\npre code {\n overflow: auto;\n word-wrap: normal;\n white-space: pre;\n}\n\npre .img {\n margin: 5px 0;\n}\n\npre .img img {\n background-color: #fff;\n display: block;\n height: auto;\n}\n\ncode a, pre a {\n color: #375f84;\n}\n\na.sourceLine:hover {\n text-decoration: none;\n}\n\n.fl {color: #1514b5;}\n.fu {color: #000000;} /* function */\n.ch,.st {color: #036a07;} /* string */\n.kw {color: #264D66;} /* keyword */\n.co {color: #888888;} /* comment */\n\n.message { color: black; font-weight: bolder;}\n.error { color: orange; font-weight: bolder;}\n.warning { color: #6A0366; font-weight: bolder;}\n\n/* Clipboard --------------------------*/\n\n.hasCopyButton {\n position: relative;\n}\n\n.btn-copy-ex {\n position: absolute;\n right: 0;\n top: 0;\n visibility: hidden;\n}\n\n.hasCopyButton:hover button.btn-copy-ex {\n visibility: visible;\n}\n\n/* headroom.js ------------------------ */\n\n.headroom {\n will-change: transform;\n transition: transform 200ms linear;\n}\n.headroom--pinned {\n transform: translateY(0%);\n}\n.headroom--unpinned {\n transform: translateY(-100%);\n}\n\n/* mark.js ----------------------------*/\n\nmark {\n background-color: rgba(255, 255, 51, 0.5);\n border-bottom: 2px solid rgba(255, 153, 51, 0.3);\n padding: 1px;\n}\n\n/* vertical spacing after htmlwidgets */\n.html-widget {\n margin-bottom: 10px;\n}\n\n/* fontawesome ------------------------ */\n\n.fab {\n font-family: \"Font Awesome 5 Brands\" !important;\n}\n\n/* don't display links in code chunks when printing */\n/* source: https://stackoverflow.com/a/10781533 */\n@media print {\n code a:link:after, code a:visited:after {\n content: \"\";\n }\n}\n" }, { "path": "docs/pkgdown.js", "content": "/* http://gregfranko.com/blog/jquery-best-practices/ */\n(function($) {\n $(function() {\n\n $('.navbar-fixed-top').headroom();\n\n $('body').css('padding-top', $('.navbar').height() + 10);\n $(window).resize(function(){\n $('body').css('padding-top', $('.navbar').height() + 10);\n });\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n\n var cur_path = paths(location.pathname);\n var links = $(\"#navbar ul li a\");\n var max_length = -1;\n var pos = -1;\n for (var i = 0; i < links.length; i++) {\n if (links[i].getAttribute(\"href\") === \"#\")\n continue;\n // Ignore external links\n if (links[i].host !== location.host)\n continue;\n\n var nav_path = paths(links[i].pathname);\n\n var length = prefix_length(nav_path, cur_path);\n if (length > max_length) {\n max_length = length;\n pos = i;\n }\n }\n\n // Add class to parent
  • , and enclosing
  • if in dropdown\n if (pos >= 0) {\n var menu_anchor = $(links[pos]);\n menu_anchor.parent().addClass(\"active\");\n menu_anchor.closest(\"li.dropdown\").addClass(\"active\");\n }\n });\n\n function paths(pathname) {\n var pieces = pathname.split(\"/\");\n pieces.shift(); // always starts with /\n\n var end = pieces[pieces.length - 1];\n if (end === \"index.html\" || end === \"\")\n pieces.pop();\n return(pieces);\n }\n\n // Returns -1 if not found\n function prefix_length(needle, haystack) {\n if (needle.length > haystack.length)\n return(-1);\n\n // Special case for length-0 haystack, since for loop won't run\n if (haystack.length === 0) {\n return(needle.length === 0 ? 0 : -1);\n }\n\n for (var i = 0; i < haystack.length; i++) {\n if (needle[i] != haystack[i])\n return(i);\n }\n\n return(haystack.length);\n }\n\n /* Clipboard --------------------------*/\n\n function changeTooltipMessage(element, msg) {\n var tooltipOriginalTitle=element.getAttribute('data-original-title');\n element.setAttribute('data-original-title', msg);\n $(element).tooltip('show');\n element.setAttribute('data-original-title', tooltipOriginalTitle);\n }\n\n if(ClipboardJS.isSupported()) {\n $(document).ready(function() {\n var copyButton = \"\";\n\n $(\".examples, div.sourceCode\").addClass(\"hasCopyButton\");\n\n // Insert copy buttons:\n $(copyButton).prependTo(\".hasCopyButton\");\n\n // Initialize tooltips:\n $('.btn-copy-ex').tooltip({container: 'body'});\n\n // Initialize clipboard:\n var clipboardBtnCopies = new ClipboardJS('[data-clipboard-copy]', {\n text: function(trigger) {\n return trigger.parentNode.textContent;\n }\n });\n\n clipboardBtnCopies.on('success', function(e) {\n changeTooltipMessage(e.trigger, 'Copied!');\n e.clearSelection();\n });\n\n clipboardBtnCopies.on('error', function() {\n changeTooltipMessage(e.trigger,'Press Ctrl+C or Command+C to copy');\n });\n });\n }\n})(window.jQuery || window.$)\n" }, { "path": "docs/pkgdown.yml", "content": "pandoc: 2.11.0.2\npkgdown: 1.6.1\npkgdown_sha: ~\narticles:\n batchtools: batchtools.html\nlast_built: 2020-10-21T07:39Z\n\n" }, { "path": "docs/reference/JobCollection.html", "content": "\n\n\n \n \n\n\n\nJobCollection Constructor — makeJobCollection • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    JobCollection Constructor

    \n Source: R/JobCollection.R\n
    JobCollection.Rd
    \n
    \n\n
    \n

    makeJobCollection takes multiple job ids and creates an object of class “JobCollection” which holds all\nnecessary information for the calculation with doJobCollection. It is implemented as an environment\nwith the following variables:

    \n
    file.dir

    file.dir of the Registry.

    \n
    work.dir:

    work.dir of the Registry.

    \n
    job.hash

    Unique identifier of the job. Used to create names on the file system.

    \n
    jobs

    data.table holding individual job information. See examples.

    \n
    log.file

    Location of the designated log file for this job.

    \n
    resources:

    Named list of of specified computational resources.

    \n
    uri

    Location of the job description file (saved with link[base]{saveRDS} on the file system.

    \n
    seed

    integer(1) Seed of the Registry.

    \n
    packages

    character with required packages to load via require.

    \n
    namespaces

    codecharacter with required packages to load via requireNamespace.

    \n
    source

    character with list of files to source before execution.

    \n
    load

    character with list of files to load before execution.

    \n
    array.var

    character(1) of the array environment variable specified by the cluster functions.

    \n
    array.jobs

    logical(1) signaling if jobs were submitted using chunks.as.arrayjobs.

    \n\n

    If your ClusterFunctions uses a template, brew will be executed in the environment of such\na collection. Thus all variables available inside the job can be used in the template.

    \n
    \n\n 
    makeJobCollection(ids = NULL, resources = list(), reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    resources

    [list]
    \nNamed list of resources. Default is list().

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [JobCollection].

    \n

    See also

    \n\n

    Other JobCollection: \ndoJobCollection()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE, packages = \"methods\")\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(identity, 1:5, reg = tmp)\n
    #> Adding 5 jobs ...
    \n# resources are usually set in submitJobs()\njc = makeJobCollection(1:3, resources = list(foo = \"bar\"), reg = tmp)\nls(jc)\n
    #>  [1] \"array.jobs\" \"array.var\"  \"compress\"   \"file.dir\"   \"job.hash\"  \n#>  [6] \"job.name\"   \"jobs\"       \"load\"       \"log.file\"   \"namespaces\"\n#> [11] \"packages\"   \"resources\"  \"seed\"       \"source\"     \"uri\"       \n#> [16] \"work.dir\"  
    jc$resources\n
    #> $foo\n#> [1] \"bar\"\n#>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/JobExperiment.html", "content": "\n\n\n \n \n\n\n\nJobs and Experiments — makeJob • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Jobs and Experiments

    \n Source: R/Job.R\n
    JobExperiment.Rd
    \n
    \n\n
    \n

    Jobs and Experiments are abstract objects which hold all information necessary to execute a single computational\njob for a Registry or ExperimentRegistry, respectively.

    \n

    They can be created using the constructor makeJob which takes a single job id.\nJobs and Experiments are passed to reduce functions like reduceResults.\nFurthermore, Experiments can be used in the functions of the Problem and Algorithm.\nJobs and Experiments hold these information:

    \n
    job.id

    Job ID as integer.

    \n
    pars

    Job parameters as named list.\n For ExperimentRegistry, the parameters are divided into the sublists “prob.pars” and “algo.pars”.

    \n
    seed

    Seed which is set via doJobCollection as scalar integer.

    \n
    resources

    Computational resources which were set for this job as named list.

    \n
    external.dir

    Path to a directory which is created exclusively for this job. You can store external files here.\n Directory is persistent between multiple restarts of the job and can be cleaned by calling resetJobs.

    \n
    fun

    Job only: User function passed to batchMap.

    \n
    prob.name

    Experiments only: Problem id.

    \n
    algo.name

    Experiments only: Algorithm id.

    \n
    problem

    Experiments only: Problem.

    \n
    instance

    Experiments only: Problem instance.

    \n
    algorithm

    Experiments only: Algorithm.

    \n
    repl

    Experiments only: Replication number.

    \n\n
    \n\n

    Note that the slots “pars”, “fun”, “algorithm” and “problem”\nlazy-load required files from the file system and construct the object on the first access.\nThe realizations are cached for all slots except “instance” (which might be stochastic).

    \n

    Jobs and Experiments can be executed manually with execJob.

    \n
    \n\n 
    makeJob(id, reader = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    id

    [integer(1) or data.table]
    \nSingle integer to specify the job or a data.table with column job.id\nand exactly one row.

    reader

    [RDSReader | NULL]
    \nReader object to retrieve files. Used internally to cache reading from the file system.\nThe default (NULL) does not make use of caching.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [Job | Experiment].

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(function(x, y) x + y, x = 1:2, more.args = list(y = 99), reg = tmp)\n
    #> Adding 2 jobs ...
    submitJobs(resources = list(foo = \"bar\"), reg = tmp)\n
    #> Submitting 2 jobs in 2 chunks using cluster functions 'Interactive' ...
    job = makeJob(1, reg = tmp)\nprint(job)\n
    #> <Job>\n#>   Inherits from: <BaseJob>\n#>   Public:\n#>     external.dir: active binding\n#>     file.dir: /tmp/batchtools-example/reg\n#>     fun: active binding\n#>     id: 1\n#>     initialize: function (file.dir, reader, id, job.pars, seed, resources) \n#>     job.id: active binding\n#>     job.pars: list\n#>     pars: active binding\n#>     reader: RDSReader, R6\n#>     resources: list\n#>     seed: 15284
    \n# Get the parameters:\njob$pars\n
    #> $x\n#> [1] 1\n#> \n#> $y\n#> [1] 99\n#> 
    \n# Get the job resources:\njob$resources\n
    #> $foo\n#> [1] \"bar\"\n#> 
    \n# Execute the job locally:\nexecJob(job)\n
    #> ### [bt]: Setting seed to 15284 ...
    #> [1] 100
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/JobNames.html", "content": "\n\n\n \n \n\n\n\nSet and Retrieve Job Names — JobNames • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Set and Retrieve Job Names

    \n Source: R/JobNames.R\n
    JobNames.Rd
    \n
    \n\n
    \n

    Set custom names for jobs. These are passed to the template as ‘job.name’.\nIf no custom name is set (or any of the job names of the chunk is missing),\nthe job hash is used as job name.\nIndividual job names can be accessed via jobs$job.name.

    \n
    \n\n 
    setJobNames(ids = NULL, names, reg = getDefaultRegistry())\n\ngetJobNames(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    names

    [character]
    \nCharacter vector of the same length as provided ids.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    setJobNames returns NULL invisibly, getJobTable\n returns a data.table with columns job.id and job.name.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    ids = batchMap(identity, 1:10, reg = tmp)\n
    #> Adding 10 jobs ...
    setJobNames(ids, letters[1:nrow(ids)], reg = tmp)\ngetJobNames(reg = tmp)\n
    #>     job.id job.name\n#>  1:      1        a\n#>  2:      2        b\n#>  3:      3        c\n#>  4:      4        d\n#>  5:      5        e\n#>  6:      6        f\n#>  7:      7        g\n#>  8:      8        h\n#>  9:      9        i\n#> 10:     10        j
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/JoinTables.html", "content": "\n\n\n \n \n\n\n\nInner, Left, Right, Outer, Semi and Anti Join for Data Tables — JoinTables • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Inner, Left, Right, Outer, Semi and Anti Join for Data Tables

    \n Source: R/Joins.R\n
    JoinTables.Rd
    \n
    \n\n
    \n

    These helper functions perform join operations on data tables.\nMost of them are basically one-liners.\nSee https://rpubs.com/ronasta/join_data_tables for a overview of join operations in\ndata table or alternatively dplyr's vignette on two table verbs.

    \n
    \n\n 
    ijoin(x, y, by = NULL)\n\nljoin(x, y, by = NULL)\n\nrjoin(x, y, by = NULL)\n\nojoin(x, y, by = NULL)\n\nsjoin(x, y, by = NULL)\n\najoin(x, y, by = NULL)\n\nujoin(x, y, all.y = FALSE, by = NULL)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    x

    [data.frame]
    \nFirst data.frame to join.

    y

    [data.frame]
    \nSecond data.frame to join.

    by

    [character]
    \nColumn name(s) of variables used to match rows in x and y.\nIf not provided, a heuristic similar to the one described in the dplyr vignette is used:

      \n

    1. If x is keyed, the existing key will be used if y has the same column(s).

      2. \n

    2. If x is not keyed, the intersect of common columns names is used if not empty.

      3. \n

    3. Raise an exception.

      4. \n

    You may pass a named character vector to merge on columns with different names in x and\ny: by = c(\"x.id\" = \"y.id\") will match x's “x.id” column with y\\'s\n“y.id” column.

    all.y

    [logical(1)]
    \nKeep columns of y which are not in x?

    \n\n

    Value

    \n\n

    [data.table] with key identical to by.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \n# Create two tables for demonstration\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(identity, x = 1:6, reg = tmp)\n
    #> Adding 6 jobs ...
    x = getJobPars(reg = tmp)\ny = findJobs(x >= 2 & x <= 5, reg = tmp)\ny$extra.col = head(letters, nrow(y))\n\n# Inner join: similar to intersect(): keep all columns of x and y with common matches\nijoin(x, y)\n
    #>    job.id  job.pars extra.col\n#> 1:      2 <list[1]>         a\n#> 2:      3 <list[1]>         b\n#> 3:      4 <list[1]>         c\n#> 4:      5 <list[1]>         d
    \n# Left join: use all ids from x, keep all columns of x and y\nljoin(x, y)\n
    #>    job.id extra.col  job.pars\n#> 1:      1      <NA> <list[1]>\n#> 2:      2         a <list[1]>\n#> 3:      3         b <list[1]>\n#> 4:      4         c <list[1]>\n#> 5:      5         d <list[1]>\n#> 6:      6      <NA> <list[1]>
    \n# Right join: use all ids from y, keep all columns of x and y\nrjoin(x, y)\n
    #>    job.id  job.pars extra.col\n#> 1:      2 <list[1]>         a\n#> 2:      3 <list[1]>         b\n#> 3:      4 <list[1]>         c\n#> 4:      5 <list[1]>         d
    \n# Outer join: similar to union(): keep all columns of x and y with matches in x or y\nojoin(x, y)\n
    #>    job.id  job.pars extra.col\n#> 1:      1 <list[1]>      <NA>\n#> 2:      2 <list[1]>         a\n#> 3:      3 <list[1]>         b\n#> 4:      4 <list[1]>         c\n#> 5:      5 <list[1]>         d\n#> 6:      6 <list[1]>      <NA>
    \n# Semi join: filter x with matches in y\nsjoin(x, y)\n
    #>    job.id  job.pars\n#> 1:      2 <list[1]>\n#> 2:      3 <list[1]>\n#> 3:      4 <list[1]>\n#> 4:      5 <list[1]>
    \n# Anti join: filter x with matches not in y\najoin(x, y)\n
    #>    job.id  job.pars\n#> 1:      1 <list[1]>\n#> 2:      6 <list[1]>
    \n# Updating join: Replace values in x with values in y\nujoin(x, y)\n
    #>    job.id  job.pars\n#> 1:      1 <list[1]>\n#> 2:      2 <list[1]>\n#> 3:      3 <list[1]>\n#> 4:      4 <list[1]>\n#> 5:      5 <list[1]>\n#> 6:      6 <list[1]>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/Tags.html", "content": "\n\n\n \n \n\n\n\nAdd or Remove Job Tags — Tags • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Add or Remove Job Tags

    \n Source: R/Tags.R\n
    Tags.Rd
    \n
    \n\n
    \n

    Add and remove arbitrary tags to jobs.

    \n
    \n\n 
    addJobTags(ids = NULL, tags, reg = getDefaultRegistry())\n\nremoveJobTags(ids = NULL, tags, reg = getDefaultRegistry())\n\ngetUsedJobTags(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    tags

    [character]
    \nTags to add or remove as strings. Each tag may consist of letters, numbers, underscore and dots (pattern “^[[:alnum:]_.]+”).

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with job ids affected (invisible).

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    ids = batchMap(sqrt, x = -3:3, reg = tmp)\n
    #> Adding 7 jobs ...
    \n# Add new tag to all ids\naddJobTags(ids, \"needs.computation\", reg = tmp)\ngetJobTags(reg = tmp)\n
    #>    job.id              tags\n#> 1:      1 needs.computation\n#> 2:      2 needs.computation\n#> 3:      3 needs.computation\n#> 4:      4 needs.computation\n#> 5:      5 needs.computation\n#> 6:      6 needs.computation\n#> 7:      7 needs.computation
    \n# Add more tags\naddJobTags(findJobs(x < 0, reg = tmp), \"x.neg\", reg = tmp)\naddJobTags(findJobs(x > 0, reg = tmp), \"x.pos\", reg = tmp)\ngetJobTags(reg = tmp)\n
    #>    job.id                    tags\n#> 1:      1 needs.computation,x.neg\n#> 2:      2 needs.computation,x.neg\n#> 3:      3 needs.computation,x.neg\n#> 4:      4       needs.computation\n#> 5:      5 needs.computation,x.pos\n#> 6:      6 needs.computation,x.pos\n#> 7:      7 needs.computation,x.pos
    \n# Submit first 5 jobs and remove tag if successful\nids = submitJobs(1:5, reg = tmp)\n
    #> Submitting 5 jobs in 5 chunks using cluster functions 'Interactive' ...
    #> Warning: NaNs produced
    #> Warning: NaNs produced
    #> Warning: NaNs produced
    if (waitForJobs(reg = tmp))\n  removeJobTags(ids, \"needs.computation\", reg = tmp)\ngetJobTags(reg = tmp)\n
    #>    job.id                    tags\n#> 1:      1                   x.neg\n#> 2:      2                   x.neg\n#> 3:      3                   x.neg\n#> 4:      4                    <NA>\n#> 5:      5                   x.pos\n#> 6:      6 needs.computation,x.pos\n#> 7:      7 needs.computation,x.pos
    \n# Grep for warning message and add a tag\naddJobTags(grepLogs(pattern = \"NaNs produced\", reg = tmp), \"div.zero\", reg = tmp)\ngetJobTags(reg = tmp)\n
    #>    job.id                    tags\n#> 1:      1                   x.neg\n#> 2:      2                   x.neg\n#> 3:      3                   x.neg\n#> 4:      4                    <NA>\n#> 5:      5                   x.pos\n#> 6:      6 needs.computation,x.pos\n#> 7:      7 needs.computation,x.pos
    \n# All tags where tag x.neg is set:\nids = findTagged(\"x.neg\", reg = tmp)\ngetUsedJobTags(ids, reg = tmp)\n
    #> [1] \"x.neg\"
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/Worker.html", "content": "\n\n\n \n \n\n\n\nCreate a Linux-Worker — Worker • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Create a Linux-Worker

    \n Source: R/Worker.R\n
    Worker.Rd
    \n
    \n\n
    \n

    R6Class to create local and remote linux workers.

    \n
    \n\n\n\n

    Format

    \n\n

    An R6Class generator object

    \n

    Value

    \n\n

    [Worker].

    \n

    Fields

    \n\n \n\n
    \n
    nodename

    Host name. Set via constructor.

    \n\n
    ncpus

    Number of CPUs. Set via constructor and defaults to a heuristic which tries to detect the number of CPUs of the machine.

    \n\n
    max.load

    Maximum load average (of the last 5 min). Set via constructor and defaults to the number of CPUs of the machine.

    \n\n
    status

    Status of the worker; one of “unknown”, “available”, “max.cpus” and “max.load”.

    \n\n
    \n

    Methods

    \n\n \n\n
    \n
    new(nodename, ncpus, max.load)

    Constructor.

    \n
    update(reg)

    Update the worker status.

    \n
    list(reg)

    List running jobs.

    \n
    start(reg, fn, outfile)

    Start job collection in file “fn” and output to “outfile”.

    \n
    kill(reg, batch.id)

    Kill job matching the “batch.id”.

    \n\n
    \n\n\n

    Examples

    \n 
    if (FALSE) {\n# create a worker for the local machine and use 4 CPUs.\nWorker$new(\"localhost\", ncpus = 4)\n}\n
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/addAlgorithm.html", "content": "\n\n\n \n \n\n\n\nDefine Algorithms for Experiments — addAlgorithm • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Define Algorithms for Experiments

    \n Source: R/Algorithm.R\n
    addAlgorithm.Rd
    \n
    \n\n
    \n

    Algorithms are functions which get the codedata part as well as the problem instance (the return value of the\nfunction defined in Problem) and return an arbitrary R object.

    \n

    This function serializes all components to the file system and registers the algorithm in the ExperimentRegistry.

    \n

    removeAlgorithm removes all jobs from the registry which depend on the specific algorithm.\nreg$algorithms holds the IDs of already defined algorithms.

    \n
    \n\n 
    addAlgorithm(name, fun = NULL, reg = getDefaultRegistry())\n\nremoveAlgorithms(name, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    name

    [character(1)]
    \nUnique identifier for the algorithm.

    fun

    [function]
    \n The algorithm function. The static problem part is passed as “data”, the generated\n problem instance is passed as “instance” and the Job/Experiment as “job”.\n Therefore, your function must have the formal arguments “job”, “data” and “instance” (or dots ...).

    \n

    If you do not provide a function, it defaults to a function which just returns the instance.

    reg

    [ExperimentRegistry]
    \nRegistry. If not explicitly passed, uses the last created registry.

    \n\n

    Value

    \n\n

    [Algorithm]. Object of class “Algorithm”.

    \n

    See also

    \n\n

    Problem, addExperiments

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/addExperiments.html", "content": "\n\n\n \n \n\n\n\nAdd Experiments to the Registry — addExperiments • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Add Experiments to the Registry

    \n Source: R/addExperiments.R\n
    addExperiments.Rd
    \n
    \n\n
    \n

    Adds experiments (parametrized combinations of problems with algorithms) to the registry and thereby defines batch jobs.

    \n

    If multiple problem designs or algorithm designs are provided, they are combined via the Cartesian product.\nE.g., if you have two problems p1 and p2 and three algorithms a1, a2 and a3,\naddExperiments creates experiments for all parameters for the combinations (p1, a1), (p1, a2),\n(p1, a3), (p2, a1), (p2, a2) and (p2, a3).

    \n
    \n\n 
    addExperiments(\n  prob.designs = NULL,\n  algo.designs = NULL,\n  repls = 1L,\n  combine = \"crossprod\",\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    prob.designs

    [named list of data.frame]
    \nNamed list of data frames (or data.table).\nThe name must match the problem name while the column names correspond to parameters of the problem.\nIf NULL, experiments for all defined problems without any parameters are added.

    algo.designs

    [named list of data.table or data.frame]
    \nNamed list of data frames (or data.table).\nThe name must match the algorithm name while the column names correspond to parameters of the algorithm.\nIf NULL, experiments for all defined algorithms without any parameters are added.

    repls

    [integer(1)]
    \nNumber of replications for each experiment.

    combine

    [character(1)]
    \nHow to combine the rows of a single problem design with the rows of a single algorithm design?\nDefault is “crossprod” which combines each row of the problem design which each row of the algorithm design\nin a cross-product fashion. Set to “bind” to just cbind the tables of\nproblem and algorithm designs where the shorter table is repeated if necessary.

    reg

    [ExperimentRegistry]
    \nRegistry. If not explicitly passed, uses the last created registry.

    \n\n

    Value

    \n\n

    [data.table] with ids of added jobs stored in column “job.id”.

    \n

    Note

    \n\n

    R's data.frame converts character vectors to factors by default in R versions prior to 4.0.0 which frequently resulted in problems using addExperiments.\nTherefore, this function will warn about factor variables if the following conditions hold:

      \n

    1. R version is < 4.0.0

      2. \n

    2. The design is passed as a data.frame, not a data.table or tibble.

      3. \n

    3. The option “stringsAsFactors” is not set or set to TRUE.

      4. \n
    \n\n

    See also

    \n\n

    Other Experiment: \nremoveExperiments(),\nsummarizeExperiments()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    \n# add first problem\nfun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\naddProblem(\"rnorm\", fun = fun, reg = tmp)\n
    #> Adding problem 'rnorm'
    \n# add second problem\nfun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\naddProblem(\"rexp\", fun = fun, reg = tmp)\n
    #> Adding problem 'rexp'
    \n# add first algorithm\nfun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\naddAlgorithm(\"average\", fun = fun, reg = tmp)\n
    #> Adding algorithm 'average'
    \n# add second algorithm\nfun = function(instance, ...) sd(instance)\naddAlgorithm(\"deviation\", fun = fun, reg = tmp)\n
    #> Adding algorithm 'deviation'
    \n# define problem and algorithm designs\nlibrary(data.table)\nprob.designs = algo.designs = list()\nprob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\nprob.designs$rexp = data.table(n = 100, lambda = 1:5)\nalgo.designs$average = data.table(method = c(\"mean\", \"median\"))\nalgo.designs$deviation = data.table()\n\n# add experiments and submit\naddExperiments(prob.designs, algo.designs, reg = tmp)\n
    #> Adding 30 experiments ('rnorm'[15] x 'average'[2] x repls[1]) ...
    #> Adding 15 experiments ('rnorm'[15] x 'deviation'[1] x repls[1]) ...
    #> Adding 10 experiments ('rexp'[5] x 'average'[2] x repls[1]) ...
    #> Adding 5 experiments ('rexp'[5] x 'deviation'[1] x repls[1]) ...
    \n# check what has been created\nsummarizeExperiments(reg = tmp)\n
    #>    problem algorithm .count\n#> 1:   rnorm   average     30\n#> 2:   rnorm deviation     15\n#> 3:    rexp   average     10\n#> 4:    rexp deviation      5
    unwrap(getJobPars(reg = tmp))\n
    #>     job.id problem algorithm   n mean sd lambda method\n#>  1:      1   rnorm   average 100   -1  1     NA   mean\n#>  2:      2   rnorm   average 100   -1  1     NA median\n#>  3:      3   rnorm   average 100   -1  2     NA   mean\n#>  4:      4   rnorm   average 100   -1  2     NA median\n#>  5:      5   rnorm   average 100   -1  3     NA   mean\n#>  6:      6   rnorm   average 100   -1  3     NA median\n#>  7:      7   rnorm   average 100   -1  4     NA   mean\n#>  8:      8   rnorm   average 100   -1  4     NA median\n#>  9:      9   rnorm   average 100   -1  5     NA   mean\n#> 10:     10   rnorm   average 100   -1  5     NA median\n#> 11:     11   rnorm   average 100    0  1     NA   mean\n#> 12:     12   rnorm   average 100    0  1     NA median\n#> 13:     13   rnorm   average 100    0  2     NA   mean\n#> 14:     14   rnorm   average 100    0  2     NA median\n#> 15:     15   rnorm   average 100    0  3     NA   mean\n#> 16:     16   rnorm   average 100    0  3     NA median\n#> 17:     17   rnorm   average 100    0  4     NA   mean\n#> 18:     18   rnorm   average 100    0  4     NA median\n#> 19:     19   rnorm   average 100    0  5     NA   mean\n#> 20:     20   rnorm   average 100    0  5     NA median\n#> 21:     21   rnorm   average 100    1  1     NA   mean\n#> 22:     22   rnorm   average 100    1  1     NA median\n#> 23:     23   rnorm   average 100    1  2     NA   mean\n#> 24:     24   rnorm   average 100    1  2     NA median\n#> 25:     25   rnorm   average 100    1  3     NA   mean\n#> 26:     26   rnorm   average 100    1  3     NA median\n#> 27:     27   rnorm   average 100    1  4     NA   mean\n#> 28:     28   rnorm   average 100    1  4     NA median\n#> 29:     29   rnorm   average 100    1  5     NA   mean\n#> 30:     30   rnorm   average 100    1  5     NA median\n#> 31:     31   rnorm deviation 100   -1  1     NA   <NA>\n#> 32:     32   rnorm deviation 100   -1  2     NA   <NA>\n#> 33:     33   rnorm deviation 100   -1  3     NA   <NA>\n#> 34:     34   rnorm deviation 100   -1  4     NA   <NA>\n#> 35:     35   rnorm deviation 100   -1  5     NA   <NA>\n#> 36:     36   rnorm deviation 100    0  1     NA   <NA>\n#> 37:     37   rnorm deviation 100    0  2     NA   <NA>\n#> 38:     38   rnorm deviation 100    0  3     NA   <NA>\n#> 39:     39   rnorm deviation 100    0  4     NA   <NA>\n#> 40:     40   rnorm deviation 100    0  5     NA   <NA>\n#> 41:     41   rnorm deviation 100    1  1     NA   <NA>\n#> 42:     42   rnorm deviation 100    1  2     NA   <NA>\n#> 43:     43   rnorm deviation 100    1  3     NA   <NA>\n#> 44:     44   rnorm deviation 100    1  4     NA   <NA>\n#> 45:     45   rnorm deviation 100    1  5     NA   <NA>\n#> 46:     46    rexp   average 100   NA NA      1   mean\n#> 47:     47    rexp   average 100   NA NA      1 median\n#> 48:     48    rexp   average 100   NA NA      2   mean\n#> 49:     49    rexp   average 100   NA NA      2 median\n#> 50:     50    rexp   average 100   NA NA      3   mean\n#> 51:     51    rexp   average 100   NA NA      3 median\n#> 52:     52    rexp   average 100   NA NA      4   mean\n#> 53:     53    rexp   average 100   NA NA      4 median\n#> 54:     54    rexp   average 100   NA NA      5   mean\n#> 55:     55    rexp   average 100   NA NA      5 median\n#> 56:     56    rexp deviation 100   NA NA      1   <NA>\n#> 57:     57    rexp deviation 100   NA NA      2   <NA>\n#> 58:     58    rexp deviation 100   NA NA      3   <NA>\n#> 59:     59    rexp deviation 100   NA NA      4   <NA>\n#> 60:     60    rexp deviation 100   NA NA      5   <NA>\n#>     job.id problem algorithm   n mean sd lambda method
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/addProblem.html", "content": "\n\n\n \n \n\n\n\nDefine Problems for Experiments — addProblem • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Define Problems for Experiments

    \n Source: R/Problem.R\n
    addProblem.Rd
    \n
    \n\n
    \n

    Problems may consist of up to two parts: A static, immutable part (data in addProblem)\nand a dynamic, stochastic part (fun in addProblem).\nFor example, for statistical learning problems a data frame would be the static problem part while\na resampling function would be the stochastic part which creates problem instance.\nThis instance is then typically passed to a learning algorithm like a wrapper around a statistical model\n(fun in addAlgorithm).

    \n

    This function serialize all components to the file system and registers the problem in the ExperimentRegistry.

    \n

    removeProblem removes all jobs from the registry which depend on the specific problem.\nreg$problems holds the IDs of already defined problems.

    \n
    \n\n 
    addProblem(\n  name,\n  data = NULL,\n  fun = NULL,\n  seed = NULL,\n  cache = FALSE,\n  reg = getDefaultRegistry()\n)\n\nremoveProblems(name, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    name

    [character(1)]
    \nUnique identifier for the problem.

    data

    [ANY]
    \nStatic problem part. Default is NULL.

    fun

    [function]
    \nThe function defining the stochastic problem part.\nThe static part is passed to this function with name “data” and the Job/Experiment\nis passed as “job”.\nTherefore, your function must have the formal arguments “job” and “data” (or dots ...).\nIf you do not provide a function, it defaults to a function which just returns the data part.

    seed

    [integer(1)]
    \nStart seed for this problem. This allows the “synchronization” of a stochastic\nproblem across algorithms, so that different algorithms are evaluated on the same stochastic instance.\nIf the problem seed is defined, the seeding mechanism works as follows:\n(1) Before the dynamic part of a problem is instantiated,\nthe seed of the problem + [replication number] - 1 is set, i.e. the first\nreplication uses the problem seed. (2) The stochastic part of the problem is\ninstantiated. (3) From now on the usual experiment seed of the registry is used,\nsee ExperimentRegistry.\nIf seed is set to NULL (default), the job seed is used to instantiate the problem and\ndifferent algorithms see different stochastic instances of the same problem.

    cache

    [logical(1)]
    \nIf TRUE and seed is set, problem instances will be cached on the file system.\nThis assumes that each problem instance is deterministic for each combination of hyperparameter setting\nand each replication number.\nThis feature is experimental.

    reg

    [ExperimentRegistry]
    \nRegistry. If not explicitly passed, uses the last created registry.

    \n\n

    Value

    \n\n

    [Problem]. Object of class “Problem” (invisibly).

    \n

    See also

    \n\n

    Algorithm, addExperiments

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    addProblem(\"p1\", fun = function(job, data) data, reg = tmp)\n
    #> Adding problem 'p1'
    addProblem(\"p2\", fun = function(job, data) job, reg = tmp)\n
    #> Adding problem 'p2'
    addAlgorithm(\"a1\", fun = function(job, data, instance) instance, reg = tmp)\n
    #> Adding algorithm 'a1'
    addExperiments(repls = 2, reg = tmp)\n
    #> Adding 2 experiments ('p1'[1] x 'a1'[1] x repls[2]) ...
    #> Adding 2 experiments ('p2'[1] x 'a1'[1] x repls[2]) ...
    \n# List problems, algorithms and job parameters:\ntmp$problems\n
    #> [1] \"p1\" \"p2\"
    tmp$algorithms\n
    #> [1] \"a1\"
    getJobPars(reg = tmp)\n
    #>    job.id problem prob.pars algorithm algo.pars\n#> 1:      1      p1 <list[0]>        a1 <list[0]>\n#> 2:      2      p1 <list[0]>        a1 <list[0]>\n#> 3:      3      p2 <list[0]>        a1 <list[0]>\n#> 4:      4      p2 <list[0]>        a1 <list[0]>
    \n# Remove one problem\nremoveProblems(\"p1\", reg = tmp)\n
    #> Removing Problem 'p1' and 2 corresponding jobs ...
    \n# List problems and algorithms:\ntmp$problems\n
    #> [1] \"p2\"
    tmp$algorithms\n
    #> [1] \"a1\"
    getJobPars(reg = tmp)\n
    #>    job.id problem prob.pars algorithm algo.pars\n#> 1:      3      p2 <list[0]>        a1 <list[0]>\n#> 2:      4      p2 <list[0]>        a1 <list[0]>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/assertRegistry.html", "content": "\n\n\n \n \n\n\n\nassertRegistry — assertRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    assertRegistry

    \n Source: R/Registry.R\n
    assertRegistry.Rd
    \n
    \n\n
    \n

    Assert that a given object is a batchtools registry.\nAdditionally can sync the registry, check if it is writeable, or check if jobs are running.\nIf any check fails, throws an error indicting the reason for the failure.

    \n
    \n\n 
    assertRegistry(\n  reg,\n  class = NULL,\n  writeable = FALSE,\n  sync = FALSE,\n  running.ok = TRUE\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    reg

    [Registry]
    \nThe object asserted to be a Registry.

    class

    [character(1)]
    \nIf NULL (default), reg must only inherit from class “Registry”.\nOtherwise check that reg is of class class.\nE.g., if set to “Registry”, a ExperimentRegistry would not pass.

    writeable

    [logical(1)]
    \nCheck if the registry is writeable.

    sync

    [logical(1)]
    \nTry to synchronize the registry by including pending results from the file system.\nSee syncRegistry.

    running.ok

    [logical(1)]
    \nIf FALSE throw an error if jobs associated with the registry are currently running.

    \n\n

    Value

    \n\n

    TRUE invisibly.

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/batchExport.html", "content": "\n\n\n \n \n\n\n\nExport Objects to the Slaves — batchExport • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Export Objects to the Slaves

    \n Source: R/Export.R\n
    batchExport.Rd
    \n
    \n\n
    \n

    Objects are saved in subdirectory “exports” of the\n“file.dir” of reg.\nThey are automatically loaded and placed in the global environment\neach time the registry is loaded or a job collection is executed.

    \n
    \n\n 
    batchExport(\n  export = list(),\n  unexport = character(0L),\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    export

    [list]
    \nNamed list of objects to export.

    unexport

    [character]
    \nVector of object names to unexport.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with name and uri to the exported objects.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    \n# list exports\nexports = batchExport(reg = tmp)\nprint(exports)\n
    #> Empty data.table (0 rows and 2 cols): name,uri
    \n# add a job and required exports\nbatchMap(function(x) x^2 + y + z, x = 1:3, reg = tmp)\n
    #> Adding 3 jobs ...
    exports = batchExport(export = list(y = 99, z = 1), reg = tmp)\n
    #> Exporting new objects: 'y','z' ...
    print(exports)\n
    #>    name                                        uri\n#> 1:    y /tmp/batchtools-example/reg/exports/PE.rds\n#> 2:    z /tmp/batchtools-example/reg/exports/PI.rds
    \nsubmitJobs(reg = tmp)\n
    #> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    stopifnot(loadResult(1, reg = tmp) == 101)\n\n# Un-export z\nexports = batchExport(unexport = \"z\", reg = tmp)\n
    #> Un-exporting exported objects: 'z' ...
    print(exports)\n
    #>    name                                        uri\n#> 1:    y /tmp/batchtools-example/reg/exports/PE.rds
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/batchMap.html", "content": "\n\n\n \n \n\n\n\nMap Operation for Batch Systems — batchMap • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Map Operation for Batch Systems

    \n Source: R/batchMap.R\n
    batchMap.Rd
    \n
    \n\n
    \n

    A parallel and asynchronous Map/mapply for batch systems.\nNote that this function only defines the computational jobs.\nThe actual computation is started with submitJobs.\nResults and partial results can be collected with reduceResultsList, reduceResults or\nloadResult.

    \n

    For a synchronous Map-like execution, see btmapply.

    \n
    \n\n 
    batchMap(\n  fun,\n  ...,\n  args = list(),\n  more.args = list(),\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    fun

    [function]
    \nFunction to map over arguments provided via ....\nParameters given via args or ... are passed as-is, in the respective order and possibly named.\nIf the function has the named formal argument “.job”, the Job is passed to the function\non the slave.

    ...

    [ANY]
    \nArguments to vectorize over (list or vector).\nShorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\nMutually exclusive with args.\nNote that although it is possible to iterate over large objects (e.g., lists of data frames or matrices), this usually\nhurts the overall performance and thus is discouraged.

    args

    [list | data.frame]
    \nArguments to vectorize over as (named) list or data frame.\nShorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\nMutually exclusive with ....

    more.args

    [list]
    \nA list of further arguments passed to fun.\nDefault is an empty list.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with ids of added jobs stored in column “job.id”.

    \n

    See also

    \n\n

    batchReduce

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(3) \n# example using \"...\" and more.args\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg1' using cluster functions 'Interactive'
    f = function(x, y) x^2 + y\nids = batchMap(f, x = 1:10, more.args = list(y = 100), reg = tmp)\n
    #> Adding 10 jobs ...
    getJobPars(reg = tmp)\n
    #>     job.id  job.pars\n#>  1:      1 <list[1]>\n#>  2:      2 <list[1]>\n#>  3:      3 <list[1]>\n#>  4:      4 <list[1]>\n#>  5:      5 <list[1]>\n#>  6:      6 <list[1]>\n#>  7:      7 <list[1]>\n#>  8:      8 <list[1]>\n#>  9:      9 <list[1]>\n#> 10:     10 <list[1]>
    testJob(6, reg = tmp) # 100 + 6^2 = 136\n
    #> ### [bt]: Setting seed to 12787 ...
    #> [1] 136
    \n# vector recycling\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg2' using cluster functions 'Interactive'
    f = function(...) list(...)\nids = batchMap(f, x = 1:3, y = 1:6, reg = tmp)\n
    #> Adding 6 jobs ...
    getJobPars(reg = tmp)\n
    #>    job.id  job.pars\n#> 1:      1 <list[2]>\n#> 2:      2 <list[2]>\n#> 3:      3 <list[2]>\n#> 4:      4 <list[2]>\n#> 5:      5 <list[2]>\n#> 6:      6 <list[2]>
    \n# example for an expand.grid()-like operation on parameters\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg3' using cluster functions 'Interactive'
    ids = batchMap(paste, args = data.table::CJ(x = letters[1:3], y = 1:3), reg = tmp)\n
    #> Adding 9 jobs ...
    getJobPars(reg = tmp)\n
    #>    job.id  job.pars\n#> 1:      1 <list[2]>\n#> 2:      2 <list[2]>\n#> 3:      3 <list[2]>\n#> 4:      4 <list[2]>\n#> 5:      5 <list[2]>\n#> 6:      6 <list[2]>\n#> 7:      7 <list[2]>\n#> 8:      8 <list[2]>\n#> 9:      9 <list[2]>
    testJob(6, reg = tmp)\n
    #> ### [bt]: Setting seed to 8571 ...
    #> [1] \"b 3\"
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/batchMapResults.html", "content": "\n\n\n \n \n\n\n\nMap Over Results to Create New Jobs — batchMapResults • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Map Over Results to Create New Jobs

    \n Source: R/batchMapResults.R\n
    batchMapResults.Rd
    \n
    \n\n
    \n

    This function allows you to create new computational jobs (just like batchMap based on the results of\na Registry.

    \n
    \n\n 
    batchMapResults(\n  fun,\n  ids = NULL,\n  ...,\n  more.args = list(),\n  target,\n  source = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    fun

    [function]
    \nFunction which takes the result as first (unnamed) argument.

    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findDone.\nInvalid ids are ignored.

    ...

    [ANY]
    \nArguments to vectorize over (list or vector). Passed to batchMap.

    more.args

    [list]
    \nA list of further arguments passed to fun.\nDefault is an empty list.

    target

    [Registry]
    \nEmpty Registry where new jobs are created for.

    source

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with ids of jobs added to target.

    \n

    Note

    \n\n

    The URI to the result files in registry source is hard coded as parameter in the target registry.\nThis means that target is currently not portable between systems for computation.

    \n

    See also

    \n\n

    Other Results: \nloadResult(),\nreduceResultsList(),\nreduceResults()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(2) \n# Source registry: calculate square of some numbers\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg1' using cluster functions 'Interactive'
    batchMap(function(x) list(square = x^2), x = 1:10, reg = tmp)\n
    #> Adding 10 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    \n# Target registry: calculate the square root on results of first registry\ntarget = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg2' using cluster functions 'Interactive'
    batchMapResults(fun = function(x, y) list(sqrt = sqrt(x$square)), ids = 4:8,\n  target = target, source = tmp)\n
    #> Adding 5 jobs ...
    submitJobs(reg = target)\n
    #> Submitting 5 jobs in 5 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = target)\n
    #> [1] TRUE
    \n# Map old to new ids. First, get a table with results and parameters\nresults = unwrap(rjoin(getJobPars(reg = target), reduceResultsDataTable(reg = target)))\nprint(results)\n
    #>    job.id .id sqrt\n#> 1:      1   4    4\n#> 2:      2   5    5\n#> 3:      3   6    6\n#> 4:      4   7    7\n#> 5:      5   8    8
    \n# Parameter '.id' points to job.id in 'source'. Use a inner join to combine:\nijoin(results, unwrap(reduceResultsDataTable(reg = tmp)), by = c(\".id\" = \"job.id\"))\n
    #>    job.id .id sqrt square\n#> 1:      1   4    4     16\n#> 2:      2   5    5     25\n#> 3:      3   6    6     36\n#> 4:      4   7    7     49\n#> 5:      5   8    8     64
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/batchReduce.html", "content": "\n\n\n \n \n\n\n\nReduce Operation for Batch Systems — batchReduce • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Reduce Operation for Batch Systems

    \n Source: R/batchReduce.R\n
    batchReduce.Rd
    \n
    \n\n
    \n

    A parallel and asynchronous Reduce for batch systems.\nNote that this function only defines the computational jobs.\nEach job reduces a certain number of elements on one slave.\nThe actual computation is started with submitJobs.\nResults and partial results can be collected with reduceResultsList, reduceResults or\nloadResult.

    \n
    \n\n 
    batchReduce(\n  fun,\n  xs,\n  init = NULL,\n  chunks = seq_along(xs),\n  more.args = list(),\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    fun

    [function(aggr, x, ...)]
    \nFunction to reduce xs with.

    xs

    [vector]
    \nVector to reduce.

    init

    [ANY]
    \nInitial object for reducing. See Reduce.

    chunks

    [integer(length(xs))]
    \nGroup for each element of xs. Can be generated with chunk.

    more.args

    [list]
    \nA list of additional arguments passed to fun.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with ids of added jobs stored in column “job.id”.

    \n

    See also

    \n\n

    batchMap

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \n# define function to reduce on slave, we want to sum a vector\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    xs = 1:100\nf = function(aggr, x) aggr + x\n\n# sum 20 numbers on each slave process, i.e. 5 jobs\nchunks = chunk(xs, chunk.size = 5)\nbatchReduce(fun = f, 1:100, init = 0, chunks = chunks, reg = tmp)\n
    #> Adding 20 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 20 jobs in 20 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    \n# now reduce one final time on master\nreduceResults(fun = function(aggr, job, res) f(aggr, res), reg = tmp)\n
    #> [1] 5050
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/batchtools-deprecated.html", "content": "\n\n\n \n \n\n\n\nDeprecated function in the batchtools package — batchtools-deprecated • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.10\n \n
    \n\n
    \n \n \n \n \n
    \n
    \n
    \n\n \n
    \n\n
    \n
    \n
    \n

    Deprecated function in the batchtools package

    \n Source: R/zzz.R\n
    batchtools-deprecated.Rd
    \n
    \n\n
    \n \n

    The following functions have been deprecated:

    \n\n
    chunkIdsdeprecated in favor of chunk, lpt and binpack
    \n \n
    \n\n \n\n
    \n
    \n

    Contents

    \n
      \n
    \n\n
    \n
    \n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown.

    \n
    \n\n
    \n
    \n\n \n\n \n\n\n" }, { "path": "docs/reference/batchtools-package.html", "content": "\n\n\n \n \n\n\n\nbatchtools: Tools for Computation on Batch Systems — batchtools-package • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    batchtools: Tools for Computation on Batch Systems

    \n Source: R/zzz.R\n
    batchtools-package.Rd
    \n
    \n\n
    \n

    For bug reports and feature requests please use the tracker:\nhttps://github.com/mllg/batchtools.

    \n
    \n\n\n\n

    Package options

    \n\n \n\n
    \n
    batchtools.verbose

    Verbosity. Set to FALSE to suppress info messages and progress bars.

    \n
    batchtools.progress

    Progress bars. Set to FALSE to disable them.

    \n
    batchtools.timestamps

    Add time stamps to log output. Set to FALSE to disable them.

    \n\n

    Furthermore, you may enable a debug mode using the debugme package by\nsetting the environment variable “DEBUGME” to “batchtools” before\nloading batchtools.

    \n

    See also

    \n\n

    Useful links:

    \n\n
    \n

    Author

    \n\n

    Maintainer: Michel Lang michellang@gmail.com (ORCID)

    \n

    Authors:

    \n\n

    Other contributors:

    \n\n\n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/btlapply.html", "content": "\n\n\n \n \n\n\n\nSynchronous Apply Functions — btlapply • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Synchronous Apply Functions

    \n Source: R/btlapply.R\n
    btlapply.Rd
    \n
    \n\n
    \n

    This is a set of functions acting as counterparts to the sequential popular apply functions in base R:\nbtlapply for lapply and btmapply for mapply.

    \n

    Internally, jobs are created using batchMap on the provided registry.\nIf no registry is provided, a temporary registry (see argument file.dir of makeRegistry) and batchMap\nwill be used.\nAfter all jobs are terminated (see waitForJobs), the results are collected and returned as a list.

    \n

    Note that these functions are only suitable for short and fail-safe operations\non batch system. If some jobs fail, you have to retrieve partial results from the\nregistry directory yourself.

    \n
    \n\n 
    btlapply(\n  X,\n  fun,\n  ...,\n  resources = list(),\n  n.chunks = NULL,\n  chunk.size = NULL,\n  reg = makeRegistry(file.dir = NA)\n)\n\nbtmapply(\n  fun,\n  ...,\n  more.args = list(),\n  simplify = FALSE,\n  use.names = TRUE,\n  resources = list(),\n  n.chunks = NULL,\n  chunk.size = NULL,\n  reg = makeRegistry(file.dir = NA)\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    X

    [vector]
    \nVector to apply over.

    fun

    [function]
    \nFunction to apply.

    ...

    [ANY]
    \nAdditional arguments passed to fun (btlapply) or vectors to map over (btmapply).

    resources

    [named list]
    \nComputational resources for the jobs to submit. The actual elements of this list\n(e.g. something like “walltime” or “nodes”) depend on your template file, exceptions are outlined in the section 'Resources'.\nDefault settings for a system can be set in the configuration file by defining the named list default.resources.\nNote that these settings are merged by name, e.g. merging list(walltime = 300) into list(walltime = 400, memory = 512)\nwill result in list(walltime = 300, memory = 512).\nSame holds for individual job resources passed as additional column of ids (c.f. section 'Resources').

    n.chunks

    [integer(1)]
    \nPassed to chunk before submitJobs.

    chunk.size

    [integer(1)]
    \nPassed to chunk before submitJobs.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    more.args

    [list]
    \nAdditional arguments passed to fun.

    simplify

    [logical(1)]
    \nSimplify the results using simplify2array?

    use.names

    [logical(1)]
    \nUse names of the input to name the output?

    \n\n

    Value

    \n\n

    [list] List with the results of the function call.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \nbtlapply(1:3, function(x) x^2)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    #> Adding 3 jobs ...
    #> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
    #> [[1]]\n#> [1] 1\n#> \n#> [[2]]\n#> [1] 4\n#> \n#> [[3]]\n#> [1] 9\n#> 
    btmapply(function(x, y, z) x + y + z, x = 1:3, y = 1:3, more.args = list(z = 1), simplify = TRUE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/RtmpsmIC4J/registry9e4369cc3bf0' using cluster functions 'Interactive'
    #> Adding 3 jobs ...
    #> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
    #> [1] 3 5 7
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/cfBrewTemplate.html", "content": "\n\n\n \n \n\n\n\nCluster Functions Helper to Write Job Description Files — cfBrewTemplate • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Cluster Functions Helper to Write Job Description Files

    \n Source: R/clusterFunctions.R\n
    cfBrewTemplate.Rd
    \n
    \n\n
    \n

    This function is only intended for use in your own cluster functions implementation.

    \n

    Calls brew silently on your template, any error will lead to an exception.\nThe file is stored at the same place as the corresponding job file in the “jobs”-subdir\nof your files directory.

    \n
    \n\n 
    cfBrewTemplate(reg, text, jc)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    text

    [character(1)]
    \nString ready to be brewed. See cfReadBrewTemplate to read a template from the file system.

    jc

    [JobCollection)]
    \nWill be used as environment to brew the template file in. See JobCollection for a list of all\navailable variables.

    \n\n

    Value

    \n\n

    [character(1)]. File path to brewed template file.

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfHandleUnknownSubmitError(),\ncfKillJob(),\ncfReadBrewTemplate(),\nmakeClusterFunctions(),\nmakeSubmitJobResult(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/cfHandleUnknownSubmitError.html", "content": "\n\n\n \n \n\n\n\nCluster Functions Helper to Handle Unknown Errors — cfHandleUnknownSubmitError • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Cluster Functions Helper to Handle Unknown Errors

    \n Source: R/clusterFunctions.R\n
    cfHandleUnknownSubmitError.Rd
    \n
    \n\n
    \n

    This function is only intended for use in your own cluster functions implementation.

    \n

    Simply constructs a SubmitJobResult object with status code 101, NA as batch id and\nan informative error message containing the output of the OS command in output.

    \n
    \n\n 
    cfHandleUnknownSubmitError(cmd, exit.code, output)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    cmd

    [character(1)]
    \nOS command used to submit the job, e.g. qsub.

    exit.code

    [integer(1)]
    \nExit code of the OS command, should not be 0.

    output

    [character]
    \nOutput of the OS command, hopefully an informative error message.\nIf these are multiple lines in a vector, they are automatically joined.

    \n\n

    Value

    \n\n

    [SubmitJobResult].

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfKillJob(),\ncfReadBrewTemplate(),\nmakeClusterFunctions(),\nmakeSubmitJobResult(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/cfKillJob.html", "content": "\n\n\n \n \n\n\n\nCluster Functions Helper to Kill Batch Jobs — cfKillJob • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Cluster Functions Helper to Kill Batch Jobs

    \n Source: R/clusterFunctions.R\n
    cfKillJob.Rd
    \n
    \n\n
    \n

    This function is only intended for use in your own cluster functions implementation.

    \n

    Calls the OS command to kill a job via system like this: “cmd batch.job.id”. If the\ncommand returns an exit code > 0, the command is repeated after a 1 second sleep\nmax.tries-1 times. If the command failed in all tries, an error is generated.

    \n
    \n\n 
    cfKillJob(\n  reg,\n  cmd,\n  args = character(0L),\n  max.tries = 3L,\n  nodename = \"localhost\"\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    cmd

    [character(1)]
    \nOS command, e.g. “qdel”.

    args

    [character]
    \nArguments to cmd, including the batch id.

    max.tries

    [integer(1)]
    \nNumber of total times to try execute the OS command in cases of failures.\nDefault is 3.

    nodename

    [character(1)]
    \nName of the SSH node to run the command on. If set to “localhost” (default), the command\nis not piped through SSH.

    \n\n

    Value

    \n\n

    TRUE on success. An exception is raised otherwise.

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfHandleUnknownSubmitError(),\ncfReadBrewTemplate(),\nmakeClusterFunctions(),\nmakeSubmitJobResult(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/cfReadBrewTemplate.html", "content": "\n\n\n \n \n\n\n\nCluster Functions Helper to Parse a Brew Template — cfReadBrewTemplate • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Cluster Functions Helper to Parse a Brew Template

    \n Source: R/clusterFunctions.R\n
    cfReadBrewTemplate.Rd
    \n
    \n\n
    \n

    This function is only intended for use in your own cluster functions implementation.

    \n

    This function is only intended for use in your own cluster functions implementation.\nSimply reads your template file and returns it as a character vector.

    \n
    \n\n 
    cfReadBrewTemplate(template, comment.string = NA_character_)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nPath to template file which is then passed to brew.

    comment.string

    [character(1)]
    \nIgnore lines starting with this string.

    \n\n

    Value

    \n\n

    [character].

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfHandleUnknownSubmitError(),\ncfKillJob(),\nmakeClusterFunctions(),\nmakeSubmitJobResult(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/chunk.html", "content": "\n\n\n \n \n\n\n\nChunk Jobs for Sequential Execution — chunk • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Chunk Jobs for Sequential Execution

    \n Source: R/chunkIds.R\n
    chunk.Rd
    \n
    \n\n
    \n

    Jobs can be partitioned into “chunks” to be executed sequentially on the computational nodes.\nChunks are defined by providing a data frame with columns “job.id” and “chunk” (integer)\nto submitJobs.\nAll jobs with the same chunk number will be grouped together on one node to form a single\ncomputational job.

    \n

    The function chunk simply splits x into either a fixed number of groups, or\ninto a variable number of groups with a fixed number of maximum elements.

    \n

    The function lpt also groups x into a fixed number of chunks,\nbut uses the actual values of x in a greedy “Longest Processing Time” algorithm.\nAs a result, the maximum sum of elements in minimized.

    \n

    binpack splits x into a variable number of groups whose sum of elements do\nnot exceed the upper limit provided by chunk.size.

    \n

    See examples of estimateRuntimes for an application of binpack and lpt.

    \n
    \n\n 
    chunk(x, n.chunks = NULL, chunk.size = NULL, shuffle = TRUE)\n\nlpt(x, n.chunks = 1L)\n\nbinpack(x, chunk.size = max(x))
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    x

    [numeric]
    \nFor chunk an atomic vector (usually the job.id).\nFor binpack and lpt, the weights to group.

    n.chunks

    [integer(1)]
    \nRequested number of chunks.\nThe function chunk distributes the number of elements in x evenly while\nlpt tries to even out the sum of elements in each chunk.\nIf more chunks than necessary are requested, empty chunks are ignored.\nMutually exclusive with chunks.size.

    chunk.size

    [integer(1)]
    \nRequested chunk size for each single chunk.\nFor chunk this is the number of elements in x, for binpack the size\nis determined by the sum of values in x.\nMutually exclusive with n.chunks.

    shuffle

    [logical(1)]
    \nShuffles the groups. Default is TRUE.

    \n\n

    Value

    \n\n

    [integer] giving the chunk number for each element of x.

    \n

    See also

    \n\n

    estimateRuntimes

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(2) \nch = chunk(1:10, n.chunks = 2)\ntable(ch)\n
    #> ch\n#> 1 2 \n#> 5 5 
    \nch = chunk(rep(1, 10), chunk.size = 2)\ntable(ch)\n
    #> ch\n#> 1 2 3 4 5 \n#> 2 2 2 2 2 
    \nset.seed(1)\nx = runif(10)\nch = lpt(x, n.chunks = 2)\nsapply(split(x, ch), sum)\n
    #>        1        2 \n#> 2.808393 2.706746 
    \nset.seed(1)\nx = runif(10)\nch = binpack(x, 1)\nsapply(split(x, ch), sum)\n
    #>         1         2         3         4         5         6 \n#> 0.9446753 0.9699941 0.8983897 0.9263065 0.8307960 0.9449773 
    \n# Job chunking\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg1' using cluster functions 'Interactive'
    ids = batchMap(identity, 1:25, reg = tmp)\n
    #> Adding 25 jobs ...
    \n### Group into chunks with 10 jobs each\nlibrary(data.table)\nids[, chunk := chunk(job.id, chunk.size = 10)]\n
    #>     job.id chunk\n#>  1:      1     3\n#>  2:      2     1\n#>  3:      3     1\n#>  4:      4     2\n#>  5:      5     3\n#>  6:      6     1\n#>  7:      7     3\n#>  8:      8     3\n#>  9:      9     2\n#> 10:     10     1\n#> 11:     11     1\n#> 12:     12     2\n#> 13:     13     2\n#> 14:     14     1\n#> 15:     15     2\n#> 16:     16     1\n#> 17:     17     3\n#> 18:     18     1\n#> 19:     19     2\n#> 20:     20     1\n#> 21:     21     2\n#> 22:     22     3\n#> 23:     23     2\n#> 24:     24     3\n#> 25:     25     3\n#>     job.id chunk
    print(ids[, .N, by = chunk])\n
    #>    chunk N\n#> 1:     3 8\n#> 2:     1 9\n#> 3:     2 8
    \n### Group into 4 chunks\nids[, chunk := chunk(job.id, n.chunks = 4)]\n
    #>     job.id chunk\n#>  1:      1     2\n#>  2:      2     3\n#>  3:      3     4\n#>  4:      4     3\n#>  5:      5     4\n#>  6:      6     1\n#>  7:      7     4\n#>  8:      8     1\n#>  9:      9     2\n#> 10:     10     2\n#> 11:     11     3\n#> 12:     12     3\n#> 13:     13     4\n#> 14:     14     1\n#> 15:     15     3\n#> 16:     16     2\n#> 17:     17     1\n#> 18:     18     2\n#> 19:     19     3\n#> 20:     20     4\n#> 21:     21     1\n#> 22:     22     2\n#> 23:     23     4\n#> 24:     24     1\n#> 25:     25     1\n#>     job.id chunk
    print(ids[, .N, by = chunk])\n
    #>    chunk N\n#> 1:     2 6\n#> 2:     3 6\n#> 3:     4 6\n#> 4:     1 7
    \n### Submit to batch system\nsubmitJobs(ids = ids, reg = tmp)\n
    #> Submitting 25 jobs in 4 chunks using cluster functions 'Interactive' ...
    \n# Grouped chunking\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg2' using cluster functions 'Interactive'
    prob = addProblem(reg = tmp, \"prob1\", data = iris, fun = function(job, data) nrow(data))\n
    #> Adding problem 'prob1'
    prob = addProblem(reg = tmp, \"prob2\", data = Titanic, fun = function(job, data) nrow(data))\n
    #> Adding problem 'prob2'
    algo = addAlgorithm(reg = tmp, \"algo\", fun = function(job, data, instance, i, ...) problem)\n
    #> Adding algorithm 'algo'
    prob.designs = list(prob1 = data.table(), prob2 = data.table(x = 1:2))\nalgo.designs = list(algo = data.table(i = 1:3))\naddExperiments(prob.designs, algo.designs, repls = 3, reg = tmp)\n
    #> Adding 9 experiments ('prob1'[1] x 'algo'[3] x repls[3]) ...
    #> Adding 18 experiments ('prob2'[2] x 'algo'[3] x repls[3]) ...
    \n### Group into chunks of 5 jobs, but do not put multiple problems into the same chunk\n# -> only one problem has to be loaded per chunk, and only once because it is cached\nids = getJobTable(reg = tmp)[, .(job.id, problem, algorithm)]\nids[, chunk := chunk(job.id, chunk.size = 5), by = \"problem\"]\n
    #>     job.id problem algorithm chunk\n#>  1:      1   prob1      algo     1\n#>  2:      2   prob1      algo     1\n#>  3:      3   prob1      algo     2\n#>  4:      4   prob1      algo     2\n#>  5:      5   prob1      algo     1\n#>  6:      6   prob1      algo     2\n#>  7:      7   prob1      algo     1\n#>  8:      8   prob1      algo     1\n#>  9:      9   prob1      algo     2\n#> 10:     10   prob2      algo     2\n#> 11:     11   prob2      algo     1\n#> 12:     12   prob2      algo     1\n#> 13:     13   prob2      algo     3\n#> 14:     14   prob2      algo     3\n#> 15:     15   prob2      algo     3\n#> 16:     16   prob2      algo     2\n#> 17:     17   prob2      algo     2\n#> 18:     18   prob2      algo     2\n#> 19:     19   prob2      algo     2\n#> 20:     20   prob2      algo     4\n#> 21:     21   prob2      algo     1\n#> 22:     22   prob2      algo     1\n#> 23:     23   prob2      algo     3\n#> 24:     24   prob2      algo     4\n#> 25:     25   prob2      algo     1\n#> 26:     26   prob2      algo     4\n#> 27:     27   prob2      algo     4\n#>     job.id problem algorithm chunk
    ids[, chunk := .GRP, by = c(\"problem\", \"chunk\")]\n
    #>     job.id problem algorithm chunk\n#>  1:      1   prob1      algo     1\n#>  2:      2   prob1      algo     1\n#>  3:      3   prob1      algo     2\n#>  4:      4   prob1      algo     2\n#>  5:      5   prob1      algo     1\n#>  6:      6   prob1      algo     2\n#>  7:      7   prob1      algo     1\n#>  8:      8   prob1      algo     1\n#>  9:      9   prob1      algo     2\n#> 10:     10   prob2      algo     3\n#> 11:     11   prob2      algo     4\n#> 12:     12   prob2      algo     4\n#> 13:     13   prob2      algo     5\n#> 14:     14   prob2      algo     5\n#> 15:     15   prob2      algo     5\n#> 16:     16   prob2      algo     3\n#> 17:     17   prob2      algo     3\n#> 18:     18   prob2      algo     3\n#> 19:     19   prob2      algo     3\n#> 20:     20   prob2      algo     6\n#> 21:     21   prob2      algo     4\n#> 22:     22   prob2      algo     4\n#> 23:     23   prob2      algo     5\n#> 24:     24   prob2      algo     6\n#> 25:     25   prob2      algo     4\n#> 26:     26   prob2      algo     6\n#> 27:     27   prob2      algo     6\n#>     job.id problem algorithm chunk
    dcast(ids, chunk ~ problem)\n
    #> Using 'chunk' as value column. Use 'value.var' to override
    #> Aggregate function missing, defaulting to 'length'
    #>    chunk prob1 prob2\n#> 1:     1     5     0\n#> 2:     2     4     0\n#> 3:     3     0     5\n#> 4:     4     0     5\n#> 5:     5     0     4\n#> 6:     6     0     4
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/chunkIds.html", "content": "\n\n\n \n \n\n\n\nChunk Jobs for Sequential Execution — chunkIds • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.10\n \n
    \n\n
    \n \n \n \n \n
    \n
    \n
    \n\n \n
    \n\n
    \n
    \n
    \n

    Chunk Jobs for Sequential Execution

    \n Source: R/chunkIds.R\n
    chunkIds.Rd
    \n
    \n\n
    \n \n

    This function is deprecated in favor of the more flexible chunk, lpt and binpack.

    \n \n
    \n\n 
    chunkIds(ids = NULL, n.chunks = NULL, chunk.size = NULL,\n  group.by = character(0L), reg = getDefaultRegistry())
    \n \n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    n.chunks

    [integer(1)]
    \nRequested number of chunks.\nThe function chunk distributes the number of elements in x evenly while\nlpt tries to even out the sum of elements in each chunk.\nIf more chunks than necessary are requested, empty chunks are ignored.\nMutually exclusive with chunks.size.

    chunk.size

    [integer(1)]
    \nRequested chunk size for each single chunk.\nFor chunk this is the number of elements in x, for binpack the size\nis determined by the sum of values in x.\nMutually exclusive with n.chunks.

    group.by

    [character(0)]
    \nIf ids is a data.frame with additional columns\n(in addition to the required column “job.id”), then the chunking is performed using\nsubgroups defined by the columns set in group.by.\nSee example.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n \n

    Value

    \n\n

    [data.table] with columns “job.id” and “chunk”.

    \n \n

    See also

    \n\n

    chunk binpack lpt

    \n \n\n
    \n
    \n

    Contents

    \n \n\n
    \n
    \n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown.

    \n
    \n\n
    \n
    \n\n \n\n \n\n\n" }, { "path": "docs/reference/clearRegistry.html", "content": "\n\n\n \n \n\n\n\nRemove All Jobs — clearRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Remove All Jobs

    \n Source: R/clearRegistry.R\n
    clearRegistry.Rd
    \n
    \n\n
    \n

    Removes all jobs from a registry and calls sweepRegistry.

    \n
    \n\n 
    clearRegistry(reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    See also

    \n\n

    Other Registry: \ngetDefaultRegistry(),\nloadRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/doJobCollection.html", "content": "\n\n\n \n \n\n\n\nExecute Jobs of a JobCollection — doJobCollection • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Execute Jobs of a JobCollection

    \n Source: R/doJobCollection.R\n
    doJobCollection.Rd
    \n
    \n\n
    \n

    Executes every job in a JobCollection.\nThis function is intended to be called on the slave.

    \n
    \n\n 
    doJobCollection(jc, output = NULL)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    jc

    [JobCollection]
    \nEither an object of class “JobCollection” as returned by\nmakeJobCollection or a string with the path to file\ncontaining a “JobCollection” as RDS file (as stored by submitJobs).

    output

    [character(1)]
    \nPath to a file to write the output to. Defaults to NULL which means\nthat output is written to the active sink.\nDo not set this if your scheduler redirects output to a log file.

    \n\n

    Value

    \n\n

    [character(1)]: Hash of the JobCollection executed.

    \n

    See also

    \n\n

    Other JobCollection: \nmakeJobCollection()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(identity, 1:2, reg = tmp)\n
    #> Adding 2 jobs ...
    jc = makeJobCollection(1:2, reg = tmp)\ndoJobCollection(jc)\n
    #> ### [bt]: This is batchtools v0.9.14\n#> ### [bt]: Starting calculation of 2 jobs\n#> ### [bt]: Setting working directory to '/home/michel/Projekte/batchtools/docs/reference'\n#> ### [bt]: Memory measurement disabled\n#> ### [bt]: Starting job [batchtools job.id=1]\n#> ### [bt]: Setting seed to 1166 ...\n#> \n#> ### [bt]: Job terminated successfully [batchtools job.id=1]\n#> ### [bt]: Starting job [batchtools job.id=2]\n#> ### [bt]: Setting seed to 1167 ...\n#> \n#> ### [bt]: Job terminated successfully [batchtools job.id=2]\n#> ### [bt]: Calculation finished!
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/estimateRuntimes.html", "content": "\n\n\n \n \n\n\n\nEstimate Remaining Runtimes — estimateRuntimes • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Estimate Remaining Runtimes

    \n Source: R/estimateRuntimes.R\n
    estimateRuntimes.Rd
    \n
    \n\n
    \n

    Estimates the runtimes of jobs using the random forest implemented in ranger.\nObserved runtimes are retrieved from the Registry and runtimes are\npredicted for unfinished jobs.

    \n

    The estimated remaining time is calculated in the print method.\nYou may also pass n here to determine the number of parallel jobs which is then used\nin a simple Longest Processing Time (LPT) algorithm to give an estimate for the parallel runtime.

    \n
    \n\n 
    estimateRuntimes(tab, ..., reg = getDefaultRegistry())\n\n# S3 method for RuntimeEstimate\nprint(x, n = 1L, ...)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    tab

    [data.table]
    \nTable with column “job.id” and additional columns to predict the runtime.\nObserved runtimes will be looked up in the registry and serve as dependent variable.\nAll columns in tab except “job.id” will be passed to ranger as\nindependent variables to fit the model.

    ...

    [ANY]
    \nAdditional parameters passed to ranger. Ignored for the print method.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    x

    [RuntimeEstimate]
    \nObject to print.

    n

    [integer(1)]
    \nNumber of parallel jobs to assume for runtime estimation.

    \n\n

    Value

    \n\n

    [RuntimeEstimate] which is a list with two named elements:\n “runtimes” is a data.table with columns “job.id”,\n “runtime” (in seconds) and “type” (“estimated” if runtime is estimated,\n “observed” if runtime was observed).\n The other element of the list named “model”] contains the fitted random forest object.

    \n

    See also

    \n\n

    binpack and lpt to chunk jobs according to their estimated runtimes.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \n# Create a simple toy registry\nset.seed(1)\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE, seed = 1)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    addProblem(name = \"iris\", data = iris, fun = function(data, ...) nrow(data), reg = tmp)\n
    #> Adding problem 'iris'
    addAlgorithm(name = \"nrow\", function(instance, ...) nrow(instance), reg = tmp)\n
    #> Adding algorithm 'nrow'
    addAlgorithm(name = \"ncol\", function(instance, ...) ncol(instance), reg = tmp)\n
    #> Adding algorithm 'ncol'
    addExperiments(algo.designs = list(nrow = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\n
    #> Adding 250 experiments ('iris'[1] x 'nrow'[250] x repls[1]) ...
    addExperiments(algo.designs = list(ncol = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\n
    #> Adding 250 experiments ('iris'[1] x 'ncol'[250] x repls[1]) ...
    \n# We use the job parameters to predict runtimes\ntab = unwrap(getJobPars(reg = tmp))\n\n# First we need to submit some jobs so that the forest can train on some data.\n# Thus, we just sample some jobs from the registry while grouping by factor variables.\nlibrary(data.table)\nids = tab[, .SD[sample(nrow(.SD), 5)], by = c(\"problem\", \"algorithm\", \"y\")]\nsetkeyv(ids, \"job.id\")\nsubmitJobs(ids, reg = tmp)\n
    #> Submitting 50 jobs in 50 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    \n# We \"simulate\" some more realistic runtimes here to demonstrate the functionality:\n# - Algorithm \"ncol\" is 5 times more expensive than \"nrow\"\n# - x has no effect on the runtime\n# - If y is \"a\" or \"b\", the runtimes are really high\nruntime = function(algorithm, x, y) {\n  ifelse(algorithm == \"nrow\", 100L, 500L) + 1000L * (y %in% letters[1:2])\n}\ntmp$status[ids, done := done + tab[ids, runtime(algorithm, x, y)]]\n
    #>      job.id def.id  submitted    started       done error mem.used resource.id\n#>   1:      1      1         NA         NA         NA  <NA>       NA          NA\n#>   2:      2      2         NA         NA         NA  <NA>       NA          NA\n#>   3:      3      3         NA         NA         NA  <NA>       NA          NA\n#>   4:      4      4         NA         NA         NA  <NA>       NA          NA\n#>   5:      5      5         NA         NA         NA  <NA>       NA          NA\n#>  ---                                                                          \n#> 496:    496    496         NA         NA         NA  <NA>       NA          NA\n#> 497:    497    497         NA         NA         NA  <NA>       NA          NA\n#> 498:    498    498         NA         NA         NA  <NA>       NA          NA\n#> 499:    499    499 1603265964 1603265964 1603266464  <NA>       NA           1\n#> 500:    500    500         NA         NA         NA  <NA>       NA          NA\n#>           batch.id log.file                            job.hash job.name repl\n#>   1:          <NA>     <NA>                                <NA>     <NA>    1\n#>   2:          <NA>     <NA>                                <NA>     <NA>    1\n#>   3:          <NA>     <NA>                                <NA>     <NA>    1\n#>   4:          <NA>     <NA>                                <NA>     <NA>    1\n#>   5:          <NA>     <NA>                                <NA>     <NA>    1\n#>  ---                                                                         \n#> 496:          <NA>     <NA>                                <NA>     <NA>    1\n#> 497:          <NA>     <NA>                                <NA>     <NA>    1\n#> 498:          <NA>     <NA>                                <NA>     <NA>    1\n#> 499: cfInteractive     <NA> joba17949c0e00e62405c8465e973297f1c     <NA>    1\n#> 500:          <NA>     <NA>                                <NA>     <NA>    1
    rjoin(sjoin(tab, ids), getJobStatus(ids, reg = tmp)[, c(\"job.id\", \"time.running\")])\n
    #>     job.id problem algorithm  x y   time.running\n#>  1:     32    iris      nrow  7 b 1100.0026 secs\n#>  2:     42    iris      nrow  9 b 1100.0024 secs\n#>  3:     47    iris      nrow 10 b 1100.0023 secs\n#>  4:     66    iris      nrow 14 a 1100.0052 secs\n#>  5:     73    iris      nrow 15 c  100.0023 secs\n#>  6:     75    iris      nrow 15 e  100.0024 secs\n#>  7:     86    iris      nrow 18 a 1100.0025 secs\n#>  8:    100    iris      nrow 20 e  100.0026 secs\n#>  9:    101    iris      nrow 21 a 1100.0024 secs\n#> 10:    103    iris      nrow 21 c  100.0024 secs\n#> 11:    123    iris      nrow 25 c  100.0024 secs\n#> 12:    125    iris      nrow 25 e  100.0028 secs\n#> 13:    161    iris      nrow 33 a 1100.0026 secs\n#> 14:    165    iris      nrow 33 e  100.0026 secs\n#> 15:    169    iris      nrow 34 d  100.0026 secs\n#> 16:    183    iris      nrow 37 c  100.0027 secs\n#> 17:    184    iris      nrow 37 d  100.0027 secs\n#> 18:    203    iris      nrow 41 c  100.0036 secs\n#> 19:    207    iris      nrow 42 b 1100.0024 secs\n#> 20:    209    iris      nrow 42 d  100.0029 secs\n#> 21:    220    iris      nrow 44 e  100.0023 secs\n#> 22:    227    iris      nrow 46 b 1100.0024 secs\n#> 23:    229    iris      nrow 46 d  100.0023 secs\n#> 24:    231    iris      nrow 47 a 1100.0023 secs\n#> 25:    244    iris      nrow 49 d  100.0022 secs\n#> 26:    260    iris      ncol  2 e  500.0024 secs\n#> 27:    276    iris      ncol  6 a 1500.0025 secs\n#> 28:    278    iris      ncol  6 c  500.0025 secs\n#> 29:    279    iris      ncol  6 d  500.0024 secs\n#> 30:    296    iris      ncol 10 a 1500.0025 secs\n#> 31:    320    iris      ncol 14 e  500.0023 secs\n#> 32:    340    iris      ncol 18 e  500.0023 secs\n#> 33:    347    iris      ncol 20 b 1500.0023 secs\n#> 34:    363    iris      ncol 23 c  500.0023 secs\n#> 35:    369    iris      ncol 24 d  500.0023 secs\n#> 36:    373    iris      ncol 25 c  500.0025 secs\n#> 37:    387    iris      ncol 28 b 1500.0023 secs\n#> 38:    410    iris      ncol 32 e  500.0024 secs\n#> 39:    421    iris      ncol 35 a 1500.0024 secs\n#> 40:    436    iris      ncol 38 a 1500.0024 secs\n#> 41:    444    iris      ncol 39 d  500.0022 secs\n#> 42:    448    iris      ncol 40 c  500.0022 secs\n#> 43:    456    iris      ncol 42 a 1500.0023 secs\n#> 44:    459    iris      ncol 42 d  500.0023 secs\n#> 45:    467    iris      ncol 44 b 1500.0023 secs\n#> 46:    468    iris      ncol 44 c  500.0023 secs\n#> 47:    475    iris      ncol 45 e  500.0024 secs\n#> 48:    482    iris      ncol 47 b 1500.0023 secs\n#> 49:    492    iris      ncol 49 b 1500.0023 secs\n#> 50:    499    iris      ncol 50 d  500.0023 secs\n#>     job.id problem algorithm  x y   time.running
    \n# Estimate runtimes:\nest = estimateRuntimes(tab, reg = tmp)\nprint(est)\n
    #> Runtime Estimate for 500 jobs with 1 CPUs\n#>   Done     : 0d 09h 43m 20.1s\n#>   Remaining: 3d 17h 37m 8.0s\n#>   Total    : 4d 03h 20m 28.1s
    rjoin(tab, est$runtimes)\n
    #>      job.id problem algorithm  x y      type   runtime\n#>   1:      1    iris      nrow  1 a estimated 1107.0568\n#>   2:      2    iris      nrow  1 b estimated 1090.8508\n#>   3:      3    iris      nrow  1 c estimated  338.2092\n#>   4:      4    iris      nrow  1 d estimated  318.6349\n#>   5:      5    iris      nrow  1 e estimated  317.3189\n#>  ---                                                  \n#> 496:    496    iris      ncol 50 a estimated 1381.9162\n#> 497:    497    iris      ncol 50 b estimated 1389.1659\n#> 498:    498    iris      ncol 50 c estimated  614.0596\n#> 499:    499    iris      ncol 50 d  observed  500.0023\n#> 500:    500    iris      ncol 50 e estimated  574.7851
    print(est, n = 10)\n
    #> Runtime Estimate for 500 jobs with 10 CPUs\n#>   Done     : 0d 09h 43m 20.1s\n#>   Remaining: 3d 17h 37m 8.0s\n#>   Parallel : 0d 08h 58m 21.4s\n#>   Total    : 4d 03h 20m 28.1s
    \n# Submit jobs with longest runtime first:\nids = est$runtimes[type == \"estimated\"][order(runtime, decreasing = TRUE)]\nprint(ids)\n
    #>      job.id      type   runtime\n#>   1:    466 estimated 1420.0934\n#>   2:    461 estimated 1418.7001\n#>   3:    462 estimated 1415.5134\n#>   4:    457 estimated 1414.7134\n#>   5:    487 estimated 1413.4847\n#>  ---                           \n#> 446:    194 estimated  133.0456\n#> 447:    185 estimated  133.0030\n#> 448:    204 estimated  131.6954\n#> 449:    174 estimated  131.5901\n#> 450:    179 estimated  130.4434
    if (FALSE) {\nsubmitJobs(ids, reg = tmp)\n}\n\n# Group jobs into chunks with runtime < 1h\nids = est$runtimes[type == \"estimated\"]\nids[, chunk := binpack(runtime, 3600)]\n
    #>      job.id      type   runtime chunk\n#>   1:      1 estimated 1107.0568    47\n#>   2:      2 estimated 1090.8508    51\n#>   3:      3 estimated  338.2092    37\n#>   4:      4 estimated  318.6349    33\n#>   5:      5 estimated  317.3189    70\n#>  ---                                 \n#> 446:    495 estimated  581.7197    17\n#> 447:    496 estimated 1381.9162    20\n#> 448:    497 estimated 1389.1659    15\n#> 449:    498 estimated  614.0596     4\n#> 450:    500 estimated  574.7851    26
    print(ids)\n
    #>      job.id      type   runtime chunk\n#>   1:      1 estimated 1107.0568    47\n#>   2:      2 estimated 1090.8508    51\n#>   3:      3 estimated  338.2092    37\n#>   4:      4 estimated  318.6349    33\n#>   5:      5 estimated  317.3189    70\n#>  ---                                 \n#> 446:    495 estimated  581.7197    17\n#> 447:    496 estimated 1381.9162    20\n#> 448:    497 estimated 1389.1659    15\n#> 449:    498 estimated  614.0596     4\n#> 450:    500 estimated  574.7851    26
    print(ids[, list(runtime = sum(runtime)), by = chunk])\n
    #>     chunk  runtime\n#>  1:    47 3493.187\n#>  2:    51 3593.783\n#>  3:    37 3598.573\n#>  4:    33 3599.900\n#>  5:    70 3493.489\n#>  6:    53 3598.723\n#>  7:    71 3491.366\n#>  8:    48 3491.841\n#>  9:    52 3597.483\n#> 10:    54 3587.877\n#> 11:    68 3499.779\n#> 12:    72 3489.223\n#> 13:    55 3583.526\n#> 14:    69 3496.272\n#> 15:    73 3483.829\n#> 16:    46 3519.591\n#> 17:    50 3599.943\n#> 18:    38 3597.396\n#> 19:    65 3512.646\n#> 20:    43 3571.763\n#> 21:    62 3522.617\n#> 22:    66 3511.003\n#> 23:    39 3599.908\n#> 24:    35 3599.575\n#> 25:    61 3533.407\n#> 26:    40 3598.645\n#> 27:    56 3571.361\n#> 28:    57 3565.133\n#> 29:    49 3481.931\n#> 30:    42 3583.160\n#> 31:    58 3555.775\n#> 32:    60 3535.954\n#> 33:    41 3588.180\n#> 34:    36 3599.425\n#> 35:    59 3545.174\n#> 36:    44 3541.279\n#> 37:    34 3599.586\n#> 38:    64 3514.492\n#> 39:    45 3540.479\n#> 40:    63 3517.610\n#> 41:    67 3507.819\n#> 42:    27 3598.911\n#> 43:    24 3599.823\n#> 44:    25 3590.607\n#> 45:    26 3598.511\n#> 46:    23 3599.593\n#> 47:    28 3573.496\n#> 48:    75 3599.916\n#> 49:    12 3559.937\n#> 50:    74 3474.824\n#> 51:     8 3593.188\n#> 52:    20 3521.159\n#> 53:    31 3599.784\n#> 54:     7 3595.855\n#> 55:     5 3594.254\n#> 56:    11 3563.352\n#> 57:    10 3575.839\n#> 58:     6 3599.450\n#> 59:    32 3598.576\n#> 60:    80 3492.129\n#> 61:    82 3471.066\n#> 62:    83 3599.780\n#> 63:    79 3501.372\n#> 64:    76 3593.842\n#> 65:    85 3588.259\n#> 66:    89 3553.760\n#> 67:    91 2151.522\n#> 68:    81 3481.753\n#> 69:    78 3513.014\n#> 70:    87 3570.795\n#> 71:    88 3563.106\n#> 72:    77 3529.443\n#> 73:     3 3599.295\n#> 74:    86 3578.904\n#> 75:    90 3529.605\n#> 76:     2 3599.210\n#> 77:    84 3596.381\n#> 78:     1 3599.788\n#> 79:     4 3595.377\n#> 80:     9 3583.777\n#> 81:    29 3558.408\n#> 82:    18 3572.866\n#> 83:    15 3583.955\n#> 84:    21 3599.004\n#> 85:    19 3567.117\n#> 86:    16 3582.283\n#> 87:    30 3550.130\n#> 88:    17 3578.532\n#> 89:    22 3599.427\n#> 90:    13 3599.265\n#> 91:    14 3595.019\n#>     chunk  runtime
    if (FALSE) {\nsubmitJobs(ids, reg = tmp)\n}\n\n# Group jobs into 10 chunks with similar runtime\nids = est$runtimes[type == \"estimated\"]\nids[, chunk := lpt(runtime, 10)]\n
    #>      job.id      type   runtime chunk\n#>   1:      1 estimated 1107.0568     4\n#>   2:      2 estimated 1090.8508     9\n#>   3:      3 estimated  338.2092     4\n#>   4:      4 estimated  318.6349     8\n#>   5:      5 estimated  317.3189     6\n#>  ---                                 \n#> 446:    495 estimated  581.7197     2\n#> 447:    496 estimated 1381.9162     9\n#> 448:    497 estimated 1389.1659     2\n#> 449:    498 estimated  614.0596     2\n#> 450:    500 estimated  574.7851     1
    print(ids[, list(runtime = sum(runtime)), by = chunk])\n
    #>     chunk  runtime\n#>  1:     4 32227.40\n#>  2:     9 32226.68\n#>  3:     8 32231.22\n#>  4:     6 32293.22\n#>  5:     1 32226.47\n#>  6:     3 32292.92\n#>  7:    10 32227.16\n#>  8:     5 32301.32\n#>  9:     2 32301.36\n#> 10:     7 32300.22
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/execJob.html", "content": "\n\n\n \n \n\n\n\nExecute a Single Jobs — execJob • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Execute a Single Jobs

    \n Source: R/execJob.R\n
    execJob.Rd
    \n
    \n\n
    \n

    Executes a single job (as created by makeJob) and returns\nits result. Also works for Experiments.

    \n
    \n\n 
    execJob(job)
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    job

    [Job | Experiment]
    \nJob/Experiment to execute.

    \n\n

    Value

    \n\n

    Result of the job.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(identity, 1:2, reg = tmp)\n
    #> Adding 2 jobs ...
    job = makeJob(1, reg = tmp)\nexecJob(job)\n
    #> ### [bt]: Setting seed to 12825 ...
    #> [1] 1
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/findConfFile.html", "content": "\n\n\n \n \n\n\n\nFind a batchtools Configuration File — findConfFile • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Find a batchtools Configuration File

    \n Source: R/config.R\n
    findConfFile.Rd
    \n
    \n\n
    \n

    This functions returns the path to the first configuration file found in the following locations:

      \n

    1. File “batchtools.conf.R” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. File “batchtools.conf.R” in the current working directory.

      3. \n

    3. File “config.R” in the user configuration directory as reported by rappdirs::user_config_dir(\"batchtools\", expand = FALSE) (depending on OS, e.g., on linux this usually resolves to “~/.config/batchtools/config.R”).

      4. \n

    4. “.batchtools.conf.R” in the home directory (“~”).

      5. \n

    5. “config.R” in the site config directory as reported by rappdirs::site_config_dir(\"batchtools\") (depending on OS). This file can be used for admins to set sane defaults for a computation site.

      6. \n
    \n\n
    \n\n 
    findConfFile()
    \n\n\n

    Value

    \n\n

    [character(1)] Path to the configuration file or NA if no configuration file was found.

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/findJobs.html", "content": "\n\n\n \n \n\n\n\nFind and Filter Jobs — findJobs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Find and Filter Jobs

    \n Source: R/findJobs.R\n
    findJobs.Rd
    \n
    \n\n
    \n

    These functions are used to find and filter jobs, depending on either their parameters (findJobs and\nfindExperiments), their tags (findTagged), or their computational status (all other functions,\nsee getStatus for an overview).

    \n

    Note that findQueued, findRunning, findOnSystem and findExpired are somewhat heuristic\nand may report misleading results, depending on the state of the system and the ClusterFunctions implementation.

    \n

    See JoinTables for convenient set operations (unions, intersects, differences) on tables with job ids.

    \n
    \n\n 
    findJobs(expr, ids = NULL, reg = getDefaultRegistry())\n\nfindExperiments(\n  ids = NULL,\n  prob.name = NA_character_,\n  prob.pattern = NA_character_,\n  algo.name = NA_character_,\n  algo.pattern = NA_character_,\n  prob.pars,\n  algo.pars,\n  repls = NULL,\n  reg = getDefaultRegistry()\n)\n\nfindSubmitted(ids = NULL, reg = getDefaultRegistry())\n\nfindNotSubmitted(ids = NULL, reg = getDefaultRegistry())\n\nfindStarted(ids = NULL, reg = getDefaultRegistry())\n\nfindNotStarted(ids = NULL, reg = getDefaultRegistry())\n\nfindDone(ids = NULL, reg = getDefaultRegistry())\n\nfindNotDone(ids = NULL, reg = getDefaultRegistry())\n\nfindErrors(ids = NULL, reg = getDefaultRegistry())\n\nfindOnSystem(ids = NULL, reg = getDefaultRegistry())\n\nfindRunning(ids = NULL, reg = getDefaultRegistry())\n\nfindQueued(ids = NULL, reg = getDefaultRegistry())\n\nfindExpired(ids = NULL, reg = getDefaultRegistry())\n\nfindTagged(tags = character(0L), ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    expr

    [expression]
    \nPredicate expression evaluated in the job parameters.\nJobs for which expr evaluates to TRUE are returned.

    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    prob.name

    [character]
    \nExact name of the problem (no substring matching).\nIf not provided, all problems are matched.

    prob.pattern

    [character]
    \nRegular expression pattern to match problem names.\nIf not provided, all problems are matched.

    algo.name

    [character]
    \nExact name of the problem (no substring matching).\nIf not provided, all algorithms are matched.

    algo.pattern

    [character]
    \nRegular expression pattern to match algorithm names.\nIf not provided, all algorithms are matched.

    prob.pars

    [expression]
    \nPredicate expression evaluated in the problem parameters.

    algo.pars

    [expression]
    \nPredicate expression evaluated in the algorithm parameters.

    repls

    [integer]
    \nWhitelist of replication numbers. If not provided, all replications are matched.

    tags

    [character]
    \nReturn jobs which are tagged with any of the tags provided.

    \n\n

    Value

    \n\n

    [data.table] with column “job.id” containing matched jobs.

    \n

    See also

    \n\n

    getStatus JoinTables

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(identity, i = 1:3, reg = tmp)\n
    #> Adding 3 jobs ...
    ids = findNotSubmitted(reg = tmp)\n\n# get all jobs:\nfindJobs(reg = tmp)\n
    #>    job.id\n#> 1:      1\n#> 2:      2\n#> 3:      3
    \n# filter for jobs with parameter i >= 2\nfindJobs(i >= 2, reg = tmp)\n\n# filter on the computational status\nfindSubmitted(reg = tmp)\n
    #> Empty data.table (0 rows and 1 cols): job.id
    findNotDone(reg = tmp)\n
    #>    job.id\n#> 1:      1\n#> 2:      2\n#> 3:      3
    \n# filter on tags\naddJobTags(2:3, \"my_tag\", reg = tmp)\nfindTagged(tags = \"my_tag\", reg = tmp)\n
    #>    job.id\n#> 1:      2\n#> 2:      3
    \n# combine filter functions using joins\n# -> jobs which are not done and not tagged (using an anti-join):\najoin(findNotDone(reg = tmp), findTagged(\"my_tag\", reg = tmp))\n
    #>    job.id\n#> 1:      1
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/findTemplateFile.html", "content": "\n\n\n \n \n\n\n\nFind a batchtools Template File — findTemplateFile • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Find a batchtools Template File

    \n Source: R/clusterFunctions.R\n
    findTemplateFile.Rd
    \n
    \n\n
    \n

    This functions returns the path to a template file on the file system.

    \n
    \n\n 
    findTemplateFile(template)
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    \n\n

    Value

    \n\n

    [character] Path to the file or NA if no template template file was found.

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/getDefaultRegistry.html", "content": "\n\n\n \n \n\n\n\nGet and Set the Default Registry — getDefaultRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Get and Set the Default Registry

    \n Source: R/getDefaultRegistry.R\n
    getDefaultRegistry.Rd
    \n
    \n\n
    \n

    getDefaultRegistry returns the registry currently set as default (or\nstops with an exception if none is set). setDefaultRegistry sets\na registry as default.

    \n
    \n\n 
    getDefaultRegistry()\n\nsetDefaultRegistry(reg)
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\nloadRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/getErrorMessages.html", "content": "\n\n\n \n \n\n\n\nRetrieve Error Messages — getErrorMessages • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Retrieve Error Messages

    \n Source: R/getErrorMessages.R\n
    getErrorMessages.Rd
    \n
    \n\n
    \n

    Extracts error messages from the internal data base and returns them in a table.

    \n
    \n\n 
    getErrorMessages(\n  ids = NULL,\n  missing.as.error = FALSE,\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findErrors.\nInvalid ids are ignored.

    missing.as.error

    [logical(1)]
    \nTreat missing results as errors? If TRUE, the error message “[not terminated]” is imputed\nfor jobs which have not terminated. Default is FALSE

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with columns “job.id”, “terminated” (logical),\n “error” (logical) and “message” (string).

    \n

    See also

    \n\n

    Other debug: \ngetStatus(),\ngrepLogs(),\nkillJobs(),\nresetJobs(),\nshowLog(),\ntestJob()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    fun = function(i) if (i == 3) stop(i) else i\nids = batchMap(fun, i = 1:5, reg = tmp)\n
    #> Adding 5 jobs ...
    submitJobs(1:4, reg = tmp)\n
    #> Submitting 4 jobs in 4 chunks using cluster functions 'Interactive' ...
    #> Error in (function (i)  : 3
    waitForJobs(1:4, reg = tmp)\n
    #> [1] FALSE
    getErrorMessages(ids, reg = tmp)\n
    #>    job.id terminated error                     message\n#> 1:      1       TRUE FALSE                        <NA>\n#> 2:      2       TRUE FALSE                        <NA>\n#> 3:      3       TRUE  TRUE Error in (function (i)  : 3\n#> 4:      4       TRUE FALSE                        <NA>\n#> 5:      5      FALSE FALSE                        <NA>
    getErrorMessages(ids, missing.as.error = TRUE, reg = tmp)\n
    #>    job.id terminated error                     message\n#> 1:      1       TRUE FALSE                        <NA>\n#> 2:      2       TRUE FALSE                        <NA>\n#> 3:      3       TRUE  TRUE Error in (function (i)  : 3\n#> 4:      4       TRUE FALSE                        <NA>\n#> 5:      5      FALSE  TRUE            [not terminated]
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/getJobTable.html", "content": "\n\n\n \n \n\n\n\nQuery Job Information — getJobTable • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Query Job Information

    \n Source: R/JobTables.R\n
    getJobTable.Rd
    \n
    \n\n
    \n

    getJobStatus returns the internal table which stores information about the computational\nstatus of jobs, getJobPars a table with the job parameters, getJobResources a table\nwith the resources which were set to submit the jobs, and getJobTags the tags of the jobs\n(see Tags).

    \n

    getJobTable returns all these tables joined.

    \n
    \n\n 
    getJobTable(ids = NULL, reg = getDefaultRegistry())\n\ngetJobStatus(ids = NULL, reg = getDefaultRegistry())\n\ngetJobResources(ids = NULL, reg = getDefaultRegistry())\n\ngetJobPars(ids = NULL, reg = getDefaultRegistry())\n\ngetJobTags(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with the following columns (not necessarily in this order):

    \n
    job.id

    Unique Job ID as integer.

    \n
    submitted

    Time the job was submitted to the batch system as POSIXct.

    \n
    started

    Time the job was started on the batch system as POSIXct.

    \n
    done

    Time the job terminated (successfully or with an error) as POSIXct.

    \n
    error

    Either NA if the job terminated successfully or the error message.

    \n
    mem.used

    Estimate of the memory usage.

    \n
    batch.id

    Batch ID as reported by the scheduler.

    \n
    log.file

    Log file. If missing, defaults to [job.hash].log.

    \n
    job.hash

    Unique string identifying the job or chunk.

    \n
    time.queued

    Time in seconds (as difftime) the job was queued.

    \n
    time.running

    Time in seconds (as difftime) the job was running.

    \n
    pars

    List of parameters/arguments for this job.

    \n
    resources

    List of computational resources set for this job.

    \n
    tags

    Tags as joined string, delimited by “,”.

    \n
    problem

    Only for ExperimentRegistry: the problem identifier.

    \n
    algorithm

    Only for ExperimentRegistry: the algorithm identifier.

    \n \n
    \n\n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    f = function(x) if (x < 0) stop(\"x must be > 0\") else sqrt(x)\nbatchMap(f, x = c(-1, 0, 1), reg = tmp)\n
    #> Adding 3 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
    #> Error in (function (x)  : x must be > 0
    waitForJobs(reg = tmp)\n
    #> [1] FALSE
    addJobTags(1:2, \"tag1\", reg = tmp)\naddJobTags(2, \"tag2\", reg = tmp)\n\n# Complete table:\ngetJobTable(reg = tmp)\n
    #>    job.id           submitted             started                done\n#> 1:      1 2020-10-21 09:39:25 2020-10-21 09:39:25 2020-10-21 09:39:25\n#> 2:      2 2020-10-21 09:39:25 2020-10-21 09:39:25 2020-10-21 09:39:25\n#> 3:      3 2020-10-21 09:39:25 2020-10-21 09:39:25 2020-10-21 09:39:25\n#>                                      error mem.used      batch.id log.file\n#> 1: Error in (function (x)  : x must be > 0       NA cfInteractive     <NA>\n#> 2:                                    <NA>       NA cfInteractive     <NA>\n#> 3:                                    <NA>       NA cfInteractive     <NA>\n#>                               job.hash job.name      time.queued\n#> 1: job5adb5742954ec70e5dc4621612e5638e     <NA> 0.002799988 secs\n#> 2: job5c0144440e7359c3570442e591fd68a7     <NA> 0.002799988 secs\n#> 3: jobcedfc453688181879b03f66b5e6e5f25     <NA> 0.002799988 secs\n#>        time.running  job.pars resources      tags\n#> 1: 0.002099991 secs <list[1]> <list[0]>      tag1\n#> 2: 0.001900196 secs <list[1]> <list[0]> tag1,tag2\n#> 3: 0.001899958 secs <list[1]> <list[0]>      <NA>
    \n# Job parameters:\ngetJobPars(reg = tmp)\n
    #>    job.id  job.pars\n#> 1:      1 <list[1]>\n#> 2:      2 <list[1]>\n#> 3:      3 <list[1]>
    \n# Set and retrieve tags:\ngetJobTags(reg = tmp)\n
    #>    job.id      tags\n#> 1:      1      tag1\n#> 2:      2 tag1,tag2\n#> 3:      3      <NA>
    \n# Job parameters with tags right-joined:\nrjoin(getJobPars(reg = tmp), getJobTags(reg = tmp))\n
    #>    job.id  job.pars      tags\n#> 1:      1 <list[1]>      tag1\n#> 2:      2 <list[1]> tag1,tag2\n#> 3:      3 <list[1]>      <NA>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/getStatus.html", "content": "\n\n\n \n \n\n\n\nSummarize the Computational Status — getStatus • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Summarize the Computational Status

    \n Source: R/getStatus.R\n
    getStatus.Rd
    \n
    \n\n
    \n

    This function gives an encompassing overview over the computational status on your system.\nThe status can be one or many of the following:

      \n

    • “defined”: Jobs which are defined via batchMap or addExperiments, but are not yet submitted.

      • \n

    • “submitted”: Jobs which are submitted to the batch system via submitJobs, scheduled for execution.

      • \n

    • “started”: Jobs which have been started.

      • \n

    • “done”: Jobs which terminated successfully.

      • \n

    • “error”: Jobs which terminated with an exception.

      • \n

    • “running”: Jobs which are listed by the cluster functions to be running on the live system. Not supported for all cluster functions.

      • \n

    • “queued”: Jobs which are listed by the cluster functions to be queued on the live system. Not supported for all cluster functions.

      • \n

    • “system”: Jobs which are listed by the cluster functions to be queued or running. Not supported for all cluster functions.

      • \n

    • “expired”: Jobs which have been submitted, but vanished from the live system. Note that this is determined heuristically and may include some false positives.

      • \n

    Here, a job which terminated successfully counts towards the jobs which are submitted, started and done.\nTo retrieve the corresponding job ids, see findJobs.

    \n
    \n\n 
    getStatus(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] (with class “Status” for printing).

    \n

    See also

    \n\n

    findJobs

    \n

    Other debug: \ngetErrorMessages(),\ngrepLogs(),\nkillJobs(),\nresetJobs(),\nshowLog(),\ntestJob()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    fun = function(i) if (i == 3) stop(i) else i\nids = batchMap(fun, i = 1:5, reg = tmp)\n
    #> Adding 5 jobs ...
    submitJobs(ids = 1:4, reg = tmp)\n
    #> Submitting 4 jobs in 4 chunks using cluster functions 'Interactive' ...
    #> Error in (function (i)  : 3
    waitForJobs(reg = tmp)\n
    #> [1] FALSE
    \ntab = getStatus(reg = tmp)\nprint(tab)\n
    #> Status for 5 jobs at 2020-10-21 09:39:26:\n#>   Submitted    : 4 ( 80.0%)\n#>   -- Queued    : 0 (  0.0%)\n#>   -- Started   : 4 ( 80.0%)\n#>   ---- Running : 0 (  0.0%)\n#>   ---- Done    : 3 ( 60.0%)\n#>   ---- Error   : 1 ( 20.0%)\n#>   ---- Expired : 0 (  0.0%)
    str(tab)\n
    #> Classes ‘Status’, ‘data.table’ and 'data.frame':\t1 obs. of  9 variables:\n#>  $ defined  : int 5\n#>  $ submitted: int 4\n#>  $ started  : int 4\n#>  $ done     : int 3\n#>  $ error    : int 1\n#>  $ queued   : int 0\n#>  $ running  : int 0\n#>  $ expired  : int 0\n#>  $ system   : int 0\n#>  - attr(*, \".internal.selfref\")=<externalptr>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/grepLogs.html", "content": "\n\n\n \n \n\n\n\nGrep Log Files for a Pattern — grepLogs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Grep Log Files for a Pattern

    \n Source: R/Logs.R\n
    grepLogs.Rd
    \n
    \n\n
    \n

    Crawls through log files and reports jobs with lines matching the pattern.\nSee showLog for an example.

    \n
    \n\n 
    grepLogs(\n  ids = NULL,\n  pattern,\n  ignore.case = FALSE,\n  fixed = FALSE,\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findStarted.\nInvalid ids are ignored.

    pattern

    [character(1L)]
    \nRegular expression or string (see fixed).

    ignore.case

    [logical(1L)]
    \nIf TRUE the match will be performed case insensitively.

    fixed

    [logical(1L)]
    \nIf FALSE (default), pattern is a regular expression and a fixed string otherwise.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with columns “job.id” and “message”.

    \n

    See also

    \n\n

    Other debug: \ngetErrorMessages(),\ngetStatus(),\nkillJobs(),\nresetJobs(),\nshowLog(),\ntestJob()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/index.html", "content": "\n\n\n \n \n\n\n\nFunction reference • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Reference

    \n
    \n\n \n\n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    \n

    Overview

    \n

    \n
    \n

    batchtools-package

    \n

    batchtools: Tools for Computation on Batch Systems

    \n

    Registry

    \n

    \n
    \n

    makeRegistry()

    \n

    Registry Constructor

    \n

    makeExperimentRegistry()

    \n

    ExperimentRegistry Constructor

    \n

    assertRegistry()

    \n

    assertRegistry

    \n

    loadRegistry()

    \n

    Load a Registry from the File System

    \n

    saveRegistry()

    \n

    Store the Registy to the File System

    \n

    syncRegistry()

    \n

    Synchronize the Registry

    \n

    sweepRegistry()

    \n

    Check Consistency and Remove Obsolete Information

    \n

    removeRegistry()

    \n

    Remove a Registry from the File System

    \n

    getDefaultRegistry() setDefaultRegistry()

    \n

    Get and Set the Default Registry

    \n

    Define Jobs

    \n

    \n
    \n

    batchMap()

    \n

    Map Operation for Batch Systems

    \n

    batchReduce()

    \n

    Reduce Operation for Batch Systems

    \n

    batchMapResults()

    \n

    Map Over Results to Create New Jobs

    \n

    addProblem() removeProblems()

    \n

    Define Problems for Experiments

    \n

    addAlgorithm() removeAlgorithms()

    \n

    Define Algorithms for Experiments

    \n

    addExperiments()

    \n

    Add Experiments to the Registry

    \n

    Submit Jobs

    \n

    \n
    \n

    submitJobs()

    \n

    Submit Jobs to the Batch Systems

    \n

    batchExport()

    \n

    Export Objects to the Slaves

    \n

    waitForJobs()

    \n

    Wait for Termination of Jobs

    \n

    chunk() lpt() binpack()

    \n

    Chunk Jobs for Sequential Execution

    \n

    setJobNames() getJobNames()

    \n

    Set and Retrieve Job Names

    \n

    Query Job Information

    \n

    \n
    \n

    getStatus()

    \n

    Summarize the Computational Status

    \n

    findJobs() findExperiments() findSubmitted() findNotSubmitted() findStarted() findNotStarted() findDone() findNotDone() findErrors() findOnSystem() findRunning() findQueued() findExpired() findTagged()

    \n

    Find and Filter Jobs

    \n

    getJobTable() getJobStatus() getJobResources() getJobPars() getJobTags()

    \n

    Query Job Information

    \n

    summarizeExperiments()

    \n

    Quick Summary over Experiments

    \n

    Retrieve Results

    \n

    \n
    \n

    reduceResults()

    \n

    Reduce Results

    \n

    reduceResultsList() reduceResultsDataTable()

    \n

    Apply Functions on Results

    \n

    loadResult()

    \n

    Load the Result of a Single Job

    \n

    unwrap() flatten()

    \n

    Unwrap Nested Data Frames

    \n

    Debugging

    \n

    \n
    \n

    resetJobs()

    \n

    Reset the Computational State of Jobs

    \n

    testJob()

    \n

    Run Jobs Interactively

    \n

    showLog() getLog()

    \n

    Inspect Log Files

    \n

    getErrorMessages()

    \n

    Retrieve Error Messages

    \n

    grepLogs()

    \n

    Grep Log Files for a Pattern

    \n

    Remove Jobs

    \n

    \n
    \n

    killJobs()

    \n

    Kill Jobs

    \n

    clearRegistry()

    \n

    Remove All Jobs

    \n

    removeExperiments()

    \n

    Remove Experiments

    \n

    Additional objects

    \n

    \n
    \n

    makeJob()

    \n

    Jobs and Experiments

    \n

    makeJobCollection()

    \n

    JobCollection Constructor

    \n

    Cluster Functions

    \n

    \n
    \n

    cfKillJob()

    \n

    Cluster Functions Helper to Kill Batch Jobs

    \n

    cfBrewTemplate()

    \n

    Cluster Functions Helper to Write Job Description Files

    \n

    cfReadBrewTemplate()

    \n

    Cluster Functions Helper to Parse a Brew Template

    \n

    cfHandleUnknownSubmitError()

    \n

    Cluster Functions Helper to Handle Unknown Errors

    \n

    makeClusterFunctions()

    \n

    ClusterFunctions Constructor

    \n

    makeClusterFunctionsDocker()

    \n

    ClusterFunctions for Docker

    \n

    makeClusterFunctionsInteractive()

    \n

    ClusterFunctions for Sequential Execution in the Running R Session

    \n

    makeClusterFunctionsLSF()

    \n

    ClusterFunctions for LSF Systems

    \n

    makeClusterFunctionsMulticore()

    \n

    ClusterFunctions for Parallel Multicore Execution

    \n

    makeClusterFunctionsOpenLava()

    \n

    ClusterFunctions for OpenLava

    \n

    makeClusterFunctionsSGE()

    \n

    ClusterFunctions for SGE Systems

    \n

    makeClusterFunctionsSSH()

    \n

    ClusterFunctions for Remote SSH Execution

    \n

    makeClusterFunctionsSlurm()

    \n

    ClusterFunctions for Slurm Systems

    \n

    makeClusterFunctionsSocket()

    \n

    ClusterFunctions for Parallel Socket Execution

    \n

    makeClusterFunctionsTORQUE()

    \n

    ClusterFunctions for OpenPBS/TORQUE Systems

    \n

    makeSubmitJobResult()

    \n

    Create a SubmitJobResult

    \n

    runHook()

    \n

    Trigger Evaluation of Custom Function

    \n

    Worker

    \n

    Create a Linux-Worker

    \n

    Miscellaneous

    \n

    \n
    \n

    addJobTags() removeJobTags() getUsedJobTags()

    \n

    Add or Remove Job Tags

    \n

    btlapply() btmapply()

    \n

    Synchronous Apply Functions

    \n

    ijoin() ljoin() rjoin() ojoin() sjoin() ajoin() ujoin()

    \n

    Inner, Left, Right, Outer, Semi and Anti Join for Data Tables

    \n

    runOSCommand()

    \n

    Run OS Commands on Local or Remote Machines

    \n

    execJob()

    \n

    Execute a Single Jobs

    \n

    doJobCollection()

    \n

    Execute Jobs of a JobCollection

    \n

    estimateRuntimes() print(<RuntimeEstimate>)

    \n

    Estimate Remaining Runtimes

    \n
    \n\n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/killJobs.html", "content": "\n\n\n \n \n\n\n\nKill Jobs — killJobs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Kill Jobs

    \n Source: R/killJobs.R\n
    killJobs.Rd
    \n
    \n\n
    \n

    Kill jobs which are currently running on the batch system.

    \n

    In case of an error when killing, the function tries - after a short sleep - to kill the remaining\nbatch jobs again. If this fails three times for some jobs, the function gives up. Jobs that could be\nsuccessfully killed are reset in the Registry.

    \n
    \n\n 
    killJobs(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findOnSystem.\nInvalid ids are ignored.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with columns “job.id”, the corresponding “batch.id” and\n the logical flag “killed” indicating success.

    \n

    See also

    \n\n

    Other debug: \ngetErrorMessages(),\ngetStatus(),\ngrepLogs(),\nresetJobs(),\nshowLog(),\ntestJob()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/loadRegistry.html", "content": "\n\n\n \n \n\n\n\nLoad a Registry from the File System — loadRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Load a Registry from the File System

    \n Source: R/loadRegistry.R\n
    loadRegistry.Rd
    \n
    \n\n
    \n

    Loads a registry from its file.dir.

    \n

    Multiple R sessions accessing the same registry simultaneously can lead to database inconsistencies.\nThis is especially dangerous if the same file.dir is accessed from multiple machines, e.g. via a mount.

    \n

    If you just need to check on the status or peek into some preliminary results while another process is still submitting or waiting\nfor pending results, you can load the registry in a read-only mode.\nAll operations that need to change the registry will raise an exception in this mode.\nFiles communicated back by the computational nodes are parsed to update the registry in memory while the registry on the file system remains unchanged.

    \n

    A heuristic tries to detect if the registry has been altered in the background by an other process and in this case automatically restricts the current registry to read-only mode.\nHowever, you should rely on this heuristic to work flawlessly.\nThus, set to writeable to TRUE if and only if you are absolutely sure that other state-changing processes are terminated.

    \n

    If you need write access, load the registry with writeable set to TRUE.

    \n
    \n\n 
    loadRegistry(\n  file.dir,\n  work.dir = NULL,\n  conf.file = findConfFile(),\n  make.default = TRUE,\n  writeable = FALSE\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    file.dir

    [character(1)]
    \n Path where all files of the registry are saved.\n Default is directory “registry” in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    \n

    If you pass NA, a temporary directory will be used.\n This way, you can create disposable registries for btlapply or examples.\n By default, the temporary directory tempdir() will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable temp.dir.

    work.dir

    [character(1)]
    \n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see getwd).\n loadRegistry uses the stored work.dir, but you may also explicitly overwrite it,\n e.g., after switching to another system.

    \n

    The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    conf.file

    [character(1)]
    \n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how batchtools interacts with the system via ClusterFunctions.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.

    \n

    The file lookup is implemented in the internal (but exported) function findConfFile which returns the first file found of the following candidates:

      \n

    1. File “batchtools.conf.R” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. File “batchtools.conf.R” in the current working directory.

      3. \n

    3. File “config.R” in the user configuration directory as reported by rappdirs::user_config_dir(\"batchtools\", expand = FALSE) (depending on OS, e.g., on linux this usually resolves to “~/.config/batchtools/config.R”).

      4. \n

    4. “.batchtools.conf.R” in the home directory (“~”).

      5. \n

    5. “config.R” in the site config directory as reported by rappdirs::site_config_dir(\"batchtools\") (depending on OS). This file can be used for admins to set sane defaults for a computation site.

      6. \n

    Set to NA if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. default.resources = list(walltime = 3600) to set default resources or “max.concurrent.jobs” to\n limit the number of jobs allowed to run simultaneously on the system.

    make.default

    [logical(1)]
    \nIf set to TRUE, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is TRUE.

    writeable

    [logical(1)]
    \nLoads the registry in read-write mode. Default is FALSE.

    \n\n

    Value

    \n\n

    [Registry].

    \n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/loadResult.html", "content": "\n\n\n \n \n\n\n\nLoad the Result of a Single Job — loadResult • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Load the Result of a Single Job

    \n Source: R/loadResult.R\n
    loadResult.Rd
    \n
    \n\n
    \n

    Loads the result of a single job.

    \n
    \n\n 
    loadResult(id, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    id

    [integer(1) or data.table]
    \nSingle integer to specify the job or a data.table with column job.id\nand exactly one row.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [ANY]. The stored result.

    \n

    See also

    \n\n

    Other Results: \nbatchMapResults(),\nreduceResultsList(),\nreduceResults()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctions.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions Constructor — makeClusterFunctions • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions Constructor

    \n Source: R/clusterFunctions.R\n
    makeClusterFunctions.Rd
    \n
    \n\n
    \n

    This is the constructor used to create custom cluster functions.\nNote that some standard implementations for TORQUE, Slurm, LSF, SGE, etc. ship\nwith the package.

    \n
    \n\n 
    makeClusterFunctions(\n  name,\n  submitJob,\n  killJob = NULL,\n  listJobsQueued = NULL,\n  listJobsRunning = NULL,\n  array.var = NA_character_,\n  store.job.collection = FALSE,\n  store.job.files = FALSE,\n  scheduler.latency = 0,\n  fs.latency = 0,\n  hooks = list()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    name

    [character(1)]
    \nName of cluster functions.

    submitJob

    [function(reg, jc, ...)]
    \nFunction to submit new jobs. Must return a SubmitJobResult object.\nThe arguments are reg (Registry) and jobs (JobCollection).

    killJob

    [function(reg, batch.id)]
    \nFunction to kill a job on the batch system. Make sure that you definitely kill the job! Return\nvalue is currently ignored. Must have the arguments reg (Registry) and\nbatch.id (character(1) as returned by submitJob).\nNote that there is a helper function cfKillJob to repeatedly try to kill jobs.\nSet killJob to NULL if killing jobs cannot be supported.

    listJobsQueued

    [function(reg)]
    \nList all queued jobs on the batch system for the current user.\nMust return an character vector of batch ids, same format as they\nare returned by submitJob.\nSet listJobsQueued to NULL if listing of queued jobs is not supported.

    listJobsRunning

    [function(reg)]
    \nList all running jobs on the batch system for the current user.\nMust return an character vector of batch ids, same format as they\nare returned by submitJob. It does not matter if you return a few job ids too many (e.g.\nall for the current user instead of all for the current registry), but you have to include all\nrelevant ones. Must have the argument are reg (Registry).\nSet listJobsRunning to NULL if listing of running jobs is not supported.

    array.var

    [character(1)]
    \nName of the environment variable set by the scheduler to identify IDs of job arrays.\nDefault is NA for no array support.

    store.job.collection

    [logical(1)]
    \nFlag to indicate that the cluster function implementation of submitJob can not directly handle JobCollection objects.\nIf set to FALSE, the JobCollection is serialized to the file system before submitting the job.

    store.job.files

    [logical(1)]
    \nFlag to indicate that job files need to be stored in the file directory.\nIf set to FALSE (default), the job file is created in a temporary directory, otherwise (or if the debug mode is enabled) in\nthe subdirectory jobs of the file.dir.

    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    hooks

    [list]
    \nNamed list of functions which will we called on certain events like “pre.submit” or “post.sync”.\nSee Hooks.

    \n\n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE()

    \n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfHandleUnknownSubmitError(),\ncfKillJob(),\ncfReadBrewTemplate(),\nmakeSubmitJobResult(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsDocker.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Docker — makeClusterFunctionsDocker • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Docker

    \n Source: R/clusterFunctionsDocker.R\n
    makeClusterFunctionsDocker.Rd
    \n
    \n\n
    \n

    Cluster functions for Docker/Docker Swarm (https://docs.docker.com/engine/swarm/).

    \n

    The submitJob function executes\ndocker [docker.args] run --detach=true [image.args] [resources] [image] [cmd].\nArguments docker.args, image.args and image can be set on construction.\nThe resources part takes the named resources ncpus and memory\nfrom submitJobs and maps them to the arguments --cpu-shares and --memory\n(in Megabytes). The resource threads is mapped to the environment variables “OMP_NUM_THREADS”\nand “OPENBLAS_NUM_THREADS”.\nTo reliably identify jobs in the swarm, jobs are labeled with “batchtools=[job.hash]” and named\nusing the current login name (label “user”) and the job hash (label “batchtools”).

    \n

    listJobsRunning uses docker [docker.args] ps --format={{.ID}} to filter for running jobs.

    \n

    killJobs uses docker [docker.args] kill [batch.id] to filter for running jobs.

    \n

    These cluster functions use a Hook to remove finished jobs before a new submit and every time the Registry\nis synchronized (using syncRegistry).\nThis is currently required because docker does not remove terminated containers automatically.\nUse docker ps -a --filter 'label=batchtools' --filter 'status=exited' to identify and remove terminated\ncontainers manually (or usa a cron job).

    \n
    \n\n 
    makeClusterFunctionsDocker(\n  image,\n  docker.args = character(0L),\n  image.args = character(0L),\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    image

    [character(1)]
    \nName of the docker image to run.

    docker.args

    [character]
    \nAdditional arguments passed to “docker” *before* the command (“run”, “ps” or “kill”) to execute (e.g., the docker host).

    image.args

    [character]
    \nAdditional arguments passed to “docker run” (e.g., to define mounts or environment variables).

    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsInteractive.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Sequential Execution in the Running R Session — makeClusterFunctionsInteractive • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Sequential Execution in the Running R Session

    \n Source: R/clusterFunctionsInteractive.R\n
    makeClusterFunctionsInteractive.Rd
    \n
    \n\n
    \n

    All jobs are executed sequentially using the current R process in which submitJobs is called.\nThus, submitJob blocks the session until the job has finished.\nThe main use of this ClusterFunctions implementation is to test and debug programs on a local computer.

    \n

    Listing jobs returns an empty vector (as no jobs can be running when you call this)\nand killJob is not implemented for the same reasons.

    \n
    \n\n 
    makeClusterFunctionsInteractive(\n  external = FALSE,\n  write.logs = TRUE,\n  fs.latency = 0\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    external

    [logical(1)]
    \nIf set to TRUE, jobs are started in a fresh R session instead of currently active but still\nwaits for its termination.\nDefault is FALSE.

    write.logs

    [logical(1)]
    \nSink the output to log files. Turning logging off can increase the speed of\ncalculations but makes it very difficult to debug.\nDefault is TRUE.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsLSF.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for LSF Systems — makeClusterFunctionsLSF • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for LSF Systems

    \n Source: R/clusterFunctionsLSF.R\n
    makeClusterFunctionsLSF.Rd
    \n
    \n\n
    \n

    Cluster functions for LSF (https://www.ibm.com/products/hpc-workload-management).

    \n

    Job files are created based on the brew template template.file. This\nfile is processed with brew and then submitted to the queue using the\nbsub command. Jobs are killed using the bkill command and the\nlist of running jobs is retrieved using bjobs -u $USER -w. The user\nmust have the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).

    \n

    The template file can access all resources passed to submitJobs\nas well as all variables stored in the JobCollection.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.

    \n
    \n\n 
    makeClusterFunctionsLSF(\n  template = \"lsf\",\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    Note

    \n\n

    Array jobs are currently not supported.

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsMulticore.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Parallel Multicore Execution — makeClusterFunctionsMulticore • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Parallel Multicore Execution

    \n Source: R/clusterFunctionsMulticore.R\n
    makeClusterFunctionsMulticore.Rd
    \n
    \n\n
    \n

    Jobs are spawned asynchronously using the functions mcparallel and mccollect (both in parallel).\nDoes not work on Windows, use makeClusterFunctionsSocket instead.

    \n
    \n\n 
    makeClusterFunctionsMulticore(ncpus = NA_integer_, fs.latency = 0)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ncpus

    [integer(1)]
    \nNumber of CPUs.\nDefault is to use all logical cores. The total number of cores \"available\" can be set via the option mc.cores\nand defaults to the heuristic implemented in detectCores.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsOpenLava.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for OpenLava — makeClusterFunctionsOpenLava • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for OpenLava

    \n Source: R/clusterFunctionsOpenLava.R\n
    makeClusterFunctionsOpenLava.Rd
    \n
    \n\n
    \n

    Cluster functions for OpenLava.

    \n

    Job files are created based on the brew template template. This\nfile is processed with brew and then submitted to the queue using the\nbsub command. Jobs are killed using the bkill command and the\nlist of running jobs is retrieved using bjobs -u $USER -w. The user\nmust have the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).

    \n

    The template file can access all resources passed to submitJobs\nas well as all variables stored in the JobCollection.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.

    \n
    \n\n 
    makeClusterFunctionsOpenLava(\n  template = \"openlava\",\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    Note

    \n\n

    Array jobs are currently not supported.

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsSGE.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for SGE Systems — makeClusterFunctionsSGE • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for SGE Systems

    \n Source: R/clusterFunctionsSGE.R\n
    makeClusterFunctionsSGE.Rd
    \n
    \n\n
    \n

    Cluster functions for Univa Grid Engine / Oracle Grid Engine /\nSun Grid Engine (https://www.univa.com/).

    \n

    Job files are created based on the brew template template. This\nfile is processed with brew and then submitted to the queue using the\nqsub command. Jobs are killed using the qdel command and the\nlist of running jobs is retrieved using qselect. The user must have\nthe appropriate privileges to submit, delete and list jobs on the cluster\n(this is usually the case).

    \n

    The template file can access all resources passed to submitJobs\nas well as all variables stored in the JobCollection.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.

    \n
    \n\n 
    makeClusterFunctionsSGE(\n  template = \"sge\",\n  nodename = \"localhost\",\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    nodename

    [character(1)]
    \nNodename of the master host. All commands are send via SSH to this host. Only works iff

      \n

    1. Passwordless authentication (e.g., via SSH public key authentication) is set up.

      2. \n

    2. The file directory is shared across machines, e.g. mounted via SSHFS.

      3. \n

    3. Either the absolute path to the file.dir is identical on the machines, or paths are provided relative to the home directory. Symbolic links should work.

      4. \n
    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    Note

    \n\n

    Array jobs are currently not supported.

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsSSH.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Remote SSH Execution — makeClusterFunctionsSSH • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Remote SSH Execution

    \n Source: R/clusterFunctionsSSH.R\n
    makeClusterFunctionsSSH.Rd
    \n
    \n\n
    \n

    Jobs are spawned by starting multiple R sessions via Rscript over SSH.\nIf the hostname of the Worker equals “localhost”,\nRscript is called directly so that you do not need to have an SSH client installed.

    \n
    \n\n 
    makeClusterFunctionsSSH(workers, fs.latency = 65)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    workers

    [list of Worker]
    \nList of Workers as constructed with Worker.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    Note

    \n\n

    If you use a custom “.ssh/config” file, make sure your\nProxyCommand passes ‘-q’ to ssh, otherwise each output will\nend with the message “Killed by signal 1” and this will break\nthe communication with the nodes.

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n

    Examples

    \n 
    if (FALSE) {\n# cluster functions for multicore execution on the local machine\nmakeClusterFunctionsSSH(list(Worker$new(\"localhost\", ncpus = 2)))\n}\n
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsSlurm.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Slurm Systems — makeClusterFunctionsSlurm • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Slurm Systems

    \n Source: R/clusterFunctionsSlurm.R\n
    makeClusterFunctionsSlurm.Rd
    \n
    \n\n
    \n

    Cluster functions for Slurm (https://slurm.schedmd.com/).

    \n

    Job files are created based on the brew template template.file. This\nfile is processed with brew and then submitted to the queue using the\nsbatch command. Jobs are killed using the scancel command and\nthe list of running jobs is retrieved using squeue. The user must\nhave the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).

    \n

    The template file can access all resources passed to submitJobs\nas well as all variables stored in the JobCollection.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.

    \n

    Note that you might have to specify the cluster name here if you do not want to use the default,\notherwise the commands for listing and killing jobs will not work.

    \n
    \n\n 
    makeClusterFunctionsSlurm(\n  template = \"slurm\",\n  array.jobs = TRUE,\n  nodename = \"localhost\",\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    array.jobs

    [logical(1)]
    \nIf array jobs are disabled on the computing site, set to FALSE.

    nodename

    [character(1)]
    \nNodename of the master host. All commands are send via SSH to this host. Only works iff

      \n

    1. Passwordless authentication (e.g., via SSH public key authentication) is set up.

      2. \n

    2. The file directory is shared across machines, e.g. mounted via SSHFS.

      3. \n

    3. Either the absolute path to the file.dir is identical on the machines, or paths are provided relative to the home directory. Symbolic links should work.

      4. \n
    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsSocket.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for Parallel Socket Execution — makeClusterFunctionsSocket • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for Parallel Socket Execution

    \n Source: R/clusterFunctionsSocket.R\n
    makeClusterFunctionsSocket.Rd
    \n
    \n\n
    \n

    Jobs are spawned asynchronously using the package snow.

    \n
    \n\n 
    makeClusterFunctionsSocket(ncpus = NA_integer_, fs.latency = 65)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ncpus

    [integer(1)]
    \nNumber of CPUs.\nDefault is to use all logical cores. The total number of cores \"available\" can be set via the option mc.cores\nand defaults to the heuristic implemented in detectCores.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsTORQUE(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeClusterFunctionsTORQUE.html", "content": "\n\n\n \n \n\n\n\nClusterFunctions for OpenPBS/TORQUE Systems — makeClusterFunctionsTORQUE • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ClusterFunctions for OpenPBS/TORQUE Systems

    \n Source: R/clusterFunctionsTORQUE.R\n
    makeClusterFunctionsTORQUE.Rd
    \n
    \n\n
    \n

    Cluster functions for TORQUE/PBS (https://adaptivecomputing.com/cherry-services/torque-resource-manager/).

    \n

    Job files are created based on the brew template template.file. This file is processed\nwith brew and then submitted to the queue using the qsub command. Jobs are killed using\nthe qdel command and the list of running jobs is retrieved using qselect. The user\nmust have the appropriate privileges to submit, delete and list jobs on the cluster (this is\nusually the case).

    \n

    The template file can access all resources passed to submitJobs\nas well as all variables stored in the JobCollection.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.

    \n
    \n\n 
    makeClusterFunctionsTORQUE(\n  template = \"torque\",\n  scheduler.latency = 1,\n  fs.latency = 65\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    template

    [character(1)]
    \nEither a path to a brew template file (with extension “tmpl”), or a short descriptive name enabling the following heuristic for the file lookup:

      \n

    1. “batchtools.[template].tmpl” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. “batchtools.[template].tmpl” in the current working directory.

      3. \n

    3. “[template].tmpl” in the user config directory (see user_config_dir); on linux this is usually “~/.config/batchtools/[template].tmpl”.

      4. \n

    4. “.batchtools.[template].tmpl” in the home directory.

      5. \n

    5. “[template].tmpl” in the package installation directory in the subfolder “templates”.

      6. \n
    scheduler.latency

    [numeric(1)]
    \nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling submitJobs.

    fs.latency

    [numeric(1)]
    \nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to 0 to disable the heuristic, e.g. if you are working on a local file system.

    \n\n

    Value

    \n\n

    [ClusterFunctions].

    \n

    See also

    \n\n

    Other ClusterFunctions: \nmakeClusterFunctionsDocker(),\nmakeClusterFunctionsInteractive(),\nmakeClusterFunctionsLSF(),\nmakeClusterFunctionsMulticore(),\nmakeClusterFunctionsOpenLava(),\nmakeClusterFunctionsSGE(),\nmakeClusterFunctionsSSH(),\nmakeClusterFunctionsSlurm(),\nmakeClusterFunctionsSocket(),\nmakeClusterFunctions()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeExperimentRegistry.html", "content": "\n\n\n \n \n\n\n\nExperimentRegistry Constructor — makeExperimentRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    ExperimentRegistry Constructor

    \n Source: R/ExperimentRegistry.R\n
    makeExperimentRegistry.Rd
    \n
    \n\n
    \n

    makeExperimentRegistry constructs a special Registry which\nis suitable for the definition of large scale computer experiments.

    \n

    Each experiments consists of a Problem and an Algorithm.\nThese can be parametrized with addExperiments to actually define computational\njobs.

    \n
    \n\n 
    makeExperimentRegistry(\n  file.dir = \"registry\",\n  work.dir = getwd(),\n  conf.file = findConfFile(),\n  packages = character(0L),\n  namespaces = character(0L),\n  source = character(0L),\n  load = character(0L),\n  seed = NULL,\n  make.default = TRUE\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    file.dir

    [character(1)]
    \n Path where all files of the registry are saved.\n Default is directory “registry” in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    \n

    If you pass NA, a temporary directory will be used.\n This way, you can create disposable registries for btlapply or examples.\n By default, the temporary directory tempdir() will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable temp.dir.

    work.dir

    [character(1)]
    \n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see getwd).\n loadRegistry uses the stored work.dir, but you may also explicitly overwrite it,\n e.g., after switching to another system.

    \n

    The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    conf.file

    [character(1)]
    \n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how batchtools interacts with the system via ClusterFunctions.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.

    \n

    The file lookup is implemented in the internal (but exported) function findConfFile which returns the first file found of the following candidates:

      \n

    1. File “batchtools.conf.R” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. File “batchtools.conf.R” in the current working directory.

      3. \n

    3. File “config.R” in the user configuration directory as reported by rappdirs::user_config_dir(\"batchtools\", expand = FALSE) (depending on OS, e.g., on linux this usually resolves to “~/.config/batchtools/config.R”).

      4. \n

    4. “.batchtools.conf.R” in the home directory (“~”).

      5. \n

    5. “config.R” in the site config directory as reported by rappdirs::site_config_dir(\"batchtools\") (depending on OS). This file can be used for admins to set sane defaults for a computation site.

      6. \n

    Set to NA if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. default.resources = list(walltime = 3600) to set default resources or “max.concurrent.jobs” to\n limit the number of jobs allowed to run simultaneously on the system.

    packages

    [character]
    \nPackages that will always be loaded on each node.\nUses require internally.\nDefault is character(0).

    namespaces

    [character]
    \nSame as packages, but the packages will not be attached.\nUses requireNamespace internally.\nDefault is character(0).

    source

    [character]
    \nFiles which should be sourced on the slaves prior to executing a job.\nCalls sys.source using the .GlobalEnv.

    load

    [character]
    \nFiles which should be loaded on the slaves prior to executing a job.\nCalls load using the .GlobalEnv.

    seed

    [integer(1)]
    \nStart seed for jobs. Each job uses the (seed + job.id) as seed.\nDefault is a random integer between 1 and 32768

    make.default

    [logical(1)]
    \nIf set to TRUE, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is TRUE.

    \n\n

    Value

    \n\n

    [ExperimentRegistry].

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    \n# Definde one problem, two algorithms and add them with some parameters:\naddProblem(reg = tmp, \"p1\",\n  fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd))\n
    #> Adding problem 'p1'
    addAlgorithm(reg = tmp, \"a1\", fun = function(job, data, instance, ...) mean(instance))\n
    #> Adding algorithm 'a1'
    addAlgorithm(reg = tmp, \"a2\", fun = function(job, data, instance, ...) median(instance))\n
    #> Adding algorithm 'a2'
    ids = addExperiments(reg = tmp, list(p1 = data.table::CJ(n = c(50, 100), mean = -2:2, sd = 1:4)))\n
    #> Adding 40 experiments ('p1'[40] x 'a1'[1] x repls[1]) ...
    #> Adding 40 experiments ('p1'[40] x 'a2'[1] x repls[1]) ...
    \n# Overview over defined experiments:\ntmp$problems\n
    #> [1] \"p1\"
    tmp$algorithms\n
    #> [1] \"a1\" \"a2\"
    summarizeExperiments(reg = tmp)\n
    #>    problem algorithm .count\n#> 1:      p1        a1     40\n#> 2:      p1        a2     40
    summarizeExperiments(reg = tmp, by = c(\"problem\", \"algorithm\", \"n\"))\n
    #>    problem algorithm   n .count\n#> 1:      p1        a1  50     20\n#> 2:      p1        a1 100     20\n#> 3:      p1        a2  50     20\n#> 4:      p1        a2 100     20
    ids = findExperiments(prob.pars = (n == 50), reg = tmp)\nprint(unwrap(getJobPars(ids, reg = tmp)))\n
    #>     job.id problem algorithm  n mean sd\n#>  1:      1      p1        a1 50   -2  1\n#>  2:      2      p1        a1 50   -2  2\n#>  3:      3      p1        a1 50   -2  3\n#>  4:      4      p1        a1 50   -2  4\n#>  5:      5      p1        a1 50   -1  1\n#>  6:      6      p1        a1 50   -1  2\n#>  7:      7      p1        a1 50   -1  3\n#>  8:      8      p1        a1 50   -1  4\n#>  9:      9      p1        a1 50    0  1\n#> 10:     10      p1        a1 50    0  2\n#> 11:     11      p1        a1 50    0  3\n#> 12:     12      p1        a1 50    0  4\n#> 13:     13      p1        a1 50    1  1\n#> 14:     14      p1        a1 50    1  2\n#> 15:     15      p1        a1 50    1  3\n#> 16:     16      p1        a1 50    1  4\n#> 17:     17      p1        a1 50    2  1\n#> 18:     18      p1        a1 50    2  2\n#> 19:     19      p1        a1 50    2  3\n#> 20:     20      p1        a1 50    2  4\n#> 21:     41      p1        a2 50   -2  1\n#> 22:     42      p1        a2 50   -2  2\n#> 23:     43      p1        a2 50   -2  3\n#> 24:     44      p1        a2 50   -2  4\n#> 25:     45      p1        a2 50   -1  1\n#> 26:     46      p1        a2 50   -1  2\n#> 27:     47      p1        a2 50   -1  3\n#> 28:     48      p1        a2 50   -1  4\n#> 29:     49      p1        a2 50    0  1\n#> 30:     50      p1        a2 50    0  2\n#> 31:     51      p1        a2 50    0  3\n#> 32:     52      p1        a2 50    0  4\n#> 33:     53      p1        a2 50    1  1\n#> 34:     54      p1        a2 50    1  2\n#> 35:     55      p1        a2 50    1  3\n#> 36:     56      p1        a2 50    1  4\n#> 37:     57      p1        a2 50    2  1\n#> 38:     58      p1        a2 50    2  2\n#> 39:     59      p1        a2 50    2  3\n#> 40:     60      p1        a2 50    2  4\n#>     job.id problem algorithm  n mean sd
    \n# Submit jobs\nsubmitJobs(reg = tmp)\n
    #> Submitting 80 jobs in 80 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    \n# Reduce the results of algorithm a1\nids.mean = findExperiments(algo.name = \"a1\", reg = tmp)\nreduceResults(ids.mean, fun = function(aggr, res, ...) c(aggr, res), reg = tmp)\n
    #>  [1] -2.00073766 -1.83988515 -2.19491590 -1.88744748 -1.16661898 -1.01586977\n#>  [7] -0.52913619 -0.63585094 -0.09331735  0.11880843  0.22120996  1.22939842\n#> [13]  1.00632897  1.26918801  2.11616673  1.81935795  1.90243597  1.93950861\n#> [19]  2.33621891  2.81313094 -1.95745534 -2.29431806 -1.82269005 -1.80889629\n#> [25] -0.91933173 -1.03820621 -0.95531362 -1.49309041  0.09203207  0.05360571\n#> [31]  0.04219920 -0.11534443  1.14488890  0.83504259  0.82821428  1.07835718\n#> [37]  2.06274541  2.28296085  2.26426388  1.74159301
    \n# Join info table with all results and calculate mean of results\n# grouped by n and algorithm\nids = findDone(reg = tmp)\npars = unwrap(getJobPars(ids, reg = tmp))\nresults = unwrap(reduceResultsDataTable(ids, fun = function(res) list(res = res), reg = tmp))\ntab = ljoin(pars, results)\ntab[, list(mres = mean(res)), by = c(\"n\", \"algorithm\")]\n
    #>      n algorithm        mres\n#> 1:  50        a1 0.270398674\n#> 2: 100        a1 0.001062847\n#> 3:  50        a2 0.068935673\n#> 4: 100        a2 0.020262158
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeRegistry.html", "content": "\n\n\n \n \n\n\n\nRegistry Constructor — makeRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Registry Constructor

    \n Source: R/Registry.R\n
    makeRegistry.Rd
    \n
    \n\n
    \n

    makeRegistry constructs the inter-communication object for all functions in batchtools.\nAll communication transactions are processed via the file system:\nAll information required to run a job is stored as JobCollection in a file in the\na subdirectory of the file.dir directory.\nEach jobs stores its results as well as computational status information (start time, end time, error message, ...)\nalso on the file system which is regular merged parsed by the master using syncRegistry.\nAfter integrating the new information into the Registry, the Registry is serialized to the file system via saveRegistry.\nBoth syncRegistry and saveRegistry are called whenever required internally.\nTherefore it should be safe to quit the R session at any time.\nWork can later be resumed by calling loadRegistry which de-serializes the registry from\nthe file system.

    \n

    The registry created last is saved in the package namespace (unless make.default is set to\nFALSE) and can be retrieved via getDefaultRegistry.

    \n

    Canceled jobs and jobs submitted multiple times may leave stray files behind.\nThese can be swept using sweepRegistry.\nclearRegistry completely erases all jobs from a registry, including log files and results,\nand thus allows you to start over.

    \n
    \n\n 
    makeRegistry(\n  file.dir = \"registry\",\n  work.dir = getwd(),\n  conf.file = findConfFile(),\n  packages = character(0L),\n  namespaces = character(0L),\n  source = character(0L),\n  load = character(0L),\n  seed = NULL,\n  make.default = TRUE\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    file.dir

    [character(1)]
    \n Path where all files of the registry are saved.\n Default is directory “registry” in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    \n

    If you pass NA, a temporary directory will be used.\n This way, you can create disposable registries for btlapply or examples.\n By default, the temporary directory tempdir() will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable temp.dir.

    work.dir

    [character(1)]
    \n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see getwd).\n loadRegistry uses the stored work.dir, but you may also explicitly overwrite it,\n e.g., after switching to another system.

    \n

    The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with “~”). Note that some templates do not handle relative paths well.

    conf.file

    [character(1)]
    \n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how batchtools interacts with the system via ClusterFunctions.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.

    \n

    The file lookup is implemented in the internal (but exported) function findConfFile which returns the first file found of the following candidates:

      \n

    1. File “batchtools.conf.R” in the path specified by the environment variable “R_BATCHTOOLS_SEARCH_PATH”.

      2. \n

    2. File “batchtools.conf.R” in the current working directory.

      3. \n

    3. File “config.R” in the user configuration directory as reported by rappdirs::user_config_dir(\"batchtools\", expand = FALSE) (depending on OS, e.g., on linux this usually resolves to “~/.config/batchtools/config.R”).

      4. \n

    4. “.batchtools.conf.R” in the home directory (“~”).

      5. \n

    5. “config.R” in the site config directory as reported by rappdirs::site_config_dir(\"batchtools\") (depending on OS). This file can be used for admins to set sane defaults for a computation site.

      6. \n

    Set to NA if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. default.resources = list(walltime = 3600) to set default resources or “max.concurrent.jobs” to\n limit the number of jobs allowed to run simultaneously on the system.

    packages

    [character]
    \nPackages that will always be loaded on each node.\nUses require internally.\nDefault is character(0).

    namespaces

    [character]
    \nSame as packages, but the packages will not be attached.\nUses requireNamespace internally.\nDefault is character(0).

    source

    [character]
    \nFiles which should be sourced on the slaves prior to executing a job.\nCalls sys.source using the .GlobalEnv.

    load

    [character]
    \nFiles which should be loaded on the slaves prior to executing a job.\nCalls load using the .GlobalEnv.

    seed

    [integer(1)]
    \nStart seed for jobs. Each job uses the (seed + job.id) as seed.\nDefault is a random integer between 1 and 32768

    make.default

    [logical(1)]
    \nIf set to TRUE, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is TRUE.

    \n\n

    Value

    \n\n

    [environment] of class “Registry” with the following slots:

    \n
    file.dir [path]:

    File directory.

    \n
    work.dir [path]:

    Working directory.

    \n
    temp.dir [path]:

    Temporary directory. Used if file.dir is NA to create temporary registries.

    \n
    packages [character()]:

    Packages to load on the slaves.

    \n
    namespaces [character()]:

    Namespaces to load on the slaves.

    \n
    seed [integer(1)]:

    Registry seed. Before each job is executed, the seed seed + job.id is set.

    \n
    cluster.functions [cluster.functions]:

    Usually set in your conf.file. Set via a call to makeClusterFunctions. See example.

    \n
    default.resources [named list()]:

    Usually set in your conf.file. Named list of default resources.

    \n
    max.concurrent.jobs [integer(1)]:

    Usually set in your conf.file. Maximum number of concurrent jobs for a single user and current registry on the system.\n submitJobs will try to respect this setting. The resource “max.concurrent.jobs” has higher precedence.

    \n
    defs [data.table]:

    Table with job definitions (i.e. parameters).

    \n
    status [data.table]:

    Table holding information about the computational status. Also see getJobStatus.

    \n
    resources [data.table]:

    Table holding information about the computational resources used for the job. Also see getJobResources.

    \n
    tags [data.table]:

    Table holding information about tags. See Tags.

    \n
    hash [character(1)]:

    Unique hash which changes each time the registry gets saved to the file system. Can be utilized to invalidate the cache of knitr.

    \n \n
    \n\n

    Details

    \n\n

    Currently batchtools understands the following options set via the configuration file:

    \n
    cluster.functions:

    As returned by a constructor, e.g. makeClusterFunctionsSlurm.

    \n
    default.resources:

    List of resources to use. Will be overruled by resources specified via submitJobs.

    \n
    temp.dir:

    Path to directory to use for temporary registries.

    \n
    sleep:

    Custom sleep function. See waitForJobs.

    \n
    expire.after:

    Number of iterations before treating jobs as expired in waitForJobs.

    \n
    compress:

    Compression algorithm to use via saveRDS.

    \n\n
    \n\n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nloadRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    print(tmp)\n
    #> Job Registry\n#>   Backend  : Interactive\n#>   File dir : /tmp/batchtools-example/reg\n#>   Work dir : /home/michel/Projekte/batchtools/docs/reference\n#>   Jobs     : 0\n#>   Seed     : 5075\n#>   Writeable: TRUE
    \n# Set cluster functions to interactive mode and start jobs in external R sessions\ntmp$cluster.functions = makeClusterFunctionsInteractive(external = TRUE)\n\n# Change packages to load\ntmp$packages = c(\"MASS\")\nsaveRegistry(reg = tmp)\n
    #> [1] TRUE
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/makeSubmitJobResult.html", "content": "\n\n\n \n \n\n\n\nCreate a SubmitJobResult — makeSubmitJobResult • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Create a SubmitJobResult

    \n Source: R/clusterFunctions.R\n
    makeSubmitJobResult.Rd
    \n
    \n\n
    \n

    This function is only intended for use in your own cluster functions implementation.

    \n

    Use this function in your implementation of makeClusterFunctions to create a return\nvalue for the submitJob function.

    \n
    \n\n 
    makeSubmitJobResult(\n  status,\n  batch.id,\n  log.file = NA_character_,\n  msg = NA_character_\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    status

    [integer(1)]
    \nLaunch status of job. 0 means success, codes between 1 and 100 are temporary errors and any\nerror greater than 100 is a permanent failure.

    batch.id

    [character()]
    \nUnique id of this job on batch system, as given by the batch system.\nMust be globally unique so that the job can be terminated using just this information.\nFor array jobs, this may be a vector of length equal to the number of jobs in the array.

    log.file

    [character()]
    \nLog file. If NA, defaults to [job.hash].log.\nSome cluster functions set this for array jobs.

    msg

    [character(1)]
    \nOptional error message in case status is not equal to 0. Default is “OK”,\n“TEMPERROR”, “ERROR”, depending on status.

    \n\n

    Value

    \n\n

    [SubmitJobResult]. A list, containing\n status, batch.id and msg.

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfHandleUnknownSubmitError(),\ncfKillJob(),\ncfReadBrewTemplate(),\nmakeClusterFunctions(),\nrunOSCommand()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/reduceResults.html", "content": "\n\n\n \n \n\n\n\nReduce Results — reduceResults • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Reduce Results

    \n Source: R/reduceResults.R\n
    reduceResults.Rd
    \n
    \n\n
    \n

    A version of Reduce for Registry objects\nwhich iterates over finished jobs and aggregates them.\nAll jobs must have terminated, an error is raised otherwise.

    \n
    \n\n 
    reduceResults(fun, ids = NULL, init, ..., reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    fun

    [function]
    \nA function to reduce the results. The result of previous iterations (or\nthe init) will be passed as first argument, the result of of the\ni-th iteration as second. See Reduce for some\nexamples.\nIf the function has the formal argument “job”, the Job/Experiment\nis also passed to the function (named).

    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findDone.\nInvalid ids are ignored.

    init

    [ANY]
    \nInitial element, as used in Reduce.\nIf missing, the reduction uses the result of the first job as init and the reduction starts\nwith the second job.

    ...

    [ANY]
    \nAdditional arguments passed to function fun.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    Aggregated results in the same order as provided ids.\n Return type depends on the user function. If ids\n is empty, reduceResults returns init (if available) or NULL otherwise.

    \n

    Note

    \n\n

    If you have thousands of jobs, disabling the progress bar (options(batchtools.progress = FALSE))\ncan significantly increase the performance.

    \n

    See also

    \n\n

    Other Results: \nbatchMapResults(),\nloadResult(),\nreduceResultsList()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(function(a, b) list(sum = a+b, prod = a*b), a = 1:3, b = 1:3, reg = tmp)\n
    #> Adding 3 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    \n# Extract element sum from each result\nreduceResults(function(aggr, res) c(aggr, res$sum), init = list(), reg = tmp)\n
    #> [[1]]\n#> [1] 2\n#> \n#> [[2]]\n#> [1] 4\n#> \n#> [[3]]\n#> [1] 6\n#> 
    \n# Aggregate element sum via '+'\nreduceResults(function(aggr, res) aggr + res$sum, init = 0, reg = tmp)\n
    #> [1] 12
    \n# Aggregate element prod via '*' where parameter b < 3\nreduce = function(aggr, res, job) {\n  if (job$pars$b >= 3)\n    return(aggr)\n  aggr * res$prod\n}\nreduceResults(reduce, init = 1, reg = tmp)\n
    #> [1] 4
    \n# Reduce to data.frame() (inefficient, use reduceResultsDataTable() instead)\nreduceResults(rbind, init = data.frame(), reg = tmp)\n
    #>   sum prod\n#> 1   2    1\n#> 2   4    4\n#> 3   6    9
    \n# Reduce to data.frame by collecting results first, then utilize vectorization of rbind:\nres = reduceResultsList(fun = as.data.frame, reg = tmp)\ndo.call(rbind, res)\n
    #>   sum prod\n#> 1   2    1\n#> 2   4    4\n#> 3   6    9
    \n# Reduce with custom combine function:\ncomb = function(x, y) list(sum = x$sum + y$sum, prod = x$prod * y$prod)\nreduceResults(comb, reg = tmp)\n
    #> $sum\n#> [1] 12\n#> \n#> $prod\n#> [1] 36\n#> 
    \n# The same with neutral element NULL\ncomb = function(x, y) if (is.null(x)) y else list(sum = x$sum + y$sum, prod = x$prod * y$prod)\nreduceResults(comb, init = NULL, reg = tmp)\n
    #> $sum\n#> [1] 12\n#> \n#> $prod\n#> [1] 36\n#> 
    \n# Alternative: Reduce in list, reduce manually in a 2nd step\nres = reduceResultsList(reg = tmp)\nReduce(comb, res)\n
    #> $sum\n#> [1] 12\n#> \n#> $prod\n#> [1] 36\n#>
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/reduceResultsList.html", "content": "\n\n\n \n \n\n\n\nApply Functions on Results — reduceResultsList • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Apply Functions on Results

    \n Source: R/reduceResults.R\n
    reduceResultsList.Rd
    \n
    \n\n
    \n

    Applies a function on the results of your finished jobs and thereby collects\nthem in a list or data.table.\nThe later requires the provided function to return a list (or data.frame) of scalar values.\nSee rbindlist for features and limitations of the aggregation.

    \n

    If not all jobs are terminated, the respective result will be NULL.

    \n
    \n\n 
    reduceResultsList(\n  ids = NULL,\n  fun = NULL,\n  ...,\n  missing.val,\n  reg = getDefaultRegistry()\n)\n\nreduceResultsDataTable(\n  ids = NULL,\n  fun = NULL,\n  ...,\n  missing.val,\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findDone.\nInvalid ids are ignored.

    fun

    [function]
    \nFunction to apply to each result. The result is passed unnamed as first argument. If NULL, the identity is used.\nIf the function has the formal argument “job”, the Job/Experiment is also passed to the function.

    ...

    [ANY]
    \nAdditional arguments passed to to function fun.

    missing.val

    [ANY]
    \nValue to impute as result for a job which is not finished.\nIf not provided and a result is missing, an exception is raised.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    reduceResultsList returns a list of the results in the same order as the provided ids.\n reduceResultsDataTable returns a data.table with columns “job.id” and additional result columns\n created via rbindlist, sorted by “job.id”.

    \n

    Note

    \n\n

    If you have thousands of jobs, disabling the progress bar (options(batchtools.progress = FALSE))\ncan significantly increase the performance.

    \n

    See also

    \n\n

    reduceResults

    \n

    Other Results: \nbatchMapResults(),\nloadResult(),\nreduceResults()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(2) \n### Example 1 - reduceResultsList\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg1' using cluster functions 'Interactive'
    batchMap(function(x) x^2, x = 1:10, reg = tmp)\n
    #> Adding 10 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
    waitForJobs(reg = tmp)\n
    #> [1] TRUE
    reduceResultsList(fun = sqrt, reg = tmp)\n
    #> [[1]]\n#> [1] 1\n#> \n#> [[2]]\n#> [1] 2\n#> \n#> [[3]]\n#> [1] 3\n#> \n#> [[4]]\n#> [1] 4\n#> \n#> [[5]]\n#> [1] 5\n#> \n#> [[6]]\n#> [1] 6\n#> \n#> [[7]]\n#> [1] 7\n#> \n#> [[8]]\n#> [1] 8\n#> \n#> [[9]]\n#> [1] 9\n#> \n#> [[10]]\n#> [1] 10\n#> 
    \n### Example 2 - reduceResultsDataTable\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg2' using cluster functions 'Interactive'
    \n# add first problem\nfun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\naddProblem(\"rnorm\", fun = fun, reg = tmp)\n
    #> Adding problem 'rnorm'
    \n# add second problem\nfun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\naddProblem(\"rexp\", fun = fun, reg = tmp)\n
    #> Adding problem 'rexp'
    \n# add first algorithm\nfun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\naddAlgorithm(\"average\", fun = fun, reg = tmp)\n
    #> Adding algorithm 'average'
    \n# add second algorithm\nfun = function(instance, ...) sd(instance)\naddAlgorithm(\"deviation\", fun = fun, reg = tmp)\n
    #> Adding algorithm 'deviation'
    \n# define problem and algorithm designs\nlibrary(data.table)\nprob.designs = algo.designs = list()\nprob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\nprob.designs$rexp = data.table(n = 100, lambda = 1:5)\nalgo.designs$average = data.table(method = c(\"mean\", \"median\"))\nalgo.designs$deviation = data.table()\n\n# add experiments and submit\naddExperiments(prob.designs, algo.designs, reg = tmp)\n
    #> Adding 30 experiments ('rnorm'[15] x 'average'[2] x repls[1]) ...
    #> Adding 15 experiments ('rnorm'[15] x 'deviation'[1] x repls[1]) ...
    #> Adding 10 experiments ('rexp'[5] x 'average'[2] x repls[1]) ...
    #> Adding 5 experiments ('rexp'[5] x 'deviation'[1] x repls[1]) ...
    submitJobs(reg = tmp)\n
    #> Submitting 60 jobs in 60 chunks using cluster functions 'Interactive' ...
    \n# collect results and join them with problem and algorithm paramters\nres = ijoin(\n  getJobPars(reg = tmp),\n  reduceResultsDataTable(reg = tmp, fun = function(x) list(res = x))\n)\nunwrap(res, sep = \".\")\n
    #>     job.id problem algorithm prob.pars.n prob.pars.mean prob.pars.sd\n#>  1:      1   rnorm   average         100             -1            1\n#>  2:      2   rnorm   average         100             -1            1\n#>  3:      3   rnorm   average         100             -1            2\n#>  4:      4   rnorm   average         100             -1            2\n#>  5:      5   rnorm   average         100             -1            3\n#>  6:      6   rnorm   average         100             -1            3\n#>  7:      7   rnorm   average         100             -1            4\n#>  8:      8   rnorm   average         100             -1            4\n#>  9:      9   rnorm   average         100             -1            5\n#> 10:     10   rnorm   average         100             -1            5\n#> 11:     11   rnorm   average         100              0            1\n#> 12:     12   rnorm   average         100              0            1\n#> 13:     13   rnorm   average         100              0            2\n#> 14:     14   rnorm   average         100              0            2\n#> 15:     15   rnorm   average         100              0            3\n#> 16:     16   rnorm   average         100              0            3\n#> 17:     17   rnorm   average         100              0            4\n#> 18:     18   rnorm   average         100              0            4\n#> 19:     19   rnorm   average         100              0            5\n#> 20:     20   rnorm   average         100              0            5\n#> 21:     21   rnorm   average         100              1            1\n#> 22:     22   rnorm   average         100              1            1\n#> 23:     23   rnorm   average         100              1            2\n#> 24:     24   rnorm   average         100              1            2\n#> 25:     25   rnorm   average         100              1            3\n#> 26:     26   rnorm   average         100              1            3\n#> 27:     27   rnorm   average         100              1            4\n#> 28:     28   rnorm   average         100              1            4\n#> 29:     29   rnorm   average         100              1            5\n#> 30:     30   rnorm   average         100              1            5\n#> 31:     31   rnorm deviation         100             -1            1\n#> 32:     32   rnorm deviation         100             -1            2\n#> 33:     33   rnorm deviation         100             -1            3\n#> 34:     34   rnorm deviation         100             -1            4\n#> 35:     35   rnorm deviation         100             -1            5\n#> 36:     36   rnorm deviation         100              0            1\n#> 37:     37   rnorm deviation         100              0            2\n#> 38:     38   rnorm deviation         100              0            3\n#> 39:     39   rnorm deviation         100              0            4\n#> 40:     40   rnorm deviation         100              0            5\n#> 41:     41   rnorm deviation         100              1            1\n#> 42:     42   rnorm deviation         100              1            2\n#> 43:     43   rnorm deviation         100              1            3\n#> 44:     44   rnorm deviation         100              1            4\n#> 45:     45   rnorm deviation         100              1            5\n#> 46:     46    rexp   average         100             NA           NA\n#> 47:     47    rexp   average         100             NA           NA\n#> 48:     48    rexp   average         100             NA           NA\n#> 49:     49    rexp   average         100             NA           NA\n#> 50:     50    rexp   average         100             NA           NA\n#> 51:     51    rexp   average         100             NA           NA\n#> 52:     52    rexp   average         100             NA           NA\n#> 53:     53    rexp   average         100             NA           NA\n#> 54:     54    rexp   average         100             NA           NA\n#> 55:     55    rexp   average         100             NA           NA\n#> 56:     56    rexp deviation         100             NA           NA\n#> 57:     57    rexp deviation         100             NA           NA\n#> 58:     58    rexp deviation         100             NA           NA\n#> 59:     59    rexp deviation         100             NA           NA\n#> 60:     60    rexp deviation         100             NA           NA\n#>     job.id problem algorithm prob.pars.n prob.pars.mean prob.pars.sd\n#>     prob.pars.lambda algo.pars.method   result.res\n#>  1:               NA             mean -1.092018851\n#>  2:               NA           median -0.863780644\n#>  3:               NA             mean -1.084890423\n#>  4:               NA           median -1.505171392\n#>  5:               NA             mean -1.381319138\n#>  6:               NA           median -1.341051423\n#>  7:               NA             mean -1.188083630\n#>  8:               NA           median  0.029939562\n#>  9:               NA             mean -0.987419910\n#> 10:               NA           median -1.673002281\n#> 11:               NA             mean  0.007267359\n#> 12:               NA           median  0.127401909\n#> 13:               NA             mean -0.179617350\n#> 14:               NA           median  0.046125736\n#> 15:               NA             mean -0.057929853\n#> 16:               NA           median  0.140366680\n#> 17:               NA             mean -0.388849478\n#> 18:               NA           median -1.207701427\n#> 19:               NA             mean  0.042334184\n#> 20:               NA           median -0.381653696\n#> 21:               NA             mean  0.907293957\n#> 22:               NA           median  1.033215601\n#> 23:               NA             mean  1.020619322\n#> 24:               NA           median  0.876104940\n#> 25:               NA             mean  0.679306393\n#> 26:               NA           median  0.977987955\n#> 27:               NA             mean  0.675594915\n#> 28:               NA           median  1.366615131\n#> 29:               NA             mean  1.189950292\n#> 30:               NA           median  0.263092706\n#> 31:               NA             <NA>  1.040412625\n#> 32:               NA             <NA>  2.392300899\n#> 33:               NA             <NA>  3.042692900\n#> 34:               NA             <NA>  4.263774294\n#> 35:               NA             <NA>  5.296092785\n#> 36:               NA             <NA>  1.071510659\n#> 37:               NA             <NA>  2.072637870\n#> 38:               NA             <NA>  3.153765268\n#> 39:               NA             <NA>  3.419489935\n#> 40:               NA             <NA>  5.558785921\n#> 41:               NA             <NA>  1.012589238\n#> 42:               NA             <NA>  2.015986064\n#> 43:               NA             <NA>  3.170339048\n#> 44:               NA             <NA>  4.281061044\n#> 45:               NA             <NA>  4.988645174\n#> 46:                1             mean  0.945622941\n#> 47:                1           median  0.619859008\n#> 48:                2             mean  0.446385905\n#> 49:                2           median  0.420682183\n#> 50:                3             mean  0.397613162\n#> 51:                3           median  0.306781776\n#> 52:                4             mean  0.277069910\n#> 53:                4           median  0.165324176\n#> 54:                5             mean  0.182106656\n#> 55:                5           median  0.144752066\n#> 56:                1             <NA>  0.942306695\n#> 57:                2             <NA>  0.453601669\n#> 58:                3             <NA>  0.292487926\n#> 59:                4             <NA>  0.210416655\n#> 60:                5             <NA>  0.189556888\n#>     prob.pars.lambda algo.pars.method   result.res
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/removeExperiments.html", "content": "\n\n\n \n \n\n\n\nRemove Experiments — removeExperiments • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Remove Experiments

    \n Source: R/removeExperiments.R\n
    removeExperiments.Rd
    \n
    \n\n
    \n

    Remove Experiments from an ExperimentRegistry.\nThis function automatically checks if any of the jobs to reset is either pending or running.\nHowever, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\nUse with care while jobs are running.

    \n
    \n\n 
    removeExperiments(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to no job.\nInvalid ids are ignored.

    reg

    [ExperimentRegistry]
    \nRegistry. If not explicitly passed, uses the last created registry.

    \n\n

    Value

    \n\n

    [data.table] of removed job ids, invisibly.

    \n

    See also

    \n\n

    Other Experiment: \naddExperiments(),\nsummarizeExperiments()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/removeRegistry.html", "content": "\n\n\n \n \n\n\n\nRemove a Registry from the File System — removeRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Remove a Registry from the File System

    \n Source: R/removeRegistry.R\n
    removeRegistry.Rd
    \n
    \n\n
    \n

    All files will be erased from the file system, including all results.\nIf you wish to remove only intermediate files, use sweepRegistry.

    \n
    \n\n 
    removeRegistry(wait = 5, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    wait

    [numeric(1)]
    \nSeconds to wait before proceeding. This is a safety measure to not\naccidentally remove your precious files. Set to 0 in\nnon-interactive scripts to disable this precaution.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [character(1)]: Path of the deleted file directory.

    \n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nloadRegistry(),\nmakeRegistry(),\nsaveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    removeRegistry(0, tmp)\n
    #> Recursively removing files in '/tmp/batchtools-example/reg' ...
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/resetJobs.html", "content": "\n\n\n \n \n\n\n\nReset the Computational State of Jobs — resetJobs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Reset the Computational State of Jobs

    \n Source: R/resetJobs.R\n
    resetJobs.Rd
    \n
    \n\n
    \n

    Resets the computational state of jobs in the Registry.\nThis function automatically checks if any of the jobs to reset is either pending or running.\nHowever, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\nUse with care while jobs are running.

    \n
    \n\n 
    resetJobs(ids = NULL, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to no job.\nInvalid ids are ignored.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] of job ids which have been reset.\n See JoinTables for examples on working with job tables.

    \n

    See also

    \n\n

    Other debug: \ngetErrorMessages(),\ngetStatus(),\ngrepLogs(),\nkillJobs(),\nshowLog(),\ntestJob()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/runHook.html", "content": "\n\n\n \n \n\n\n\nTrigger Evaluation of Custom Function — runHook • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Trigger Evaluation of Custom Function

    \n Source: R/Hooks.R\n
    runHook.Rd
    \n
    \n\n
    \n

    Hooks allow to trigger functions calls on specific events.\nThey can be specified via the ClusterFunctions and are triggered on the following events:

    \n
    pre.sync

    function(reg, fns, ...): Run before synchronizing the registry on the master. fn is the character vector of paths to the update files.

    \n
    post.sync

    function(reg, updates, ...): Run after synchronizing the registry on the master. updates is the data.table of processed updates.

    \n
    pre.submit.job

    function(reg, ...): Run before a job is successfully submitted to the scheduler on the master.

    \n
    post.submit.job

    function(reg, ...): Run after a job is successfully submitted to the scheduler on the master.

    \n
    pre.submit

    function(reg, ...): Run before any job is submitted to the scheduler.

    \n
    post.submit

    function(reg, ...): Run after a jobs are submitted to the schedule.

    \n
    pre.do.collection

    function(reg, reader, ...): Run before starting the job collection on the slave.\n reader is an internal cache object.

    \n
    post.do.collection

    function(reg, updates, reader, ...): Run after all jobs in the chunk are terminated on the slave.\n updates is a data.table of updates which will be merged with the Registry by the master.\n reader is an internal cache object.

    \n
    pre.kill

    function(reg, ids, ...): Run before any job is killed.

    \n
    post.kill

    function(reg, ids, ...): Run after jobs are killed. ids is the return value of killJobs.

    \n\n
    \n\n
    \n\n 
    runHook(obj, hook, ...)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    obj

    [Registry | JobCollection]
    \nRegistry which contains the ClusterFunctions with element “hooks”\nor a JobCollection which holds the subset of functions which are executed\nremotely.

    hook

    [character(1)]
    \nID of the hook as string.

    ...

    [ANY]
    \nAdditional arguments passed to the function referenced by hook.\nSee description.

    \n\n

    Value

    \n\n

    Return value of the called function, or NULL if there is no hook\n with the specified ID.

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/runOSCommand.html", "content": "\n\n\n \n \n\n\n\nRun OS Commands on Local or Remote Machines — runOSCommand • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Run OS Commands on Local or Remote Machines

    \n Source: R/runOSCommand.R\n
    runOSCommand.Rd
    \n
    \n\n
    \n

    This is a helper function to run arbitrary OS commands on local or remote machines.\nThe interface is similar to system2, but it always returns the exit status\nand the output.

    \n
    \n\n 
    runOSCommand(\n  sys.cmd,\n  sys.args = character(0L),\n  stdin = \"\",\n  nodename = \"localhost\"\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    sys.cmd

    [character(1)]
    \nCommand to run.

    sys.args

    [character]
    \nArguments for sys.cmd.

    stdin

    [character(1)]
    \nArgument passed to system2.

    nodename

    [character(1)]
    \nName of the SSH node to run the command on. If set to “localhost” (default), the command\nis not piped through SSH.

    \n\n

    Value

    \n\n

    [named list] with “sys.cmd”, “sys.args”, “exit.code” (integer), “output” (character).

    \n

    See also

    \n\n

    Other ClusterFunctionsHelper: \ncfBrewTemplate(),\ncfHandleUnknownSubmitError(),\ncfKillJob(),\ncfReadBrewTemplate(),\nmakeClusterFunctions(),\nmakeSubmitJobResult()

    \n\n

    Examples

    \n 
    if (FALSE) {\nrunOSCommand(\"ls\")\nrunOSCommand(\"ls\", \"-al\")\nrunOSCommand(\"notfound\")\n}\n
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/saveRegistry.html", "content": "\n\n\n \n \n\n\n\nStore the Registy to the File System — saveRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Store the Registy to the File System

    \n Source: R/saveRegistry.R\n
    saveRegistry.Rd
    \n
    \n\n
    \n

    Stores the registry on the file system in its “file.dir” (specified\nfor construction in makeRegistry, can be accessed via\nreg$file.dir).\nThis function is usually called internally whenever needed.

    \n
    \n\n 
    saveRegistry(reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [logical(1)]: TRUE if the registry was saved,\n FALSE otherwise (if the registry is read-only).

    \n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nloadRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsweepRegistry(),\nsyncRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/showLog.html", "content": "\n\n\n \n \n\n\n\nInspect Log Files — showLog • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Inspect Log Files

    \n Source: R/Logs.R\n
    showLog.Rd
    \n
    \n\n
    \n

    showLog opens the log in the pager. For customization, see file.show.\ngetLog returns the log as character vector.

    \n
    \n\n 
    showLog(id, reg = getDefaultRegistry())\n\ngetLog(id, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n
    id

    [integer(1) or data.table]
    \nSingle integer to specify the job or a data.table with column job.id\nand exactly one row.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    Nothing.

    \n

    See also

    \n\n

    Other debug: \ngetErrorMessages(),\ngetStatus(),\ngrepLogs(),\nkillJobs(),\nresetJobs(),\ntestJob()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    \n# Create some dummy jobs\nfun = function(i) {\n  if (i == 3) stop(i)\n  if (i %% 2 == 1) warning(\"That's odd.\")\n}\nids = batchMap(fun, i = 1:5, reg = tmp)\n
    #> Adding 5 jobs ...
    submitJobs(reg = tmp)\n
    #> Submitting 5 jobs in 5 chunks using cluster functions 'Interactive' ...
    #> Warning: That's odd.
    #> Error in (function (i)  : 3
    #> Warning: That's odd.
    waitForJobs(reg = tmp)\n
    #> [1] FALSE
    getStatus(reg = tmp)\n
    #> Status for 5 jobs at 2020-10-21 09:39:31:\n#>   Submitted    : 5 (100.0%)\n#>   -- Queued    : 0 (  0.0%)\n#>   -- Started   : 5 (100.0%)\n#>   ---- Running : 0 (  0.0%)\n#>   ---- Done    : 4 ( 80.0%)\n#>   ---- Error   : 1 ( 20.0%)\n#>   ---- Expired : 0 (  0.0%)
    \nwriteLines(getLog(ids[1], reg = tmp))\n
    #> ### [bt]: This is batchtools v0.9.14\n#> ### [bt]: Starting calculation of 1 jobs\n#> ### [bt]: Setting working directory to '/home/michel/Projekte/batchtools/docs/reference'\n#> ### [bt]: Memory measurement disabled\n#> ### [bt]: Starting job [batchtools job.id=1]\n#> ### [bt]: Setting seed to 5192 ...\n#> \n#> ### [bt]: Job terminated successfully [batchtools job.id=1]\n#> ### [bt]: Calculation finished!
    if (FALSE) {\nshowLog(ids[1], reg = tmp)\n}\n\ngrepLogs(pattern = \"warning\", ignore.case = TRUE, reg = tmp)\n
    #> Empty data.table (0 rows and 2 cols): job.id,matches
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/submitJobs.html", "content": "\n\n\n \n \n\n\n\nSubmit Jobs to the Batch Systems — submitJobs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Submit Jobs to the Batch Systems

    \n Source: R/submitJobs.R\n
    submitJobs.Rd
    \n
    \n\n
    \n

    Submits defined jobs to the batch system.

    \n

    After submitting the jobs, you can use waitForJobs to wait for the\ntermination of jobs or call reduceResultsList/reduceResults\nto collect partial results.\nThe progress can be monitored with getStatus.

    \n
    \n\n 
    submitJobs(\n  ids = NULL,\n  resources = list(),\n  sleep = NULL,\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findNotSubmitted.\nInvalid ids are ignored.

    resources

    [named list]
    \nComputational resources for the jobs to submit. The actual elements of this list\n(e.g. something like “walltime” or “nodes”) depend on your template file, exceptions are outlined in the section 'Resources'.\nDefault settings for a system can be set in the configuration file by defining the named list default.resources.\nNote that these settings are merged by name, e.g. merging list(walltime = 300) into list(walltime = 400, memory = 512)\nwill result in list(walltime = 300, memory = 512).\nSame holds for individual job resources passed as additional column of ids (c.f. section 'Resources').

    sleep

    [function(i) | numeric(1)]
    \nParameter to control the duration to sleep between temporary errors.\nYou can pass an absolute numeric value in seconds or a function(i) which returns the number of seconds to sleep in the i-th\niteration between temporary errors.\nIf not provided (NULL), tries to read the value (number/function) from the configuration file (stored in reg$sleep) or defaults to\na function with exponential backoff between 5 and 120 seconds.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [data.table] with columns “job.id” and “chunk”.

    \n

    Note

    \n\n

    If you a large number of jobs, disabling the progress bar (options(batchtools.progress = FALSE))\ncan significantly increase the performance of submitJobs.

    \n

    Resources

    \n\n \n\n

    You can pass arbitrary resources to submitJobs() which then are available in the cluster function template.\nSome resources' names are standardized and it is good practice to stick to the following nomenclature to avoid confusion:

    \n
    walltime:

    Upper time limit in seconds for jobs before they get killed by the scheduler. Can be passed as additional column as part of ids to set per-job resources.

    \n
    memory:

    Memory limit in Mb. If jobs exceed this limit, they are usually killed by the scheduler. Can be passed as additional column as part of ids to set per-job resources.

    \n
    ncpus:

    Number of (physical) CPUs to use on the slave. Can be passed as additional column as part of ids to set per-job resources.

    \n
    omp.threads:

    Number of threads to use via OpenMP. Used to set environment variable “OMP_NUM_THREADS”. Can be passed as additional column as part of ids to set per-job resources.

    \n
    pp.size:

    Maximum size of the pointer protection stack, see Memory.

    \n
    blas.threads:

    Number of threads to use for the BLAS backend. Used to set environment variables “MKL_NUM_THREADS” and “OPENBLAS_NUM_THREADS”. Can be passed as additional column as part of ids to set per-job resources.

    \n
    measure.memory:

    Enable memory measurement for jobs. Comes with a small runtime overhead.

    \n
    chunks.as.arrayjobs:

    Execute chunks as array jobs.

    \n
    pm.backend:

    Start a parallelMap backend on the slave.

    \n
    foreach.backend:

    Start a foreach backend on the slave.

    \n
    clusters:

    Resource used for Slurm to select the set of clusters to run sbatch/squeue/scancel on.

    \n\n
    \n\n

    Chunking of Jobs

    \n\n \n\n

    Multiple jobs can be grouped (chunked) together to be executed sequentially on the batch system as a single batch job.\nThis is especially useful to avoid overburding the scheduler by submitting thousands of jobs simultaneously.\nTo chunk jobs together, job ids must be provided as data.frame with columns “job.id” and “chunk” (integer).\nAll jobs with the same chunk number will be executed sequentially inside the same batch job.\nThe utility functions chunk, binpack and lpt\ncan assist in grouping jobs.

    \n

    Array Jobs

    \n\n \n\n

    If your cluster supports array jobs, you can set the resource chunks.as.arrayjobs to TRUE in order\nto execute chunks as job arrays on the cluster.\nFor each chunk of size n, batchtools creates a JobCollection of (possibly heterogeneous) jobs which is\nsubmitted to the scheduler as a single array job with n repetitions.\nFor each repetition, the JobCollection is first read from the file system, then subsetted to the i-th job using\nthe environment variable reg$cluster.functions$array.var (depending on the cluster backend, defined automatically) and finally\nexecuted.

    \n

    Order of Submission

    \n\n \n\n

    Jobs are submitted in the order of chunks, i.e. jobs which have chunk number\nsort(unique(ids$chunk))[1] first, then jobs with chunk number sort(unique(ids$chunk))[2]\nand so on. If no chunks are provided, jobs are submitted in the order of ids$job.id.

    \n

    Limiting the Number of Jobs

    \n\n \n\n

    If requested, submitJobs tries to limit the number of concurrent jobs of the user by waiting until jobs terminate\nbefore submitting new ones.\nThis can be controlled by setting “max.concurrent.jobs” in the configuration file (see Registry)\nor by setting the resource “max.concurrent.jobs” to the maximum number of jobs to run simultaneously.\nIf both are set, the setting via the resource takes precedence over the setting in the configuration.

    \n

    Measuring Memory

    \n\n \n\n

    Setting the resource measure.memory to TRUE turns on memory measurement:\ngc is called directly before and after the job and the difference is\nstored in the internal database. Note that this is just a rough estimate and does\nneither work reliably for external code like C/C++ nor in combination with threading.

    \n

    Inner Parallelization

    \n\n \n\n

    Inner parallelization is typically done via threading, sockets or MPI.\nTwo backends are supported to assist in setting up inner parallelization.

    \n

    The first package is parallelMap.\nIf you set the resource “pm.backend” to “multicore”, “socket” or “mpi”,\nparallelStart is called on the slave before the first job in the chunk is started\nand parallelStop is called after the last job terminated.\nThis way, the resources for inner parallelization can be set and get automatically stored just like other computational resources.\nThe function provided by the user just has to call parallelMap to start parallelization using the preconfigured backend.

    \n

    To control the number of CPUs, you have to set the resource ncpus.\nOtherwise ncpus defaults to the number of available CPUs (as reported by (see detectCores))\non the executing machine for multicore and socket mode and defaults to the return value of mpi.universe.size-1 for MPI.\nYour template must be set up to handle the parallelization, e.g. request the right number of CPUs or start R with mpirun.\nYou may pass further options like level to parallelStart via the named list “pm.opts”.

    \n

    The second supported parallelization backend is foreach.\nIf you set the resource “foreach.backend” to “seq” (sequential mode), “parallel” (doParallel) or\n“mpi” (doMPI), the requested foreach backend is automatically registered on the slave.\nAgain, the resource ncpus is used to determine the number of CPUs.

    \n

    Neither the namespace of parallelMap nor the namespace foreach are attached.\nYou have to do this manually via library or let the registry load the packages for you.

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(3) \n### Example 1: Submit subsets of jobs\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg1' using cluster functions 'Interactive'
    \n# toy function which fails if x is even and an input file does not exists\nfun = function(x, fn) if (x %% 2 == 0 && !file.exists(fn)) stop(\"file not found\") else x\n\n# define jobs via batchMap\nfn = tempfile()\nids = batchMap(fun, 1:20, reg = tmp, fn = fn)\n
    #> Adding 20 jobs ...
    \n# submit some jobs\nids = 1:10\nsubmitJobs(ids, reg = tmp)\n
    #> Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
    #> Error in (function (x, fn)  : file not found\n#> Error in (function (x, fn)  : file not found\n#> Error in (function (x, fn)  : file not found\n#> Error in (function (x, fn)  : file not found\n#> Error in (function (x, fn)  : file not found
    waitForJobs(ids, reg = tmp)\n
    #> [1] FALSE
    getStatus(reg = tmp)\n
    #> Status for 20 jobs at 2020-10-21 09:39:31:\n#>   Submitted    : 10 ( 50.0%)\n#>   -- Queued    :  0 (  0.0%)\n#>   -- Started   : 10 ( 50.0%)\n#>   ---- Running :  0 (  0.0%)\n#>   ---- Done    :  5 ( 25.0%)\n#>   ---- Error   :  5 ( 25.0%)\n#>   ---- Expired :  0 (  0.0%)
    \n# create the required file and re-submit failed jobs\nfile.create(fn)\n
    #> [1] TRUE
    submitJobs(findErrors(ids, reg = tmp), reg = tmp)\n
    #> Submitting 5 jobs in 5 chunks using cluster functions 'Interactive' ...
    getStatus(reg = tmp)\n
    #> Status for 20 jobs at 2020-10-21 09:39:31:\n#>   Submitted    : 10 ( 50.0%)\n#>   -- Queued    :  0 (  0.0%)\n#>   -- Started   : 10 ( 50.0%)\n#>   ---- Running :  0 (  0.0%)\n#>   ---- Done    : 10 ( 50.0%)\n#>   ---- Error   :  0 (  0.0%)\n#>   ---- Expired :  0 (  0.0%)
    \n# submit remaining jobs which have not yet been submitted\nids = findNotSubmitted(reg = tmp)\nsubmitJobs(ids, reg = tmp)\n
    #> Submitting 10 jobs in 10 chunks using cluster functions 'Interactive' ...
    getStatus(reg = tmp)\n
    #> Status for 20 jobs at 2020-10-21 09:39:31:\n#>   Submitted    : 20 (100.0%)\n#>   -- Queued    :  0 (  0.0%)\n#>   -- Started   : 20 (100.0%)\n#>   ---- Running :  0 (  0.0%)\n#>   ---- Done    : 20 (100.0%)\n#>   ---- Error   :  0 (  0.0%)\n#>   ---- Expired :  0 (  0.0%)
    \n# collect results\nreduceResultsList(reg = tmp)\n
    #> [[1]]\n#> [1] 1\n#> \n#> [[2]]\n#> [1] 2\n#> \n#> [[3]]\n#> [1] 3\n#> \n#> [[4]]\n#> [1] 4\n#> \n#> [[5]]\n#> [1] 5\n#> \n#> [[6]]\n#> [1] 6\n#> \n#> [[7]]\n#> [1] 7\n#> \n#> [[8]]\n#> [1] 8\n#> \n#> [[9]]\n#> [1] 9\n#> \n#> [[10]]\n#> [1] 10\n#> \n#> [[11]]\n#> [1] 11\n#> \n#> [[12]]\n#> [1] 12\n#> \n#> [[13]]\n#> [1] 13\n#> \n#> [[14]]\n#> [1] 14\n#> \n#> [[15]]\n#> [1] 15\n#> \n#> [[16]]\n#> [1] 16\n#> \n#> [[17]]\n#> [1] 17\n#> \n#> [[18]]\n#> [1] 18\n#> \n#> [[19]]\n#> [1] 19\n#> \n#> [[20]]\n#> [1] 20\n#> 
    \n### Example 2: Using memory measurement\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg2' using cluster functions 'Interactive'
    \n# Toy function which creates a large matrix and returns the column sums\nfun = function(n, p) colMeans(matrix(runif(n*p), n, p))\n\n# Arguments to fun:\nargs = data.table::CJ(n = c(1e4, 1e5), p = c(10, 50)) # like expand.grid()\nprint(args)\n
    #>        n  p\n#> 1: 1e+04 10\n#> 2: 1e+04 50\n#> 3: 1e+05 10\n#> 4: 1e+05 50
    \n# Map function to create jobs\nids = batchMap(fun, args = args, reg = tmp)\n
    #> Adding 4 jobs ...
    \n# Set resources: enable memory measurement\nres = list(measure.memory = TRUE)\n\n# Submit jobs using the currently configured cluster functions\nsubmitJobs(ids, resources = res, reg = tmp)\n
    #> Submitting 4 jobs in 4 chunks using cluster functions 'Interactive' ...
    \n# Retrive information about memory, combine with parameters\ninfo = ijoin(getJobStatus(reg = tmp)[, .(job.id, mem.used)], getJobPars(reg = tmp))\nprint(unwrap(info))\n
    #>    job.id mem.used     n  p\n#> 1:      1 145.3133 1e+04 10\n#> 2:      2 145.3144 1e+04 50\n#> 3:      3 145.3146 1e+05 10\n#> 4:      4 145.3153 1e+05 50
    \n# Combine job info with results -> each job is aggregated using mean()\nunwrap(ijoin(info, reduceResultsDataTable(fun = function(res) list(res = mean(res)), reg = tmp)))\n
    #>    job.id mem.used     n  p       res\n#> 1:      1 145.3133 1e+04 10 0.5005778\n#> 2:      2 145.3144 1e+04 50 0.4992527\n#> 3:      3 145.3146 1e+05 10 0.5000026\n#> 4:      4 145.3153 1e+05 50 0.4999301
    \n### Example 3: Multicore execution on the slave\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg3' using cluster functions 'Interactive'
    \n# Function which sleeps 10 seconds, i-times\nf = function(i) {\n  parallelMap::parallelMap(Sys.sleep, rep(10, i))\n}\n\n# Create one job with parameter i=4\nids = batchMap(f, i = 4, reg = tmp)\n
    #> Adding 1 jobs ...
    \n# Set resources: Use parallelMap in multicore mode with 4 CPUs\n# batchtools internally loads the namespace of parallelMap and then\n# calls parallelStart() before the job and parallelStop() right\n# after the job last job in the chunk terminated.\nres = list(pm.backend = \"multicore\", ncpus = 4)\n\nif (FALSE) {\n# Submit both jobs and wait for them\nsubmitJobs(resources = res, reg = tmp)\nwaitForJobs(reg = tmp)\n\n# If successfull, the running time should be ~10s\ngetJobTable(reg = tmp)[, .(job.id, time.running)]\n\n# There should also be a note in the log:\ngrepLogs(pattern = \"parallelMap\", reg = tmp)\n}\n
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/summarizeExperiments.html", "content": "\n\n\n \n \n\n\n\nQuick Summary over Experiments — summarizeExperiments • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Quick Summary over Experiments

    \n Source: R/summarizeExperiments.R\n
    summarizeExperiments.Rd
    \n
    \n\n
    \n

    Returns a frequency table of defined experiments.\nSee ExperimentRegistry for an example.

    \n
    \n\n 
    summarizeExperiments(\n  ids = NULL,\n  by = c(\"problem\", \"algorithm\"),\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.

    by

    [character]
    \nSplit the resulting table by columns of getJobPars.

    reg

    [ExperimentRegistry]
    \nRegistry. If not explicitly passed, uses the last created registry.

    \n\n

    Value

    \n\n

    [data.table] of frequencies.

    \n

    See also

    \n\n

    Other Experiment: \naddExperiments(),\nremoveExperiments()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/sweepRegistry.html", "content": "\n\n\n \n \n\n\n\nCheck Consistency and Remove Obsolete Information — sweepRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Check Consistency and Remove Obsolete Information

    \n Source: R/sweepRegistry.R\n
    sweepRegistry.Rd
    \n
    \n\n
    \n

    Canceled jobs and jobs submitted multiple times may leave stray files behind.\nThis function checks the registry for consistency and removes obsolete files\nand redundant data base entries.

    \n
    \n\n 
    sweepRegistry(reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nloadRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsyncRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/syncRegistry.html", "content": "\n\n\n \n \n\n\n\nSynchronize the Registry — syncRegistry • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Synchronize the Registry

    \n Source: R/syncRegistry.R\n
    syncRegistry.Rd
    \n
    \n\n
    \n

    Parses update files written by the slaves to the file system and updates the\ninternal data base.

    \n
    \n\n 
    syncRegistry(reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n
    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [logical(1)]: TRUE if the state has changed, FALSE otherwise.

    \n

    See also

    \n\n

    Other Registry: \nclearRegistry(),\ngetDefaultRegistry(),\nloadRegistry(),\nmakeRegistry(),\nremoveRegistry(),\nsaveRegistry(),\nsweepRegistry()

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/testJob.html", "content": "\n\n\n \n \n\n\n\nRun Jobs Interactively — testJob • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Run Jobs Interactively

    \n Source: R/testJob.R\n
    testJob.Rd
    \n
    \n\n
    \n

    Starts a single job on the local machine.

    \n
    \n\n 
    testJob(id, external = FALSE, reg = getDefaultRegistry())
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    id

    [integer(1) or data.table]
    \nSingle integer to specify the job or a data.table with column job.id\nand exactly one row.

    external

    [logical(1)]
    \n Run the job in an external R session? If TRUE, starts a fresh R\n session on the local machine to execute the with execJob.\n You will not be able to use debug tools like traceback\n or browser.

    \n

    If external is set to FALSE (default) on the other hand,\n testJob will execute the job in the current R session and the usual\n debugging tools work. However, spotting missing variable declarations (as they\n are possibly resolved in the global environment) is impossible.\n Same holds for missing package dependency declarations.

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    Returns the result of the job if successful.

    \n

    See also

    \n\n

    Other debug: \ngetErrorMessages(),\ngetStatus(),\ngrepLogs(),\nkillJobs(),\nresetJobs(),\nshowLog()

    \n\n

    Examples

    \n 
     batchtools:::example_push_temp(1) \ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n
    #> No readable configuration file found
    #> Created registry in '/tmp/batchtools-example/reg' using cluster functions 'Interactive'
    batchMap(function(x) if (x == 2) xxx else x, 1:2, reg = tmp)\n
    #> Adding 2 jobs ...
    testJob(1, reg = tmp)\n
    #> ### [bt]: Setting seed to 11688 ...
    #> [1] 1
    if (FALSE) {\ntestJob(2, reg = tmp)\n}\n
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/unwrap.html", "content": "\n\n\n \n \n\n\n\nUnwrap Nested Data Frames — unwrap • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Unwrap Nested Data Frames

    \n Source: R/unwrap.R\n
    unwrap.Rd
    \n
    \n\n
    \n

    Some functions (e.g., getJobPars, getJobResources or reduceResultsDataTable\nreturn a data.table with columns of type list.\nThese columns can be unnested/unwrapped with this function.\nThe contents of these columns will be transformed to a data.table and cbind-ed\nto the input data.frame x, replacing the original nested column.

    \n
    \n\n 
    unwrap(x, cols = NULL, sep = NULL)\n\nflatten(x, cols = NULL, sep = NULL)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    x

    [data.frame | data.table]
    \nData frame to flatten.

    cols

    [character]
    \nColumns to consider for this operation. If set to NULL (default),\nwill operate on all columns of type “list”.

    sep

    [character(1)]
    \nIf NULL (default), the column names of the additional columns will re-use the names\nof the nested list/data.frame.\nThis may lead to name clashes.\nIf you provide sep, the variable column name will be constructed as\n“[column name of x][sep][inner name]”.

    \n\n

    Value

    \n\n

    [data.table].

    \n

    Note

    \n\n

    There is a name clash with function flatten in package purrr.\nThe function flatten is discouraged to use for this reason in favor of unwrap.

    \n\n

    Examples

    \n 
    x = data.table::data.table(\n  id = 1:3,\n  values = list(list(a = 1, b = 3), list(a = 2, b = 2), list(a = 3))\n)\nunwrap(x)\n
    #>    id a  b\n#> 1:  1 1  3\n#> 2:  2 2  2\n#> 3:  3 3 NA
    unwrap(x, sep = \".\")\n
    #>    id values.a values.b\n#> 1:  1        1        3\n#> 2:  2        2        2\n#> 3:  3        3       NA
    \n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "docs/reference/waitForJobs.html", "content": "\n\n\n \n \n\n\n\nWait for Termination of Jobs — waitForJobs • batchtools\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n \n
    \n
    \n
    \n
    \n
    \n \n \n batchtools\n 0.9.14\n \n
    \n\n
    \n \n \n \n
    \n
    \n
    \n\n \n\n
    \n\n
    \n
    \n
    \n

    Wait for Termination of Jobs

    \n Source: R/waitForJobs.R\n
    waitForJobs.Rd
    \n
    \n\n
    \n

    This function simply waits until all jobs are terminated.

    \n
    \n\n 
    waitForJobs(\n  ids = NULL,\n  sleep = NULL,\n  timeout = 604800,\n  expire.after = NULL,\n  stop.on.error = FALSE,\n  stop.on.expire = FALSE,\n  reg = getDefaultRegistry()\n)
    \n\n

    Arguments

    \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    ids

    [data.frame or integer]
    \nA data.frame (or data.table)\nwith a column named “job.id”.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of findSubmitted.\nInvalid ids are ignored.

    sleep

    [function(i) | numeric(1)]
    \nParameter to control the duration to sleep between queries.\nYou can pass an absolute numeric value in seconds or a function(i) which returns\nthe number of seconds to sleep in the i-th iteration.\nIf not provided (NULL), tries to read the value (number/function) from the configuration file\n(stored in reg$sleep) or defaults to a function with exponential backoff between\n5 and 120 seconds.

    timeout

    [numeric(1)]
    \nAfter waiting timeout seconds, show a message and return\nFALSE. This argument may be required on some systems where, e.g.,\nexpired jobs or jobs on hold are problematic to detect. If you don't want\na timeout, set this to Inf. Default is 604800 (one week).

    expire.after

    [integer(1)]
    \nJobs count as “expired” if they are not found on the system but have not communicated back\ntheir results (or error message). This frequently happens on managed system if the scheduler kills\na job because the job has hit the walltime or request more memory than reserved.\nOn the other hand, network file systems often require several seconds for new files to be found,\nwhich can lead to false positives in the detection heuristic.\nwaitForJobs treats such jobs as expired after they have not been detected on the system\nfor expire.after iterations.\nIf not provided (NULL), tries to read the value from the configuration file (stored in reg$expire.after),\nand finally defaults to 3.

    stop.on.error

    [logical(1)]
    \nImmediately cancel if a job terminates with an error? Default is\nFALSE.

    stop.on.expire

    [logical(1)]
    \nImmediately cancel if jobs are detected to be expired? Default is FALSE.\nExpired jobs will then be ignored for the remainder of waitForJobs().

    reg

    [Registry]
    \nRegistry. If not explicitly passed, uses the default registry (see setDefaultRegistry).

    \n\n

    Value

    \n\n

    [logical(1)]. Returns TRUE if all jobs terminated\n successfully and FALSE if either the timeout is reached or at least\n one job terminated with an exception or expired.

    \n\n
    \n
    \n \n
    \n
    \n\n\n
    \n
    \n

    Developed by Michel Lang, Bernd Bischl.

    \n
    \n\n
    \n

    Site built with pkgdown 1.6.1.

    \n
    \n\n
    \n
    \n\n \n\n\n \n\n\n\n" }, { "path": "inst/CITATION", "content": "## -*- mode: r -*-\n\ncitHeader(\"To cite BatchJobs, BatchExperiments or batchtools in publications use:\")\n\nbibentry(\"Article\",\n title = \"batchtools: Tools for R to work on batch systems\",\n author = c(as.person(\"Michel Lang\"),\n as.person(\"Bernd Bischl\"),\n as.person(\"Dirk Surmann\")),\n journal = \"The Journal of Open Source Software\",\n year = \"2017\",\n month = \"feb\",\n number = \"10\",\n doi = \"10.21105/joss.00135\",\n url = \"https://doi.org/10.21105/joss.00135\",\n textVersion =\n paste(\"Michel Lang, Bernd Bischl, Dirk Surmann (2017).\",\n \"batchtools: Tools for R to work on batch systems.\",\n \"The Journal of Open Source Software, 2(10).\",\n \"URL https://doi.org/10.21105/joss.00135.\")\n)\n\nbibentry(\"Article\",\n title = \"{BatchJobs} and {BatchExperiments}: Abstraction Mechanisms for Using {R} in Batch Environments\",\n author = c(as.person(\"Bernd Bischl\"),\n as.person(\"Michel Lang\"),\n as.person(\"Olaf Mersmann\"),\n as.person(\"J{\\\\\\\"o}rg Rahnenf{\\\\\\\"u}hrer\"),\n as.person(\"Claus Weihs\")),\n journal = \"Journal of Statistical Software\",\n year = \"2015\",\n volume = \"64\",\n number = \"11\",\n pages = \"1--25\",\n doi = \"10.18637/jss.v064.i11\",\n url = \"https://www.jstatsoft.org/v64/i11/\",\n textVersion =\n paste(\"Bernd Bischl, Michel Lang, Olaf Mersmann, Joerg Rahnenfuehrer, Claus Weihs (2015).\",\n \"BatchJobs and BatchExperiments: Abstraction Mechanisms for Using R in Batch Environments.\",\n \"Journal of Statistical Software, 64(11), 1-25.\",\n \"URL https://www.jstatsoft.org/v64/i11/.\")\n)\n" }, { "path": "inst/bin/linux-helper", "content": "#!/bin/bash\n\n## linux-helper: Helper for the multicore and SSH cluster functions of the BatchJobs R\n## package.\n##\n## Requires the following Unix command line utilities:\n##\n## * grep, wc, ps, kill, uptime, echo, cat, possibly setsid\n##\n## The following commands are implemented. First argument is always the command name.\n## For other arguments see below. Each command returns a character vector.\n##\n## number-of-cpus\n## Return the number of PEs on worker.\n##\n## start-job NICE JOBFILE OUTFILE\n## Start an Rscript process running doJobCollection on $JOBFILE and log\n## the output in $OUTFILE.\n## Returns: PID of sh process which spawned R. We use that as batch.id.\n##\n## kill-job PID\n## Kill the R job with PID $PID. The PID is the PID of\n## the sh process returned by start-job.\n## First a TERM is sent, then 1 sec delay, then KILL.\n##\n## status FILEDIR\n## Return 4 numbers:\n## - load average of last 1 min, as given by e.g. uptime\n## - number of R processes by _all_ users\n## - number of R processes by _all_ users which have a load of >= 50%\n## - number of R processes by current user\n## which match $FILEDIR/jobs in the cmd call of R\n##\n## list-jobs FILEDIR\n## Return the PIDs of running R jobs operating on $FILEDIR/jobs.\n\necho \"[bt] --BOF--\"\n\nCMD=\"$1\"; shift\nexport LC_ALL=C ### Avoid any localization issues.\nshopt -s nocasematch ### Case insensitive regular expressions\n\ncase $CMD in\n number-of-cpus)\n if [[ `uname` =~ \"Linux\" ]]; then\n NCPU=`cat /proc/cpuinfo | grep '^processor' | wc -l`\n else ## darwin\n NCPU=`sysctl -n hw.ncpu`\n fi\n echo \"[bt]\" $NCPU\n ;;\n start-job)\n Rscript -e \"batchtools::doJobCollection('$1')\" > \"$2\" 2>&1 &\n echo \"[bt]\" $!\n ;;\n kill-job)\n kill -TERM $1 > /dev/null 2> /dev/null\n sleep 1\n kill -KILL $1 > /dev/null 2> /dev/null\n exit 0\n ;;\n status)\n # remove everyting till load average(s), then delete commas\n LOAD=$(uptime | awk '{gsub(/.*:/,\"\"); {gsub(/,/,\"\")}; print $1}')\n JOBDIR=\"$1/jobs\"\n # print 3 columns for all processes\n # use ww for unlimited width in ps for command output\n # we count all R procs, all R50, and all where JOBDIR was in the call args\n ps -e -ww -o pcpu= -o ucomm= -o command= | \\\n awk -v j=$JOBDIR -v sysload=$LOAD '\n BEGIN {rprocs=0;rprocs_50=0;njobs=0}\n $2 != \"R\" {next}\n {rprocs++}\n $1 > 50.0 {rprocs_50++}\n $0 ~ j {njobs++}\n END {print \"[bt] \" sysload \" \" rprocs \" \" rprocs_50 \" \" njobs}'\n ;;\n list-jobs)\n JOBDIR=\"$1/jobs\"\n ps -e -ww -o pid= -o ucomm= -o command= | awk -v j=$JOBDIR '$2 == \"R\" && $0 ~ j { print \"[bt] \" $1 }'\n ;;\n *)\nesac\n\necho \"[bt] --EOF--\"\n" }, { "path": "inst/templates/lsf-simple.tmpl", "content": "## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named list.\n\n#BSUB -J <%= job.name %> # Name of the job\n#BSUB -o <%= log.file %> # Output is sent to logfile, stdout + stderr by default\n#BSUB -q <%= resources$queue %> # Job queue\n#BSUB -W <%= round(resources$walltime / 60, 1) %> # Walltime (LSF requires minutes, batchtools uses seconds)\n#BSUB -M <%= resources$memory %> # Memory requirements, e.g. \"5000KB\", \"500MB\", \"5GB\" etc. \n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\nRscript -e 'batchtools::doJobCollection(\"<%= uri %>\")'\n" }, { "path": "inst/templates/openlava-simple.tmpl", "content": "## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named list.\n\n## Remove [*] if arrayjobs are not supported\n#BSUB-J <%= job.name %>[1-<%= nrow(jobs) %>] # name of the job / number of jobs in chunk\n#BSUB-o <%= log.file %> # output is sent to logfile, stdout + stderr by default\n#BSUB-q <%= resources$queue %> # Job queue\n#BSUB-W <%= resources$walltime %> # Walltime in minutes\n#BSUB-M <%= resources$memory %> # Memory requirements in Kbytes\n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\nRscript -e 'batchtools::doJobCollection(\"<%= uri %>\")'\n" }, { "path": "inst/templates/sge-simple.tmpl", "content": "#!/bin/bash\n\n## The name of the job, can be anything, simply used when displaying the list of running jobs\n#$ -N <%= job.name %>\n\n## Combining output/error messages into one file\n#$ -j y\n\n## Giving the name of the output log file\n#$ -o <%= log.file %>\n\n## One needs to tell the queue system to use the current directory as the working directory\n## Or else the script may fail as it will execute in your top level home directory /home/username\n#$ -cwd\n\n## Use environment variables\n#$ -V\n\n## Use correct queue\n#$ -q <%= resources$queue %>\n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\nRscript -e 'batchtools::doJobCollection(\"<%= uri %>\")'\nexit 0\n" }, { "path": "inst/templates/slurm-dortmund.tmpl", "content": "#!/bin/bash\n<%\nbackend = resources$pm.backend %??% \"local\"\nncpus = resources$ncpus %??% 1L\nwalltime = asInt(resources$walltime, lower = 1L, upper = 172800L)\nmemory = asInt(resources$memory, lower = 100L, upper = 64000L)\n\nif (backend == \"mpi\") {\n cmd = \"mpirun -np 1 Rscript\"\n mincpus = 2L\n} else {\n cmd = \"Rscript\"\n mincpus = 1L\n}\n\n# relative paths are not handled well by Slurm\nlog.file = fs::path_expand(log.file)\n-%>\n\n#SBATCH --job-name=<%= job.name %>\n#SBATCH --output=<%= log.file %>\n#SBATCH --error=<%= log.file %>\n#SBATCH --time=<%= ceiling(walltime / 60L)%>\n#SBATCH --ntasks=<%= if (backend == \"mpi\") ncpus else 1L %>\n#SBATCH --mincpus=<%= mincpus %>\n#SBATCH --cpus-per-task=<%= if (backend == \"mpi\") 1L else ncpus %>\n#SBATCH --mem-per-cpu=<%= memory %>\n#SBATCH --partition=all\n\nmkdir /tmp/${USER}-${SLURM_JOBID}\nexport TMPDIR=/tmp/${USER}-${SLURM_JOBID}\n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\nsource /etc/profile\n<%= cmd %> -e 'batchtools::doJobCollection(\"<%= uri %>\")'\nrm -rf /tmp/${USER}-${SLURM_JOBID}\n" }, { "path": "inst/templates/slurm-lido3.tmpl", "content": "#!/bin/bash\n\n## Job Resource Interface Definition\n##\n## ncpus [integer(1)]: Number of required cpus per task,\n## Set larger than 1 if you want to further parallelize\n## with multicore/parallel within each task.\n## walltime [integer(1)]: Walltime for this job, in seconds.\n## Must be at least 1 minute.\n## memory [integer(1)]: Memory in megabytes for each cpu.\n## Must be at least 100 (when I tried lower values my\n## jobs did not start at all).\n##\n## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named list.\n\n<%\n\n# queue\nwalltime = asInt(resources$walltime, lower = 60L, upper = 31L * 24L * 60L * 60L)\nmemory = asInt(resources$memory, lower = 100L, upper = 1024L * 1024L)\nwalltimes = c(2L, 8L, 48L, 672L) * 3600L\nqueue = c(\"short\", \"med\", \"long\", \"ultralong\")[wf(walltime <= walltimes)]\nncpus = if (!is.null(resources$ncpus)) ncpus = assertInt(resources$ncpus, lower = 1L) else 1L\n\n# modules\nmodules = paste(resources$modules, resources$R)\n\n# cli args\ncli.args = \"\"\nif (!is.null(resources$pp.size))\n cli.args = sprintf(\"--max-ppsize=%i\", assertInt(pp.size, upper = 500000L))\n-%>\n\n#SBATCH --job-name=<%= job.name %>\n#SBATCH --output=<%= log.file %>\n#SBATCH --error=<%= log.file %>\n#SBATCH --time=<%= ceiling(walltime / 60L) %>\n#SBATCH --partition=<%= queue %>\n#SBATCH --cpus-per-task=<%= ncpus %>\n#SBATCH --mem-per-cpu=<%= memory %>\n<%= if (array.jobs) sprintf(\"#SBATCH --array=1-%i\", nrow(jobs)) else \"\" %>\n\n## Initialize work environment like\nmodule add <%= modules %>\n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n\n## Use /scratch on the node, TMPDIR is mounted as tmpfs\nexport TMPDIR=/scratch/${USER}/${SLURM_JOBID}\nmkdir -p ${TMPDIR}\n\n## Run R:\n## we merge R output with stdout from SLURM, which gets then logged via --output option\nRscript <%= cli.args -%> -e 'batchtools::doJobCollection(\"<%= uri %>\")'\n" }, { "path": "inst/templates/slurm-simple.tmpl", "content": "#!/bin/bash\n\n## Job Resource Interface Definition\n##\n## ntasks [integer(1)]: Number of required tasks,\n## Set larger than 1 if you want to further parallelize\n## with MPI within your job.\n## ncpus [integer(1)]: Number of required cpus per task,\n## Set larger than 1 if you want to further parallelize\n## with multicore/parallel within each task.\n## walltime [integer(1)]: Walltime for this job, in seconds.\n## Must be at least 60 seconds for Slurm to work properly.\n## memory [integer(1)]: Memory in megabytes for each cpu.\n## Must be at least 100 (when I tried lower values my\n## jobs did not start at all).\n##\n## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named list.\n\n<%\n# relative paths are not handled well by Slurm\nlog.file = fs::path_expand(log.file)\n-%>\n\n\n#SBATCH --job-name=<%= job.name %>\n#SBATCH --output=<%= log.file %>\n#SBATCH --error=<%= log.file %>\n#SBATCH --time=<%= ceiling(resources$walltime / 60) %>\n#SBATCH --ntasks=1\n#SBATCH --cpus-per-task=<%= resources$ncpus %>\n#SBATCH --mem-per-cpu=<%= resources$memory %>\n<%= if (!is.null(resources$partition)) sprintf(paste0(\"#SBATCH --partition='\", resources$partition, \"'\")) %>\n<%= if (array.jobs) sprintf(\"#SBATCH --array=1-%i\", nrow(jobs)) else \"\" %>\n\n## Initialize work environment like\n## source /etc/profile\n## module add ...\n\n## Export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\n## Run R:\n## we merge R output with stdout from SLURM, which gets then logged via --output option\nRscript -e 'batchtools::doJobCollection(\"<%= uri %>\")'\n" }, { "path": "inst/templates/testJob.tmpl", "content": "options(warn = 1L)\nSys.setenv(DEBUGME = \"<%= Sys.getenv('DEBUGME') %>\")\nrequireNamespace(\"batchtools\", quietly = TRUE)\njc = force(readRDS(\"<%= jc %>\"))\nsetwd(jc$work.dir)\nbatchtools:::loadRegistryDependencies(jc, must.work = TRUE)\nres = batchtools::execJob(jc)\nsaveRDS(res, file = \"<%= result %>\", version = 2L)\nquit(save = \"no\", status = 0L)\n\n# vim: ft=r\n" }, { "path": "inst/templates/torque-lido.tmpl", "content": "#!/bin/bash\n<%\n## Check some resources and set sane defaults\nresources$walltime = asInt(resources$walltime, lower = 60L, upper = 3600L * 672L)\nresources$memory = asInt(resources$memory, lower = 100L, upper = 64L * 1024L)\nresources$ncpus = if (is.null(resources$ncpus)) 1L else asInt(resources$ncpus, lower = 1L)\nresources$modules = if (is.null(resources$modules)) character(0L) else assertCharacter(resources$modules, any.missing = FALSE)\nresources$R = if (is.null(resources$R)) \"R/3.4.1-gcc49-base\" else assertString(resources$R)\nresources$omp.threads = if (is.null(resources$omp.threads)) 1L else asInt(resources$omp.threads, lower = 1L)\nresources$blas.threads = if (is.null(resources$blas.threads)) 1L else asInt(resources$blas.threads, lower = 1L)\n\nif (!is.null(resources$type)) assertString(resources$type)\nif (resources$memory > 15000)\n resources$type = \"quad\"\n\nuse.mpi = (resources$pm.backend %??% \"default\") == \"mpi\"\n\n## first string of queue, selected by walltime\nwalltimes = 3600L * c(1L, 8L, 48L, 672L)\nqueue = c(\"short\", \"med\", \"long\", \"ultralong\")[wf(resources$walltime <= walltimes)]\n## check default modules\nmodules.default = c(binutils = \"binutils/2.25\", gcc = \"gcc/4.9.3\", openblas = \"openblas/0.2.17\")\nmodules = resources$modules\nfor (i in seq_along(modules.default)) {\n if (!any(grepl(paste0(\"^\", names(modules.default[i]), \"/?[0-9\\\\.]*$\"), modules))) {\n modules = paste(modules, modules.default[i])\n }\n}\n## add R\nmodules = paste(modules, resources$R)\n## add mpi\nif (use.mpi)\n modules = paste(modules, \"openmpi/gcc4.9.x\")\n\n## very ugly hack because we cannot log to data (nobackup) filesystem on lido,\n## only home fs is available\n## unfortunately there seems to be no generic solution\n## does log path start with /data/?\nlog.file = log.file\nif (length(grep(\"^/data/\", log.file)) > 0L) {\n ## strip that\n log.file = substr(log.file, 7L, nchar(log.file))\n ## find next forward slash\n i = regexpr(\"/\", log.file)\n if (i != -1) {\n ## this must be \"user\": e.g. /data/bischl/...\n user = substr(log.file, 1L, i-1L)\n ## put together\n log.file = sprintf(\"/home/%s/nobackup%s\", user, substr(log.file, i, nchar(log.file)))\n }\n}\n-%>\n\n#PBS -N <%= job.name %>\n#PBS -o <%= log.file %>\n#PBS -l walltime=<%= resources$walltime %>,nodes=<%= if (use.mpi) resources$ncpus else 1 %>:ppn=<%= if (use.mpi) 1 else resources$ncpus %><%= if (!is.null(resources$type)) paste0(\":\", resources$type) %>,vmem=<%= resources$memory %>M\n#PBS -q <%= queue %>\n#PBS -j oe\n<%= if (array.jobs) sprintf(\"#PBS -t 1-%i\", nrow(jobs)) else \"\" %>\n\n## setup modules\nsource /sysdata/shared/sfw/Modules/default/init/bash\nmodule add <%= modules %>\n\n## create our own temp dir (and clean it up later), lido does not do this automatically\nmkdir /scratch/${USER}-${PBS_JOBID}\nexport TMPDIR=/scratch/${USER}-${PBS_JOBID}\n\n<%= sprintf(\"export OMP_NUM_THREADS=%i\", resources$omp.threads) -%>\n<%= sprintf(\"export OPENBLAS_NUM_THREADS=%i\", resources$blas.threads) -%>\n<%= sprintf(\"export MKL_NUM_THREADS=%i\", resources$blas.threads) -%>\n\n## export value of DEBUGME environemnt var to slave\nexport DEBUGME=<%= Sys.getenv(\"DEBUGME\") %>\n\n## run R\n<%= if (use.mpi) \"mpirun -np 1 \" else \"\" %>Rscript -e 'batchtools::doJobCollection(\"<%= uri %>\")'\n\n## Cleanup\nrm -rf /scratch/${USER}-${PBS_JOBID}\n" }, { "path": "man/JobCollection.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/JobCollection.R\n\\name{makeJobCollection}\n\\alias{makeJobCollection}\n\\alias{JobCollection}\n\\title{JobCollection Constructor}\n\\usage{\nmakeJobCollection(ids = NULL, resources = list(), reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{resources}{[\\code{list}]\\cr\nNamed list of resources. Default is \\code{list()}.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{JobCollection}].\n}\n\\description{\n\\code{makeJobCollection} takes multiple job ids and creates an object of class \\dQuote{JobCollection} which holds all\nnecessary information for the calculation with \\code{\\link{doJobCollection}}. It is implemented as an environment\nwith the following variables:\n\\describe{\n \\item{file.dir}{\\code{file.dir} of the \\link{Registry}.}\n \\item{work.dir:}{\\code{work.dir} of the \\link{Registry}.}\n \\item{job.hash}{Unique identifier of the job. Used to create names on the file system.}\n \\item{jobs}{\\code{\\link[data.table]{data.table}} holding individual job information. See examples.}\n \\item{log.file}{Location of the designated log file for this job.}\n \\item{resources:}{Named list of of specified computational resources.}\n \\item{uri}{Location of the job description file (saved with \\code{link[base]{saveRDS}} on the file system.}\n \\item{seed}{\\code{integer(1)} Seed of the \\link{Registry}.}\n \\item{packages}{\\code{character} with required packages to load via \\code{\\link[base]{require}}.}\n \\item{namespaces}{\\code{character} with required packages to load via \\code{\\link[base]{requireNamespace}}.}\n \\item{source}{\\code{character} with list of files to source before execution.}\n \\item{load}{\\code{character} with list of files to load before execution.}\n \\item{array.var}{\\code{character(1)} of the array environment variable specified by the cluster functions.}\n \\item{array.jobs}{\\code{logical(1)} signaling if jobs were submitted using \\code{chunks.as.arrayjobs}.}\n}\nIf your \\link{ClusterFunctions} uses a template, \\code{\\link[brew]{brew}} will be executed in the environment of such\na collection. Thus all variables available inside the job can be used in the template.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE, packages = \"methods\")\nbatchMap(identity, 1:5, reg = tmp)\n\n# resources are usually set in submitJobs()\njc = makeJobCollection(1:3, resources = list(foo = \"bar\"), reg = tmp)\nls(jc)\njc$resources\n}\n\\seealso{\nOther JobCollection: \n\\code{\\link{doJobCollection}()}\n}\n\\concept{JobCollection}\n" }, { "path": "man/JobExperiment.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Job.R\n\\name{makeJob}\n\\alias{makeJob}\n\\alias{Job}\n\\alias{Experiment}\n\\title{Jobs and Experiments}\n\\usage{\nmakeJob(id, reader = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{id}{[\\code{integer(1)} or \\code{data.table}]\\cr\nSingle integer to specify the job or a \\code{data.table} with column \\code{job.id}\nand exactly one row.}\n\n\\item{reader}{[\\code{RDSReader} | \\code{NULL}]\\cr\nReader object to retrieve files. Used internally to cache reading from the file system.\nThe default (\\code{NULL}) does not make use of caching.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{Job} | \\code{Experiment}].\n}\n\\description{\nJobs and Experiments are abstract objects which hold all information necessary to execute a single computational\njob for a \\code{\\link{Registry}} or \\code{\\link{ExperimentRegistry}}, respectively.\n\nThey can be created using the constructor \\code{makeJob} which takes a single job id.\nJobs and Experiments are passed to reduce functions like \\code{\\link{reduceResults}}.\nFurthermore, Experiments can be used in the functions of the \\code{\\link{Problem}} and \\code{\\link{Algorithm}}.\nJobs and Experiments hold these information:\n\\describe{\n \\item{\\code{job.id}}{Job ID as integer.}\n \\item{\\code{pars}}{\n Job parameters as named list.\n For \\code{\\link{ExperimentRegistry}}, the parameters are divided into the sublists \\dQuote{prob.pars} and \\dQuote{algo.pars}.\n }\n \\item{\\code{seed}}{Seed which is set via \\code{\\link{doJobCollection}} as scalar integer.}\n \\item{\\code{resources}}{Computational resources which were set for this job as named list.}\n \\item{\\code{external.dir}}{\n Path to a directory which is created exclusively for this job. You can store external files here.\n Directory is persistent between multiple restarts of the job and can be cleaned by calling \\code{\\link{resetJobs}}.\n }\n \\item{\\code{fun}}{Job only: User function passed to \\code{\\link{batchMap}}.}\n \\item{\\code{prob.name}}{Experiments only: Problem id.}\n \\item{\\code{algo.name}}{Experiments only: Algorithm id.}\n \\item{\\code{problem}}{Experiments only: \\code{\\link{Problem}}.}\n \\item{\\code{instance}}{Experiments only: Problem instance.}\n \\item{\\code{algorithm}}{Experiments only: \\code{\\link{Algorithm}}.}\n \\item{\\code{repl}}{Experiments only: Replication number.}\n}\n\nNote that the slots \\dQuote{pars}, \\dQuote{fun}, \\dQuote{algorithm} and \\dQuote{problem}\nlazy-load required files from the file system and construct the object on the first access.\nThe realizations are cached for all slots except \\dQuote{instance} (which might be stochastic).\n\nJobs and Experiments can be executed manually with \\code{\\link{execJob}}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(function(x, y) x + y, x = 1:2, more.args = list(y = 99), reg = tmp)\nsubmitJobs(resources = list(foo = \"bar\"), reg = tmp)\njob = makeJob(1, reg = tmp)\nprint(job)\n\n# Get the parameters:\njob$pars\n\n# Get the job resources:\njob$resources\n\n# Execute the job locally:\nexecJob(job)\n}\n" }, { "path": "man/JobNames.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/JobNames.R\n\\name{JobNames}\n\\alias{JobNames}\n\\alias{setJobNames}\n\\alias{getJobNames}\n\\title{Set and Retrieve Job Names}\n\\usage{\nsetJobNames(ids = NULL, names, reg = getDefaultRegistry())\n\ngetJobNames(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{names}{[\\code{character}]\\cr\nCharacter vector of the same length as provided ids.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n\\code{setJobNames} returns \\code{NULL} invisibly, \\code{getJobTable}\n returns a \\code{data.table} with columns \\code{job.id} and \\code{job.name}.\n}\n\\description{\nSet custom names for jobs. These are passed to the template as \\sQuote{job.name}.\nIf no custom name is set (or any of the job names of the chunk is missing),\nthe job hash is used as job name.\nIndividual job names can be accessed via \\code{jobs$job.name}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nids = batchMap(identity, 1:10, reg = tmp)\nsetJobNames(ids, letters[1:nrow(ids)], reg = tmp)\ngetJobNames(reg = tmp)\n}\n" }, { "path": "man/JoinTables.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Joins.R\n\\name{JoinTables}\n\\alias{JoinTables}\n\\alias{ijoin}\n\\alias{ljoin}\n\\alias{rjoin}\n\\alias{ojoin}\n\\alias{sjoin}\n\\alias{ajoin}\n\\alias{ujoin}\n\\title{Inner, Left, Right, Outer, Semi and Anti Join for Data Tables}\n\\usage{\nijoin(x, y, by = NULL)\n\nljoin(x, y, by = NULL)\n\nrjoin(x, y, by = NULL)\n\nojoin(x, y, by = NULL)\n\nsjoin(x, y, by = NULL)\n\najoin(x, y, by = NULL)\n\nujoin(x, y, all.y = FALSE, by = NULL)\n}\n\\arguments{\n\\item{x}{[\\code{\\link{data.frame}}]\\cr\nFirst data.frame to join.}\n\n\\item{y}{[\\code{\\link{data.frame}}]\\cr\nSecond data.frame to join.}\n\n\\item{by}{[\\code{character}]\\cr\nColumn name(s) of variables used to match rows in \\code{x} and \\code{y}.\nIf not provided, a heuristic similar to the one described in the \\pkg{dplyr} vignette is used:\n\\enumerate{\n \\item If \\code{x} is keyed, the existing key will be used if \\code{y} has the same column(s).\n \\item If \\code{x} is not keyed, the intersect of common columns names is used if not empty.\n \\item Raise an exception.\n}\nYou may pass a named character vector to merge on columns with different names in \\code{x} and\n\\code{y}: \\code{by = c(\"x.id\" = \"y.id\")} will match \\code{x}'s \\dQuote{x.id} column with \\code{y}\\'s\n\\dQuote{y.id} column.}\n\n\\item{all.y}{[logical(1)]\\cr\nKeep columns of \\code{y} which are not in \\code{x}?}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with key identical to \\code{by}.\n}\n\\description{\nThese helper functions perform join operations on data tables.\nMost of them are basically one-liners.\nSee \\url{https://rpubs.com/ronasta/join_data_tables} for a overview of join operations in\ndata table or alternatively \\pkg{dplyr}'s vignette on two table verbs.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\n# Create two tables for demonstration\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(identity, x = 1:6, reg = tmp)\nx = getJobPars(reg = tmp)\ny = findJobs(x >= 2 & x <= 5, reg = tmp)\ny$extra.col = head(letters, nrow(y))\n\n# Inner join: similar to intersect(): keep all columns of x and y with common matches\nijoin(x, y)\n\n# Left join: use all ids from x, keep all columns of x and y\nljoin(x, y)\n\n# Right join: use all ids from y, keep all columns of x and y\nrjoin(x, y)\n\n# Outer join: similar to union(): keep all columns of x and y with matches in x or y\nojoin(x, y)\n\n# Semi join: filter x with matches in y\nsjoin(x, y)\n\n# Anti join: filter x with matches not in y\najoin(x, y)\n\n# Updating join: Replace values in x with values in y\nujoin(x, y)\n}\n" }, { "path": "man/Tags.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Tags.R\n\\name{Tags}\n\\alias{Tags}\n\\alias{addJobTags}\n\\alias{removeJobTags}\n\\alias{getUsedJobTags}\n\\title{Add or Remove Job Tags}\n\\usage{\naddJobTags(ids = NULL, tags, reg = getDefaultRegistry())\n\nremoveJobTags(ids = NULL, tags, reg = getDefaultRegistry())\n\ngetUsedJobTags(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{tags}{[\\code{character}]\\cr\nTags to add or remove as strings. Each tag may consist of letters, numbers, underscore and dots (pattern \\dQuote{^[[:alnum:]_.]+}).}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with job ids affected (invisible).\n}\n\\description{\nAdd and remove arbitrary tags to jobs.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nids = batchMap(sqrt, x = -3:3, reg = tmp)\n\n# Add new tag to all ids\naddJobTags(ids, \"needs.computation\", reg = tmp)\ngetJobTags(reg = tmp)\n\n# Add more tags\naddJobTags(findJobs(x < 0, reg = tmp), \"x.neg\", reg = tmp)\naddJobTags(findJobs(x > 0, reg = tmp), \"x.pos\", reg = tmp)\ngetJobTags(reg = tmp)\n\n# Submit first 5 jobs and remove tag if successful\nids = submitJobs(1:5, reg = tmp)\nif (waitForJobs(reg = tmp))\n removeJobTags(ids, \"needs.computation\", reg = tmp)\ngetJobTags(reg = tmp)\n\n# Grep for warning message and add a tag\naddJobTags(grepLogs(pattern = \"NaNs produced\", reg = tmp), \"div.zero\", reg = tmp)\ngetJobTags(reg = tmp)\n\n# All tags where tag x.neg is set:\nids = findTagged(\"x.neg\", reg = tmp)\ngetUsedJobTags(ids, reg = tmp)\n}\n" }, { "path": "man/Worker.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Worker.R\n\\docType{class}\n\\name{Worker}\n\\alias{Worker}\n\\title{Create a Linux-Worker}\n\\format{\nAn \\code{\\link[R6]{R6Class}} generator object\n}\n\\value{\n[\\code{\\link{Worker}}].\n}\n\\description{\n\\code{\\link[R6]{R6Class}} to create local and remote linux workers.\n}\n\\section{Fields}{\n\n\\describe{\n\\item{\\code{nodename}}{Host name. Set via constructor.}\n\n\\item{\\code{ncpus}}{Number of CPUs. Set via constructor and defaults to a heuristic which tries to detect the number of CPUs of the machine.}\n\n\\item{\\code{max.load}}{Maximum load average (of the last 5 min). Set via constructor and defaults to the number of CPUs of the machine.}\n\n\\item{\\code{status}}{Status of the worker; one of \\dQuote{unknown}, \\dQuote{available}, \\dQuote{max.cpus} and \\dQuote{max.load}.}\n}}\n\n\\section{Methods}{\n\n\\describe{\n \\item{\\code{new(nodename, ncpus, max.load)}}{Constructor.}\n \\item{\\code{update(reg)}}{Update the worker status.}\n \\item{\\code{list(reg)}}{List running jobs.}\n \\item{\\code{start(reg, fn, outfile)}}{Start job collection in file \\dQuote{fn} and output to \\dQuote{outfile}.}\n \\item{\\code{kill(reg, batch.id)}}{Kill job matching the \\dQuote{batch.id}.}\n}\n}\n\n\\examples{\n\\dontrun{\n# create a worker for the local machine and use 4 CPUs.\nWorker$new(\"localhost\", ncpus = 4)\n}\n}\n" }, { "path": "man/addAlgorithm.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Algorithm.R\n\\name{addAlgorithm}\n\\alias{addAlgorithm}\n\\alias{Algorithm}\n\\alias{removeAlgorithms}\n\\title{Define Algorithms for Experiments}\n\\usage{\naddAlgorithm(name, fun = NULL, reg = getDefaultRegistry())\n\nremoveAlgorithms(name, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{name}{[\\code{character(1)}]\\cr\nUnique identifier for the algorithm.}\n\n\\item{fun}{[\\code{function}]\\cr\n The algorithm function. The static problem part is passed as \\dQuote{data}, the generated\n problem instance is passed as \\dQuote{instance} and the \\code{\\link{Job}}/\\code{\\link{Experiment}} as \\dQuote{job}.\n Therefore, your function must have the formal arguments \\dQuote{job}, \\dQuote{data} and \\dQuote{instance} (or dots \\code{...}).\n\n If you do not provide a function, it defaults to a function which just returns the instance.}\n\n\\item{reg}{[\\code{\\link{ExperimentRegistry}}]\\cr\nRegistry. If not explicitly passed, uses the last created registry.}\n}\n\\value{\n[\\code{Algorithm}]. Object of class \\dQuote{Algorithm}.\n}\n\\description{\nAlgorithms are functions which get the \\code{data} part as well as the problem instance (the return value of the\nfunction defined in \\code{\\link{Problem}}) and return an arbitrary R object.\n\nThis function serializes all components to the file system and registers the algorithm in the \\code{\\link{ExperimentRegistry}}.\n\n\\code{removeAlgorithm} removes all jobs from the registry which depend on the specific algorithm.\n\\code{reg$algorithms} holds the IDs of already defined algorithms.\n}\n\\seealso{\n\\code{\\link{Problem}}, \\code{\\link{addExperiments}}\n}\n" }, { "path": "man/addExperiments.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/addExperiments.R\n\\name{addExperiments}\n\\alias{addExperiments}\n\\title{Add Experiments to the Registry}\n\\usage{\naddExperiments(\n prob.designs = NULL,\n algo.designs = NULL,\n repls = 1L,\n combine = \"crossprod\",\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{prob.designs}{[named list of \\code{\\link[base]{data.frame}}]\\cr\nNamed list of data frames (or \\code{\\link[data.table]{data.table}}).\nThe name must match the problem name while the column names correspond to parameters of the problem.\nIf \\code{NULL}, experiments for all defined problems without any parameters are added.}\n\n\\item{algo.designs}{[named list of \\code{\\link[data.table]{data.table}} or \\code{\\link[base]{data.frame}}]\\cr\nNamed list of data frames (or \\code{\\link[data.table]{data.table}}).\nThe name must match the algorithm name while the column names correspond to parameters of the algorithm.\nIf \\code{NULL}, experiments for all defined algorithms without any parameters are added.}\n\n\\item{repls}{[\\code{integer()}]\\cr\nNumber of replications for each problem design in `prob.designs` (automatically replicated to\nthe correct length).}\n\n\\item{combine}{[\\code{character(1)}]\\cr\nHow to combine the rows of a single problem design with the rows of a single algorithm design?\nDefault is \\dQuote{crossprod} which combines each row of the problem design which each row of the algorithm design\nin a cross-product fashion. Set to \\dQuote{bind} to just \\code{\\link[base]{cbind}} the tables of\nproblem and algorithm designs where the shorter table is repeated if necessary.}\n\n\\item{reg}{[\\code{\\link{ExperimentRegistry}}]\\cr\nRegistry. If not explicitly passed, uses the last created registry.}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n}\n\\description{\nAdds experiments (parametrized combinations of problems with algorithms) to the registry and thereby defines batch jobs.\n\nIf multiple problem designs or algorithm designs are provided, they are combined via the Cartesian product.\nE.g., if you have two problems \\code{p1} and \\code{p2} and three algorithms \\code{a1}, \\code{a2} and \\code{a3},\n\\code{addExperiments} creates experiments for all parameters for the combinations \\code{(p1, a1)}, \\code{(p1, a2)},\n\\code{(p1, a3)}, \\code{(p2, a1)}, \\code{(p2, a2)} and \\code{(p2, a3)}.\n}\n\\note{\nR's \\code{data.frame} converts character vectors to factors by default in R versions prior to 4.0.0 which frequently resulted in problems using \\code{addExperiments}.\nTherefore, this function will warn about factor variables if the following conditions hold:\n\\enumerate{\n \\item R version is < 4.0.0\n \\item The design is passed as a \\code{data.frame}, not a \\code{\\link[data.table]{data.table}} or \\code{\\link[tibble]{tibble}}.\n \\item The option \\dQuote{stringsAsFactors} is not set or set to \\code{TRUE}.\n}\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n\n# add first problem\nfun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\naddProblem(\"rnorm\", fun = fun, reg = tmp)\n\n# add second problem\nfun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\naddProblem(\"rexp\", fun = fun, reg = tmp)\n\n# add first algorithm\nfun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\naddAlgorithm(\"average\", fun = fun, reg = tmp)\n\n# add second algorithm\nfun = function(instance, ...) sd(instance)\naddAlgorithm(\"deviation\", fun = fun, reg = tmp)\n\n# define problem and algorithm designs\nlibrary(data.table)\nprob.designs = algo.designs = list()\nprob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\nprob.designs$rexp = data.table(n = 100, lambda = 1:5)\nalgo.designs$average = data.table(method = c(\"mean\", \"median\"))\nalgo.designs$deviation = data.table()\n\n# add experiments and submit\naddExperiments(prob.designs, algo.designs, reg = tmp)\n\n# check what has been created\nsummarizeExperiments(reg = tmp)\nunwrap(getJobPars(reg = tmp))\n}\n\\seealso{\nOther Experiment: \n\\code{\\link{removeExperiments}()},\n\\code{\\link{summarizeExperiments}()}\n}\n\\concept{Experiment}\n" }, { "path": "man/addProblem.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Problem.R\n\\name{addProblem}\n\\alias{addProblem}\n\\alias{Problem}\n\\alias{removeProblems}\n\\title{Define Problems for Experiments}\n\\usage{\naddProblem(\n name,\n data = NULL,\n fun = NULL,\n seed = NULL,\n cache = FALSE,\n reg = getDefaultRegistry()\n)\n\nremoveProblems(name, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{name}{[\\code{character(1)}]\\cr\nUnique identifier for the problem.}\n\n\\item{data}{[\\code{ANY}]\\cr\nStatic problem part. Default is \\code{NULL}.}\n\n\\item{fun}{[\\code{function}]\\cr\nThe function defining the stochastic problem part.\nThe static part is passed to this function with name \\dQuote{data} and the \\code{\\link{Job}}/\\code{\\link{Experiment}}\nis passed as \\dQuote{job}.\nTherefore, your function must have the formal arguments \\dQuote{job} and \\dQuote{data} (or dots \\code{...}).\nIf you do not provide a function, it defaults to a function which just returns the data part.}\n\n\\item{seed}{[\\code{integer(1)}]\\cr\nStart seed for this problem. This allows the \\dQuote{synchronization} of a stochastic\nproblem across algorithms, so that different algorithms are evaluated on the same stochastic instance.\nIf the problem seed is defined, the seeding mechanism works as follows:\n(1) Before the dynamic part of a problem is instantiated,\nthe seed of the problem + [replication number] - 1 is set, i.e. the first\nreplication uses the problem seed. (2) The stochastic part of the problem is\ninstantiated. (3) From now on the usual experiment seed of the registry is used,\nsee \\code{\\link{ExperimentRegistry}}.\nIf \\code{seed} is set to \\code{NULL} (default), the job seed is used to instantiate the problem and\ndifferent algorithms see different stochastic instances of the same problem.}\n\n\\item{cache}{[\\code{logical(1)}]\\cr\nIf \\code{TRUE} and \\code{seed} is set, problem instances will be cached on the file system.\nThis assumes that each problem instance is deterministic for each combination of hyperparameter setting\nand each replication number.\nThis feature is experimental.}\n\n\\item{reg}{[\\code{\\link{ExperimentRegistry}}]\\cr\nRegistry. If not explicitly passed, uses the last created registry.}\n}\n\\value{\n[\\code{Problem}]. Object of class \\dQuote{Problem} (invisibly).\n}\n\\description{\nProblems may consist of up to two parts: A static, immutable part (\\code{data} in \\code{addProblem})\nand a dynamic, stochastic part (\\code{fun} in \\code{addProblem}).\nFor example, for statistical learning problems a data frame would be the static problem part while\na resampling function would be the stochastic part which creates problem instance.\nThis instance is then typically passed to a learning algorithm like a wrapper around a statistical model\n(\\code{fun} in \\code{\\link{addAlgorithm}}).\n\nThis function serialize all components to the file system and registers the problem in the \\code{\\link{ExperimentRegistry}}.\n\n\\code{removeProblem} removes all jobs from the registry which depend on the specific problem.\n\\code{reg$problems} holds the IDs of already defined problems.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\naddProblem(\"p1\", fun = function(job, data) data, reg = tmp)\naddProblem(\"p2\", fun = function(job, data) job, reg = tmp)\naddAlgorithm(\"a1\", fun = function(job, data, instance) instance, reg = tmp)\naddExperiments(repls = 2, reg = tmp)\n\n# List problems, algorithms and job parameters:\ntmp$problems\ntmp$algorithms\ngetJobPars(reg = tmp)\n\n# Remove one problem\nremoveProblems(\"p1\", reg = tmp)\n\n# List problems and algorithms:\ntmp$problems\ntmp$algorithms\ngetJobPars(reg = tmp)\n}\n\\seealso{\n\\code{\\link{Algorithm}}, \\code{\\link{addExperiments}}\n}\n" }, { "path": "man/assertRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Registry.R\n\\name{assertRegistry}\n\\alias{assertRegistry}\n\\title{assertRegistry}\n\\usage{\nassertRegistry(\n reg,\n class = NULL,\n writeable = FALSE,\n sync = FALSE,\n running.ok = TRUE\n)\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nThe object asserted to be a \\code{Registry}.}\n\n\\item{class}{[\\code{character(1)}]\\cr\nIf \\code{NULL} (default), \\code{reg} must only inherit from class \\dQuote{Registry}.\nOtherwise check that \\code{reg} is of class \\code{class}.\nE.g., if set to \\dQuote{Registry}, a \\code{\\link{ExperimentRegistry}} would not pass.}\n\n\\item{writeable}{[\\code{logical(1)}]\\cr\nCheck if the registry is writeable.}\n\n\\item{sync}{[\\code{logical(1)}]\\cr\nTry to synchronize the registry by including pending results from the file system.\nSee \\code{\\link{syncRegistry}}.}\n\n\\item{running.ok}{[\\code{logical(1)}]\\cr\nIf \\code{FALSE} throw an error if jobs associated with the registry are currently running.}\n}\n\\value{\n\\code{TRUE} invisibly.\n}\n\\description{\nAssert that a given object is a \\code{batchtools} registry.\nAdditionally can sync the registry, check if it is writeable, or check if jobs are running.\nIf any check fails, throws an error indicting the reason for the failure.\n}\n" }, { "path": "man/batchExport.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Export.R\n\\name{batchExport}\n\\alias{batchExport}\n\\title{Export Objects to the Slaves}\n\\usage{\nbatchExport(\n export = list(),\n unexport = character(0L),\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{export}{[\\code{list}]\\cr\nNamed list of objects to export.}\n\n\\item{unexport}{[\\code{character}]\\cr\nVector of object names to unexport.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{data.table}] with name and uri to the exported objects.\n}\n\\description{\nObjects are saved in subdirectory \\dQuote{exports} of the\n\\dQuote{file.dir} of \\code{reg}.\nThey are automatically loaded and placed in the global environment\neach time the registry is loaded or a job collection is executed.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n\n# list exports\nexports = batchExport(reg = tmp)\nprint(exports)\n\n# add a job and required exports\nbatchMap(function(x) x^2 + y + z, x = 1:3, reg = tmp)\nexports = batchExport(export = list(y = 99, z = 1), reg = tmp)\nprint(exports)\n\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\nstopifnot(loadResult(1, reg = tmp) == 101)\n\n# Un-export z\nexports = batchExport(unexport = \"z\", reg = tmp)\nprint(exports)\n}\n" }, { "path": "man/batchMap.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchMap.R\n\\name{batchMap}\n\\alias{batchMap}\n\\title{Map Operation for Batch Systems}\n\\usage{\nbatchMap(\n fun,\n ...,\n args = list(),\n more.args = list(),\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{fun}{[\\code{function}]\\cr\nFunction to map over arguments provided via \\code{...}.\nParameters given via \\code{args} or \\code{...} are passed as-is, in the respective order and possibly named.\nIf the function has the named formal argument \\dQuote{.job}, the \\code{\\link{Job}} is passed to the function\non the slave.}\n\n\\item{...}{[ANY]\\cr\nArguments to vectorize over (list or vector).\nShorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\nMutually exclusive with \\code{args}.\nNote that although it is possible to iterate over large objects (e.g., lists of data frames or matrices), this usually\nhurts the overall performance and thus is discouraged.}\n\n\\item{args}{[\\code{list} | \\code{data.frame}]\\cr\nArguments to vectorize over as (named) list or data frame.\nShorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).\nMutually exclusive with \\code{...}.}\n\n\\item{more.args}{[\\code{list}]\\cr\nA list of further arguments passed to \\code{fun}.\nDefault is an empty list.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n}\n\\description{\nA parallel and asynchronous \\code{\\link[base]{Map}}/\\code{\\link[base]{mapply}} for batch systems.\nNote that this function only defines the computational jobs.\nThe actual computation is started with \\code{\\link{submitJobs}}.\nResults and partial results can be collected with \\code{\\link{reduceResultsList}}, \\code{\\link{reduceResults}} or\n\\code{\\link{loadResult}}.\n\nFor a synchronous \\code{\\link[base]{Map}}-like execution, see \\code{\\link{btmapply}}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(3) }\n# example using \"...\" and more.args\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nf = function(x, y) x^2 + y\nids = batchMap(f, x = 1:10, more.args = list(y = 100), reg = tmp)\ngetJobPars(reg = tmp)\ntestJob(6, reg = tmp) # 100 + 6^2 = 136\n\n# vector recycling\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nf = function(...) list(...)\nids = batchMap(f, x = 1:3, y = 1:6, reg = tmp)\ngetJobPars(reg = tmp)\n\n# example for an expand.grid()-like operation on parameters\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nids = batchMap(paste, args = data.table::CJ(x = letters[1:3], y = 1:3), reg = tmp)\ngetJobPars(reg = tmp)\ntestJob(6, reg = tmp)\n}\n\\seealso{\n\\code{\\link{batchReduce}}\n}\n" }, { "path": "man/batchMapResults.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchMapResults.R\n\\name{batchMapResults}\n\\alias{batchMapResults}\n\\title{Map Over Results to Create New Jobs}\n\\usage{\nbatchMapResults(\n fun,\n ids = NULL,\n ...,\n more.args = list(),\n target,\n source = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{fun}{[\\code{function}]\\cr\nFunction which takes the result as first (unnamed) argument.}\n\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findDone}}.\nInvalid ids are ignored.}\n\n\\item{...}{[ANY]\\cr\nArguments to vectorize over (list or vector). Passed to \\code{\\link{batchMap}}.}\n\n\\item{more.args}{[\\code{list}]\\cr\nA list of further arguments passed to \\code{fun}.\nDefault is an empty list.}\n\n\\item{target}{[\\code{\\link{Registry}}]\\cr\nEmpty Registry where new jobs are created for.}\n\n\\item{source}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with ids of jobs added to \\code{target}.\n}\n\\description{\nThis function allows you to create new computational jobs (just like \\code{\\link{batchMap}} based on the results of\na \\code{\\link{Registry}}.\n}\n\\note{\nThe URI to the result files in registry \\code{source} is hard coded as parameter in the \\code{target} registry.\nThis means that \\code{target} is currently not portable between systems for computation.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(2) }\n# Source registry: calculate square of some numbers\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(function(x) list(square = x^2), x = 1:10, reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\n\n# Target registry: calculate the square root on results of first registry\ntarget = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMapResults(fun = function(x, y) list(sqrt = sqrt(x$square)), ids = 4:8,\n target = target, source = tmp)\nsubmitJobs(reg = target)\nwaitForJobs(reg = target)\n\n# Map old to new ids. First, get a table with results and parameters\nresults = unwrap(rjoin(getJobPars(reg = target), reduceResultsDataTable(reg = target)))\nprint(results)\n\n# Parameter '.id' points to job.id in 'source'. Use a inner join to combine:\nijoin(results, unwrap(reduceResultsDataTable(reg = tmp)), by = c(\".id\" = \"job.id\"))\n}\n\\seealso{\nOther Results: \n\\code{\\link{loadResult}()},\n\\code{\\link{reduceResults}()},\n\\code{\\link{reduceResultsList}()}\n}\n\\concept{Results}\n" }, { "path": "man/batchReduce.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchReduce.R\n\\name{batchReduce}\n\\alias{batchReduce}\n\\title{Reduce Operation for Batch Systems}\n\\usage{\nbatchReduce(\n fun,\n xs,\n init = NULL,\n chunks = seq_along(xs),\n more.args = list(),\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{fun}{[\\code{function(aggr, x, ...)}]\\cr\nFunction to reduce \\code{xs} with.}\n\n\\item{xs}{[\\code{vector}]\\cr\nVector to reduce.}\n\n\\item{init}{[ANY]\\cr\nInitial object for reducing. See \\code{\\link[base]{Reduce}}.}\n\n\\item{chunks}{[\\code{integer(length(xs))}]\\cr\nGroup for each element of \\code{xs}. Can be generated with \\code{\\link{chunk}}.}\n\n\\item{more.args}{[\\code{list}]\\cr\nA list of additional arguments passed to \\code{fun}.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with ids of added jobs stored in column \\dQuote{job.id}.\n}\n\\description{\nA parallel and asynchronous \\code{\\link[base]{Reduce}} for batch systems.\nNote that this function only defines the computational jobs.\nEach job reduces a certain number of elements on one slave.\nThe actual computation is started with \\code{\\link{submitJobs}}.\nResults and partial results can be collected with \\code{\\link{reduceResultsList}}, \\code{\\link{reduceResults}} or\n\\code{\\link{loadResult}}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\n# define function to reduce on slave, we want to sum a vector\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nxs = 1:100\nf = function(aggr, x) aggr + x\n\n# sum 20 numbers on each slave process, i.e. 5 jobs\nchunks = chunk(xs, chunk.size = 5)\nbatchReduce(fun = f, 1:100, init = 0, chunks = chunks, reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\n\n# now reduce one final time on master\nreduceResults(fun = function(aggr, job, res) f(aggr, res), reg = tmp)\n}\n\\seealso{\n\\code{\\link{batchMap}}\n}\n" }, { "path": "man/batchtools-package.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/zzz.R\n\\docType{package}\n\\name{batchtools-package}\n\\alias{batchtools}\n\\alias{batchtools-package}\n\\title{batchtools: Tools for Computation on Batch Systems}\n\\description{\nFor bug reports and feature requests please use the tracker:\n\\url{https://github.com/mlr-org/batchtools}.\n}\n\\section{Package options}{\n\n\\describe{\n \\item{\\code{batchtools.verbose}}{\n Verbosity. Set to \\code{FALSE} to suppress info messages and progress bars.\n }\n \\item{\\code{batchtools.progress}}{\n Progress bars. Set to \\code{FALSE} to disable them.\n }\n \\item{\\code{batchtools.timestamps}}{\n Add time stamps to log output. Set to \\code{FALSE} to disable them.\n }\n}\nFurthermore, you may enable a debug mode using the \\pkg{debugme} package by\nsetting the environment variable \\dQuote{DEBUGME} to \\dQuote{batchtools} before\nloading \\pkg{batchtools}.\n}\n\n\\seealso{\nUseful links:\n\\itemize{\n \\item \\url{https://github.com/mlr-org/batchtools}\n \\item \\url{https://batchtools.mlr-org.com}\n \\item Report bugs at \\url{https://github.com/mlr-org/batchtools/issues}\n}\n\n}\n\\author{\n\\strong{Maintainer}: Michel Lang \\email{michellang@gmail.com} (\\href{https://orcid.org/0000-0001-9754-0393}{ORCID})\n\nAuthors:\n\\itemize{\n \\item Bernd Bischl \\email{bernd_bischl@gmx.net}\n}\n\nOther contributors:\n\\itemize{\n \\item Dirk Surmann \\email{surmann@statistik.tu-dortmund.de} (\\href{https://orcid.org/0000-0003-0873-137X}{ORCID}) [contributor]\n}\n\n}\n" }, { "path": "man/btlapply.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/btlapply.R\n\\name{btlapply}\n\\alias{btlapply}\n\\alias{btmapply}\n\\title{Synchronous Apply Functions}\n\\usage{\nbtlapply(\n X,\n fun,\n ...,\n resources = list(),\n n.chunks = NULL,\n chunk.size = NULL,\n reg = makeRegistry(file.dir = NA)\n)\n\nbtmapply(\n fun,\n ...,\n more.args = list(),\n simplify = FALSE,\n use.names = TRUE,\n resources = list(),\n n.chunks = NULL,\n chunk.size = NULL,\n reg = makeRegistry(file.dir = NA)\n)\n}\n\\arguments{\n\\item{X}{[\\code{\\link[base]{vector}}]\\cr\nVector to apply over.}\n\n\\item{fun}{[\\code{function}]\\cr\nFunction to apply.}\n\n\\item{...}{[\\code{ANY}]\\cr\nAdditional arguments passed to \\code{fun} (\\code{btlapply}) or vectors to map over (\\code{btmapply}).}\n\n\\item{resources}{[\\code{named list}]\\cr\nComputational resources for the jobs to submit. The actual elements of this list\n(e.g. something like \\dQuote{walltime} or \\dQuote{nodes}) depend on your template file, exceptions are outlined in the section 'Resources'.\nDefault settings for a system can be set in the configuration file by defining the named list \\code{default.resources}.\nNote that these settings are merged by name, e.g. merging \\code{list(walltime = 300)} into \\code{list(walltime = 400, memory = 512)}\nwill result in \\code{list(walltime = 300, memory = 512)}.\nSame holds for individual job resources passed as additional column of \\code{ids} (c.f. section 'Resources').}\n\n\\item{n.chunks}{[\\code{integer(1)}]\\cr\nPassed to \\code{\\link{chunk}} before \\code{\\link{submitJobs}}.}\n\n\\item{chunk.size}{[\\code{integer(1)}]\\cr\nPassed to \\code{\\link{chunk}} before \\code{\\link{submitJobs}}.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n\n\\item{more.args}{[\\code{list}]\\cr\nAdditional arguments passed to \\code{fun}.}\n\n\\item{simplify}{[\\code{logical(1)}]\\cr\nSimplify the results using \\code{\\link[base]{simplify2array}}?}\n\n\\item{use.names}{[\\code{logical(1)}]\\cr\nUse names of the input to name the output?}\n}\n\\value{\n[\\code{list}] List with the results of the function call.\n}\n\\description{\nThis is a set of functions acting as counterparts to the sequential popular apply functions in base R:\n\\code{btlapply} for \\code{\\link[base]{lapply}} and \\code{btmapply} for \\code{\\link[base]{mapply}}.\n\nInternally, jobs are created using \\code{\\link{batchMap}} on the provided registry.\nIf no registry is provided, a temporary registry (see argument \\code{file.dir} of \\code{\\link{makeRegistry}}) and \\code{\\link{batchMap}}\nwill be used.\nAfter all jobs are terminated (see \\code{\\link{waitForJobs}}), the results are collected and returned as a list.\n\nNote that these functions are only suitable for short and fail-safe operations\non batch system. If some jobs fail, you have to retrieve partial results from the\nregistry directory yourself.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\nbtlapply(1:3, function(x) x^2)\nbtmapply(function(x, y, z) x + y + z, x = 1:3, y = 1:3, more.args = list(z = 1), simplify = TRUE)\n}\n" }, { "path": "man/cfBrewTemplate.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfBrewTemplate}\n\\alias{cfBrewTemplate}\n\\title{Cluster Functions Helper to Write Job Description Files}\n\\usage{\ncfBrewTemplate(reg, text, jc)\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n\n\\item{text}{[\\code{character(1)}]\\cr\nString ready to be brewed. See \\code{\\link{cfReadBrewTemplate}} to read a template from the file system.}\n\n\\item{jc}{[\\code{\\link{JobCollection})}]\\cr\nWill be used as environment to brew the template file in. See \\code{\\link{JobCollection}} for a list of all\navailable variables.}\n}\n\\value{\n[\\code{character(1)}]. File path to brewed template file.\n}\n\\description{\nThis function is only intended for use in your own cluster functions implementation.\n\nCalls brew silently on your template, any error will lead to an exception.\nThe file is stored at the same place as the corresponding job file in the \\dQuote{jobs}-subdir\nof your files directory.\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeSubmitJobResult}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/cfHandleUnknownSubmitError.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfHandleUnknownSubmitError}\n\\alias{cfHandleUnknownSubmitError}\n\\title{Cluster Functions Helper to Handle Unknown Errors}\n\\usage{\ncfHandleUnknownSubmitError(cmd, exit.code, output)\n}\n\\arguments{\n\\item{cmd}{[\\code{character(1)}]\\cr\nOS command used to submit the job, e.g. qsub.}\n\n\\item{exit.code}{[\\code{integer(1)}]\\cr\nExit code of the OS command, should not be 0.}\n\n\\item{output}{[\\code{character}]\\cr\nOutput of the OS command, hopefully an informative error message.\nIf these are multiple lines in a vector, they are automatically joined.}\n}\n\\value{\n[\\code{\\link{SubmitJobResult}}].\n}\n\\description{\nThis function is only intended for use in your own cluster functions implementation.\n\nSimply constructs a \\code{\\link{SubmitJobResult}} object with status code 101, NA as batch id and\nan informative error message containing the output of the OS command in \\code{output}.\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeSubmitJobResult}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/cfKillJob.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfKillJob}\n\\alias{cfKillJob}\n\\title{Cluster Functions Helper to Kill Batch Jobs}\n\\usage{\ncfKillJob(\n reg,\n cmd,\n args = character(0L),\n max.tries = 3L,\n nodename = \"localhost\"\n)\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n\n\\item{cmd}{[\\code{character(1)}]\\cr\nOS command, e.g. \\dQuote{qdel}.}\n\n\\item{args}{[\\code{character}]\\cr\nArguments to \\code{cmd}, including the batch id.}\n\n\\item{max.tries}{[\\code{integer(1)}]\\cr\nNumber of total times to try execute the OS command in cases of failures.\nDefault is \\code{3}.}\n\n\\item{nodename}{[\\code{character(1)}]\\cr\nName of the SSH node to run the command on. If set to \\dQuote{localhost} (default), the command\nis not piped through SSH.}\n}\n\\value{\n\\code{TRUE} on success. An exception is raised otherwise.\n}\n\\description{\nThis function is only intended for use in your own cluster functions implementation.\n\nCalls the OS command to kill a job via \\code{\\link[base]{system}} like this: \\dQuote{cmd batch.job.id}. If the\ncommand returns an exit code > 0, the command is repeated after a 1 second sleep\n\\code{max.tries-1} times. If the command failed in all tries, an error is generated.\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeSubmitJobResult}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/cfReadBrewTemplate.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfReadBrewTemplate}\n\\alias{cfReadBrewTemplate}\n\\title{Cluster Functions Helper to Parse a Brew Template}\n\\usage{\ncfReadBrewTemplate(template, comment.string = NA_character_)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nPath to template file which is then passed to \\code{\\link[brew]{brew}}.}\n\n\\item{comment.string}{[\\code{character(1)}]\\cr\nIgnore lines starting with this string.}\n}\n\\value{\n[\\code{character}].\n}\n\\description{\nThis function is only intended for use in your own cluster functions implementation.\n\nThis function is only intended for use in your own cluster functions implementation.\nSimply reads your template file and returns it as a character vector.\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeSubmitJobResult}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/chunk.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/chunkIds.R\n\\name{chunk}\n\\alias{chunk}\n\\alias{lpt}\n\\alias{binpack}\n\\title{Chunk Jobs for Sequential Execution}\n\\usage{\nchunk(x, n.chunks = NULL, chunk.size = NULL, shuffle = TRUE)\n\nlpt(x, n.chunks = 1L)\n\nbinpack(x, chunk.size = max(x))\n}\n\\arguments{\n\\item{x}{[\\code{numeric}]\\cr\nFor \\code{chunk} an atomic vector (usually the \\code{job.id}).\nFor \\code{binpack} and \\code{lpt}, the weights to group.}\n\n\\item{n.chunks}{[\\code{integer(1)}]\\cr\nRequested number of chunks.\nThe function \\code{chunk} distributes the number of elements in \\code{x} evenly while\n\\code{lpt} tries to even out the sum of elements in each chunk.\nIf more chunks than necessary are requested, empty chunks are ignored.\nMutually exclusive with \\code{chunks.size}.}\n\n\\item{chunk.size}{[\\code{integer(1)}]\\cr\nRequested chunk size for each single chunk.\nFor \\code{chunk} this is the number of elements in \\code{x}, for \\code{binpack} the size\nis determined by the sum of values in \\code{x}.\nMutually exclusive with \\code{n.chunks}.}\n\n\\item{shuffle}{[\\code{logical(1)}]\\cr\nShuffles the groups. Default is \\code{TRUE}.}\n}\n\\value{\n[\\code{integer}] giving the chunk number for each element of \\code{x}.\n}\n\\description{\nJobs can be partitioned into \\dQuote{chunks} to be executed sequentially on the computational nodes.\nChunks are defined by providing a data frame with columns \\dQuote{job.id} and \\dQuote{chunk} (integer)\nto \\code{\\link{submitJobs}}.\nAll jobs with the same chunk number will be grouped together on one node to form a single\ncomputational job.\n\nThe function \\code{chunk} simply splits \\code{x} into either a fixed number of groups, or\ninto a variable number of groups with a fixed number of maximum elements.\n\nThe function \\code{lpt} also groups \\code{x} into a fixed number of chunks,\nbut uses the actual values of \\code{x} in a greedy \\dQuote{Longest Processing Time} algorithm.\nAs a result, the maximum sum of elements in minimized.\n\n\\code{binpack} splits \\code{x} into a variable number of groups whose sum of elements do\nnot exceed the upper limit provided by \\code{chunk.size}.\n\nSee examples of \\code{\\link{estimateRuntimes}} for an application of \\code{binpack} and \\code{lpt}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(2) }\nch = chunk(1:10, n.chunks = 2)\ntable(ch)\n\nch = chunk(rep(1, 10), chunk.size = 2)\ntable(ch)\n\nset.seed(1)\nx = runif(10)\nch = lpt(x, n.chunks = 2)\nsapply(split(x, ch), sum)\n\nset.seed(1)\nx = runif(10)\nch = binpack(x, 1)\nsapply(split(x, ch), sum)\n\n# Job chunking\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nids = batchMap(identity, 1:25, reg = tmp)\n\n### Group into chunks with 10 jobs each\nlibrary(data.table)\nids[, chunk := chunk(job.id, chunk.size = 10)]\nprint(ids[, .N, by = chunk])\n\n### Group into 4 chunks\nids[, chunk := chunk(job.id, n.chunks = 4)]\nprint(ids[, .N, by = chunk])\n\n### Submit to batch system\nsubmitJobs(ids = ids, reg = tmp)\n\n# Grouped chunking\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\nprob = addProblem(reg = tmp, \"prob1\", data = iris, fun = function(job, data) nrow(data))\nprob = addProblem(reg = tmp, \"prob2\", data = Titanic, fun = function(job, data) nrow(data))\nalgo = addAlgorithm(reg = tmp, \"algo\", fun = function(job, data, instance, i, ...) problem)\nprob.designs = list(prob1 = data.table(), prob2 = data.table(x = 1:2))\nalgo.designs = list(algo = data.table(i = 1:3))\naddExperiments(prob.designs, algo.designs, repls = 3, reg = tmp)\n\n### Group into chunks of 5 jobs, but do not put multiple problems into the same chunk\n# -> only one problem has to be loaded per chunk, and only once because it is cached\nids = getJobTable(reg = tmp)[, .(job.id, problem, algorithm)]\nids[, chunk := chunk(job.id, chunk.size = 5), by = \"problem\"]\nids[, chunk := .GRP, by = c(\"problem\", \"chunk\")]\ndcast(ids, chunk ~ problem)\n}\n\\seealso{\n\\code{\\link{estimateRuntimes}}\n}\n" }, { "path": "man/clearRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clearRegistry.R\n\\name{clearRegistry}\n\\alias{clearRegistry}\n\\title{Remove All Jobs}\n\\usage{\nclearRegistry(reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\description{\nRemoves all jobs from a registry and calls \\code{\\link{sweepRegistry}}.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/doJobCollection.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/doJobCollection.R\n\\name{doJobCollection}\n\\alias{doJobCollection}\n\\title{Execute Jobs of a JobCollection}\n\\usage{\ndoJobCollection(jc, output = NULL)\n}\n\\arguments{\n\\item{jc}{[\\code{\\link{JobCollection}}]\\cr\nEither an object of class \\dQuote{JobCollection} as returned by\n\\code{\\link{makeJobCollection}} or a string with the path to file\ncontaining a \\dQuote{JobCollection} as RDS file (as stored by \\code{\\link{submitJobs}}).}\n\n\\item{output}{[\\code{character(1)}]\\cr\nPath to a file to write the output to. Defaults to \\code{NULL} which means\nthat output is written to the active \\code{\\link[base]{sink}}.\nDo not set this if your scheduler redirects output to a log file.}\n}\n\\value{\n[\\code{character(1)}]: Hash of the \\code{\\link{JobCollection}} executed.\n}\n\\description{\nExecutes every job in a \\code{\\link{JobCollection}}.\nThis function is intended to be called on the slave.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(identity, 1:2, reg = tmp)\njc = makeJobCollection(1:2, reg = tmp)\ndoJobCollection(jc)\n}\n\\seealso{\nOther JobCollection: \n\\code{\\link{makeJobCollection}()}\n}\n\\concept{JobCollection}\n" }, { "path": "man/estimateRuntimes.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/estimateRuntimes.R\n\\name{estimateRuntimes}\n\\alias{estimateRuntimes}\n\\alias{print.RuntimeEstimate}\n\\title{Estimate Remaining Runtimes}\n\\usage{\nestimateRuntimes(tab, ..., reg = getDefaultRegistry())\n\n\\method{print}{RuntimeEstimate}(x, n = 1L, ...)\n}\n\\arguments{\n\\item{tab}{[\\code{\\link[data.table]{data.table}}]\\cr\nTable with column \\dQuote{job.id} and additional columns to predict the runtime.\nObserved runtimes will be looked up in the registry and serve as dependent variable.\nAll columns in \\code{tab} except \\dQuote{job.id} will be passed to \\code{\\link[ranger]{ranger}} as\nindependent variables to fit the model.}\n\n\\item{...}{[ANY]\\cr\nAdditional parameters passed to \\code{\\link[ranger]{ranger}}. Ignored for the \\code{print} method.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n\n\\item{x}{[\\code{RuntimeEstimate}]\\cr\nObject to print.}\n\n\\item{n}{[\\code{integer(1)}]\\cr\nNumber of parallel jobs to assume for runtime estimation.}\n}\n\\value{\n[\\code{RuntimeEstimate}] which is a \\code{list} with two named elements:\n \\dQuote{runtimes} is a \\code{\\link[data.table]{data.table}} with columns \\dQuote{job.id},\n \\dQuote{runtime} (in seconds) and \\dQuote{type} (\\dQuote{estimated} if runtime is estimated,\n \\dQuote{observed} if runtime was observed).\n The other element of the list named \\dQuote{model}] contains the fitted random forest object.\n}\n\\description{\nEstimates the runtimes of jobs using the random forest implemented in \\pkg{ranger}.\nObserved runtimes are retrieved from the \\code{\\link{Registry}} and runtimes are\npredicted for unfinished jobs.\n\nThe estimated remaining time is calculated in the \\code{print} method.\nYou may also pass \\code{n} here to determine the number of parallel jobs which is then used\nin a simple Longest Processing Time (LPT) algorithm to give an estimate for the parallel runtime.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\n# Create a simple toy registry\nset.seed(1)\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE, seed = 1)\naddProblem(name = \"iris\", data = iris, fun = function(data, ...) nrow(data), reg = tmp)\naddAlgorithm(name = \"nrow\", function(instance, ...) nrow(instance), reg = tmp)\naddAlgorithm(name = \"ncol\", function(instance, ...) ncol(instance), reg = tmp)\naddExperiments(algo.designs = list(nrow = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\naddExperiments(algo.designs = list(ncol = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)\n\n# We use the job parameters to predict runtimes\ntab = unwrap(getJobPars(reg = tmp))\n\n# First we need to submit some jobs so that the forest can train on some data.\n# Thus, we just sample some jobs from the registry while grouping by factor variables.\nlibrary(data.table)\nids = tab[, .SD[sample(nrow(.SD), 5)], by = c(\"problem\", \"algorithm\", \"y\")]\nsetkeyv(ids, \"job.id\")\nsubmitJobs(ids, reg = tmp)\nwaitForJobs(reg = tmp)\n\n# We \"simulate\" some more realistic runtimes here to demonstrate the functionality:\n# - Algorithm \"ncol\" is 5 times more expensive than \"nrow\"\n# - x has no effect on the runtime\n# - If y is \"a\" or \"b\", the runtimes are really high\nruntime = function(algorithm, x, y) {\n ifelse(algorithm == \"nrow\", 100L, 500L) + 1000L * (y \\%in\\% letters[1:2])\n}\ntmp$status[ids, done := done + tab[ids, runtime(algorithm, x, y)]]\nrjoin(sjoin(tab, ids), getJobStatus(ids, reg = tmp)[, c(\"job.id\", \"time.running\")])\n\n# Estimate runtimes:\nest = estimateRuntimes(tab, reg = tmp)\nprint(est)\nrjoin(tab, est$runtimes)\nprint(est, n = 10)\n\n# Submit jobs with longest runtime first:\nids = est$runtimes[type == \"estimated\"][order(runtime, decreasing = TRUE)]\nprint(ids)\n\\dontrun{\nsubmitJobs(ids, reg = tmp)\n}\n\n# Group jobs into chunks with runtime < 1h\nids = est$runtimes[type == \"estimated\"]\nids[, chunk := binpack(runtime, 3600)]\nprint(ids)\nprint(ids[, list(runtime = sum(runtime)), by = chunk])\n\\dontrun{\nsubmitJobs(ids, reg = tmp)\n}\n\n# Group jobs into 10 chunks with similar runtime\nids = est$runtimes[type == \"estimated\"]\nids[, chunk := lpt(runtime, 10)]\nprint(ids[, list(runtime = sum(runtime)), by = chunk])\n}\n\\seealso{\n\\code{\\link{binpack}} and \\code{\\link{lpt}} to chunk jobs according to their estimated runtimes.\n}\n" }, { "path": "man/execJob.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/execJob.R\n\\name{execJob}\n\\alias{execJob}\n\\title{Execute a Single Jobs}\n\\usage{\nexecJob(job)\n}\n\\arguments{\n\\item{job}{[\\code{\\link{Job}} | \\code{\\link{Experiment}}]\\cr\nJob/Experiment to execute.}\n}\n\\value{\nResult of the job.\n}\n\\description{\nExecutes a single job (as created by \\code{\\link{makeJob}}) and returns\nits result. Also works for Experiments.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(identity, 1:2, reg = tmp)\njob = makeJob(1, reg = tmp)\nexecJob(job)\n}\n" }, { "path": "man/findConfFile.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/config.R\n\\name{findConfFile}\n\\alias{findConfFile}\n\\title{Find a batchtools Configuration File}\n\\usage{\nfindConfFile()\n}\n\\value{\n[\\code{character(1)}] Path to the configuration file or \\code{NA} if no configuration file was found.\n}\n\\description{\nThis functions returns the path to the first configuration file found in the following locations:\n \\enumerate{\n \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n }\n}\n\\keyword{internal}\n" }, { "path": "man/findJobs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/findJobs.R\n\\name{findJobs}\n\\alias{findJobs}\n\\alias{findExperiments}\n\\alias{findSubmitted}\n\\alias{findNotSubmitted}\n\\alias{findStarted}\n\\alias{findNotStarted}\n\\alias{findDone}\n\\alias{findNotDone}\n\\alias{findErrors}\n\\alias{findOnSystem}\n\\alias{findRunning}\n\\alias{findQueued}\n\\alias{findExpired}\n\\alias{findTagged}\n\\title{Find and Filter Jobs}\n\\usage{\nfindJobs(expr, ids = NULL, reg = getDefaultRegistry())\n\nfindExperiments(\n ids = NULL,\n prob.name = NA_character_,\n prob.pattern = NA_character_,\n algo.name = NA_character_,\n algo.pattern = NA_character_,\n prob.pars,\n algo.pars,\n repls = NULL,\n reg = getDefaultRegistry()\n)\n\nfindSubmitted(ids = NULL, reg = getDefaultRegistry())\n\nfindNotSubmitted(ids = NULL, reg = getDefaultRegistry())\n\nfindStarted(ids = NULL, reg = getDefaultRegistry())\n\nfindNotStarted(ids = NULL, reg = getDefaultRegistry())\n\nfindDone(ids = NULL, reg = getDefaultRegistry())\n\nfindNotDone(ids = NULL, reg = getDefaultRegistry())\n\nfindErrors(ids = NULL, reg = getDefaultRegistry())\n\nfindOnSystem(ids = NULL, reg = getDefaultRegistry())\n\nfindRunning(ids = NULL, reg = getDefaultRegistry())\n\nfindQueued(ids = NULL, reg = getDefaultRegistry())\n\nfindExpired(ids = NULL, reg = getDefaultRegistry())\n\nfindTagged(tags = character(0L), ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{expr}{[\\code{expression}]\\cr\nPredicate expression evaluated in the job parameters.\nJobs for which \\code{expr} evaluates to \\code{TRUE} are returned.}\n\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n\n\\item{prob.name}{[\\code{character}]\\cr\nExact name of the problem (no substring matching).\nIf not provided, all problems are matched.}\n\n\\item{prob.pattern}{[\\code{character}]\\cr\nRegular expression pattern to match problem names.\nIf not provided, all problems are matched.}\n\n\\item{algo.name}{[\\code{character}]\\cr\nExact name of the problem (no substring matching).\nIf not provided, all algorithms are matched.}\n\n\\item{algo.pattern}{[\\code{character}]\\cr\nRegular expression pattern to match algorithm names.\nIf not provided, all algorithms are matched.}\n\n\\item{prob.pars}{[\\code{expression}]\\cr\nPredicate expression evaluated in the problem parameters.}\n\n\\item{algo.pars}{[\\code{expression}]\\cr\nPredicate expression evaluated in the algorithm parameters.}\n\n\\item{repls}{[\\code{integer}]\\cr\nWhitelist of replication numbers. If not provided, all replications are matched.}\n\n\\item{tags}{[\\code{character}]\\cr\nReturn jobs which are tagged with any of the tags provided.}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with column \\dQuote{job.id} containing matched jobs.\n}\n\\description{\nThese functions are used to find and filter jobs, depending on either their parameters (\\code{findJobs} and\n\\code{findExperiments}), their tags (\\code{findTagged}), or their computational status (all other functions,\nsee \\code{\\link{getStatus}} for an overview).\n\nNote that \\code{findQueued}, \\code{findRunning}, \\code{findOnSystem} and \\code{findExpired} are somewhat heuristic\nand may report misleading results, depending on the state of the system and the \\code{\\link{ClusterFunctions}} implementation.\n\nSee \\code{\\link{JoinTables}} for convenient set operations (unions, intersects, differences) on tables with job ids.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(identity, i = 1:3, reg = tmp)\nids = findNotSubmitted(reg = tmp)\n\n# get all jobs:\nfindJobs(reg = tmp)\n\n# filter for jobs with parameter i >= 2\nfindJobs(i >= 2, reg = tmp)\n\n# filter on the computational status\nfindSubmitted(reg = tmp)\nfindNotDone(reg = tmp)\n\n# filter on tags\naddJobTags(2:3, \"my_tag\", reg = tmp)\nfindTagged(tags = \"my_tag\", reg = tmp)\n\n# combine filter functions using joins\n# -> jobs which are not done and not tagged (using an anti-join):\najoin(findNotDone(reg = tmp), findTagged(\"my_tag\", reg = tmp))\n}\n\\seealso{\n\\code{\\link{getStatus}} \\code{\\link{JoinTables}}\n}\n" }, { "path": "man/findTemplateFile.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{findTemplateFile}\n\\alias{findTemplateFile}\n\\title{Find a batchtools Template File}\n\\usage{\nfindTemplateFile(template)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n}\n\\value{\n[\\code{character}] Path to the file or \\code{NA} if no template template file was found.\n}\n\\description{\nThis functions returns the path to a template file on the file system.\n}\n\\keyword{internal}\n" }, { "path": "man/getDefaultRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/getDefaultRegistry.R\n\\name{getDefaultRegistry}\n\\alias{getDefaultRegistry}\n\\alias{setDefaultRegistry}\n\\title{Get and Set the Default Registry}\n\\usage{\ngetDefaultRegistry()\n\nsetDefaultRegistry(reg)\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\description{\n\\code{getDefaultRegistry} returns the registry currently set as default (or\nstops with an exception if none is set). \\code{setDefaultRegistry} sets\na registry as default.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/getErrorMessages.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/getErrorMessages.R\n\\name{getErrorMessages}\n\\alias{getErrorMessages}\n\\title{Retrieve Error Messages}\n\\usage{\ngetErrorMessages(\n ids = NULL,\n missing.as.error = FALSE,\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findErrors}}.\nInvalid ids are ignored.}\n\n\\item{missing.as.error}{[\\code{logical(1)}]\\cr\nTreat missing results as errors? If \\code{TRUE}, the error message \\dQuote{[not terminated]} is imputed\nfor jobs which have not terminated. Default is \\code{FALSE}}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id}, \\dQuote{terminated} (logical),\n \\dQuote{error} (logical) and \\dQuote{message} (string).\n}\n\\description{\nExtracts error messages from the internal data base and returns them in a table.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nfun = function(i) if (i == 3) stop(i) else i\nids = batchMap(fun, i = 1:5, reg = tmp)\nsubmitJobs(1:4, reg = tmp)\nwaitForJobs(1:4, reg = tmp)\ngetErrorMessages(ids, reg = tmp)\ngetErrorMessages(ids, missing.as.error = TRUE, reg = tmp)\n}\n\\seealso{\nOther debug: \n\\code{\\link{getStatus}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{showLog}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/getJobTable.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/JobTables.R\n\\name{getJobTable}\n\\alias{getJobTable}\n\\alias{getJobStatus}\n\\alias{getJobResources}\n\\alias{getJobPars}\n\\alias{getJobTags}\n\\title{Query Job Information}\n\\usage{\ngetJobTable(ids = NULL, reg = getDefaultRegistry())\n\ngetJobStatus(ids = NULL, reg = getDefaultRegistry())\n\ngetJobResources(ids = NULL, reg = getDefaultRegistry())\n\ngetJobPars(ids = NULL, reg = getDefaultRegistry())\n\ngetJobTags(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with the following columns (not necessarily in this order):\n \\describe{\n \\item{job.id}{Unique Job ID as integer.}\n \\item{submitted}{Time the job was submitted to the batch system as \\code{\\link[base]{POSIXct}}.}\n \\item{started}{Time the job was started on the batch system as \\code{\\link[base]{POSIXct}}.}\n \\item{done}{Time the job terminated (successfully or with an error) as \\code{\\link[base]{POSIXct}}.}\n \\item{error}{Either \\code{NA} if the job terminated successfully or the error message.}\n \\item{mem.used}{Estimate of the memory usage.}\n \\item{batch.id}{Batch ID as reported by the scheduler.}\n \\item{log.file}{Log file. If missing, defaults to \\code{[job.hash].log}.}\n \\item{job.hash}{Unique string identifying the job or chunk.}\n \\item{time.queued}{Time in seconds (as \\code{\\link[base]{difftime}}) the job was queued.}\n \\item{time.running}{Time in seconds (as \\code{\\link[base]{difftime}}) the job was running.}\n \\item{pars}{List of parameters/arguments for this job.}\n \\item{resources}{List of computational resources set for this job.}\n \\item{tags}{Tags as joined string, delimited by \\dQuote{,}.}\n \\item{problem}{Only for \\code{\\link{ExperimentRegistry}}: the problem identifier.}\n \\item{algorithm}{Only for \\code{\\link{ExperimentRegistry}}: the algorithm identifier.}\n }\n}\n\\description{\n\\code{getJobStatus} returns the internal table which stores information about the computational\nstatus of jobs, \\code{getJobPars} a table with the job parameters, \\code{getJobResources} a table\nwith the resources which were set to submit the jobs, and \\code{getJobTags} the tags of the jobs\n(see \\link{Tags}).\n\n\\code{getJobTable} returns all these tables joined.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nf = function(x) if (x < 0) stop(\"x must be > 0\") else sqrt(x)\nbatchMap(f, x = c(-1, 0, 1), reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\naddJobTags(1:2, \"tag1\", reg = tmp)\naddJobTags(2, \"tag2\", reg = tmp)\n\n# Complete table:\ngetJobTable(reg = tmp)\n\n# Job parameters:\ngetJobPars(reg = tmp)\n\n# Set and retrieve tags:\ngetJobTags(reg = tmp)\n\n# Job parameters with tags right-joined:\nrjoin(getJobPars(reg = tmp), getJobTags(reg = tmp))\n}\n" }, { "path": "man/getStatus.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/getStatus.R\n\\name{getStatus}\n\\alias{getStatus}\n\\title{Summarize the Computational Status}\n\\usage{\ngetStatus(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] (with class \\dQuote{Status} for printing).\n}\n\\description{\nThis function gives an encompassing overview over the computational status on your system.\nThe status can be one or many of the following:\n\\itemize{\n \\item \\dQuote{defined}: Jobs which are defined via \\code{\\link{batchMap}} or \\code{\\link{addExperiments}}, but are not yet submitted.\n \\item \\dQuote{submitted}: Jobs which are submitted to the batch system via \\code{\\link{submitJobs}}, scheduled for execution.\n \\item \\dQuote{started}: Jobs which have been started.\n \\item \\dQuote{done}: Jobs which terminated successfully.\n \\item \\dQuote{error}: Jobs which terminated with an exception.\n \\item \\dQuote{running}: Jobs which are listed by the cluster functions to be running on the live system. Not supported for all cluster functions.\n \\item \\dQuote{queued}: Jobs which are listed by the cluster functions to be queued on the live system. Not supported for all cluster functions.\n \\item \\dQuote{system}: Jobs which are listed by the cluster functions to be queued or running. Not supported for all cluster functions.\n \\item \\dQuote{expired}: Jobs which have been submitted, but vanished from the live system. Note that this is determined heuristically and may include some false positives.\n}\nHere, a job which terminated successfully counts towards the jobs which are submitted, started and done.\nTo retrieve the corresponding job ids, see \\code{\\link{findJobs}}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nfun = function(i) if (i == 3) stop(i) else i\nids = batchMap(fun, i = 1:5, reg = tmp)\nsubmitJobs(ids = 1:4, reg = tmp)\nwaitForJobs(reg = tmp)\n\ntab = getStatus(reg = tmp)\nprint(tab)\nstr(tab)\n}\n\\seealso{\n\\code{\\link{findJobs}}\n\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{showLog}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/grepLogs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Logs.R\n\\name{grepLogs}\n\\alias{grepLogs}\n\\title{Grep Log Files for a Pattern}\n\\usage{\ngrepLogs(\n ids = NULL,\n pattern,\n ignore.case = FALSE,\n fixed = FALSE,\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findStarted}}.\nInvalid ids are ignored.}\n\n\\item{pattern}{[\\code{character(1L)}]\\cr\nRegular expression or string (see \\code{fixed}).}\n\n\\item{ignore.case}{[\\code{logical(1L)}]\\cr\nIf \\code{TRUE} the match will be performed case insensitively.}\n\n\\item{fixed}{[\\code{logical(1L)}]\\cr\nIf \\code{FALSE} (default), \\code{pattern} is a regular expression and a fixed string otherwise.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id} and \\dQuote{message}.\n}\n\\description{\nCrawls through log files and reports jobs with lines matching the \\code{pattern}.\nSee \\code{\\link{showLog}} for an example.\n}\n\\seealso{\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{getStatus}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{showLog}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/killJobs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/killJobs.R\n\\name{killJobs}\n\\alias{killJobs}\n\\title{Kill Jobs}\n\\usage{\nkillJobs(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findOnSystem}}.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id}, the corresponding \\dQuote{batch.id} and\n the logical flag \\dQuote{killed} indicating success.\n}\n\\description{\nKill jobs which are currently running on the batch system.\n\nIn case of an error when killing, the function tries - after a short sleep - to kill the remaining\nbatch jobs again. If this fails three times for some jobs, the function gives up. Jobs that could be\nsuccessfully killed are reset in the \\link{Registry}.\n}\n\\seealso{\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{getStatus}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{showLog}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/loadRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/loadRegistry.R\n\\name{loadRegistry}\n\\alias{loadRegistry}\n\\title{Load a Registry from the File System}\n\\usage{\nloadRegistry(\n file.dir,\n work.dir = NULL,\n conf.file = findConfFile(),\n make.default = TRUE,\n writeable = FALSE\n)\n}\n\\arguments{\n\\item{file.dir}{[\\code{character(1)}]\\cr\n Path where all files of the registry are saved.\n Default is directory \\dQuote{registry} in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.\n\n If you pass \\code{NA}, a temporary directory will be used.\n This way, you can create disposable registries for \\code{\\link{btlapply}} or examples.\n By default, the temporary directory \\code{\\link[base]{tempdir}()} will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable \\code{temp.dir}.}\n\n\\item{work.dir}{[\\code{character(1)}]\\cr\n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see \\code{\\link[base]{getwd}}).\n \\code{loadRegistry} uses the stored \\code{work.dir}, but you may also explicitly overwrite it,\n e.g., after switching to another system.\n\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.}\n\n\\item{conf.file}{[\\code{character(1)}]\\cr\n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how \\pkg{batchtools} interacts with the system via \\code{\\link{ClusterFunctions}}.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.\n\n The file lookup is implemented in the internal (but exported) function \\code{findConfFile} which returns the first file found of the following candidates:\n \\enumerate{\n \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n }\n Set to \\code{NA} if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. \\code{default.resources = list(walltime = 3600)} to set default resources or \\dQuote{max.concurrent.jobs} to\n limit the number of jobs allowed to run simultaneously on the system.}\n\n\\item{make.default}{[\\code{logical(1)}]\\cr\nIf set to \\code{TRUE}, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is \\code{TRUE}.}\n\n\\item{writeable}{[\\code{logical(1)}]\\cr\nLoads the registry in read-write mode. Default is \\code{FALSE}.}\n}\n\\value{\n[\\code{\\link{Registry}}].\n}\n\\description{\nLoads a registry from its \\code{file.dir}.\n\nMultiple R sessions accessing the same registry simultaneously can lead to database inconsistencies.\nThis is especially dangerous if the same \\code{file.dir} is accessed from multiple machines, e.g. via a mount.\n\nIf you just need to check on the status or peek into some preliminary results while another process is still submitting or waiting\nfor pending results, you can load the registry in a read-only mode.\nAll operations that need to change the registry will raise an exception in this mode.\nFiles communicated back by the computational nodes are parsed to update the registry in memory while the registry on the file system remains unchanged.\n\nA heuristic tries to detect if the registry has been altered in the background by an other process and in this case automatically restricts the current registry to read-only mode.\nHowever, you should rely on this heuristic to work flawlessly.\nThus, set to \\code{writeable} to \\code{TRUE} if and only if you are absolutely sure that other state-changing processes are terminated.\n\nIf you need write access, load the registry with \\code{writeable} set to \\code{TRUE}.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/loadResult.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/loadResult.R\n\\name{loadResult}\n\\alias{loadResult}\n\\title{Load the Result of a Single Job}\n\\usage{\nloadResult(id, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{id}{[\\code{integer(1)} or \\code{data.table}]\\cr\nSingle integer to specify the job or a \\code{data.table} with column \\code{job.id}\nand exactly one row.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{ANY}]. The stored result.\n}\n\\description{\nLoads the result of a single job.\n}\n\\seealso{\nOther Results: \n\\code{\\link{batchMapResults}()},\n\\code{\\link{reduceResults}()},\n\\code{\\link{reduceResultsList}()}\n}\n\\concept{Results}\n" }, { "path": "man/makeClusterFunctions.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{makeClusterFunctions}\n\\alias{makeClusterFunctions}\n\\alias{ClusterFunctions}\n\\title{ClusterFunctions Constructor}\n\\usage{\nmakeClusterFunctions(\n name,\n submitJob,\n killJob = NULL,\n listJobsQueued = NULL,\n listJobsRunning = NULL,\n array.var = NA_character_,\n store.job.collection = FALSE,\n store.job.files = FALSE,\n scheduler.latency = 0,\n fs.latency = 0,\n hooks = list()\n)\n}\n\\arguments{\n\\item{name}{[\\code{character(1)}]\\cr\nName of cluster functions.}\n\n\\item{submitJob}{[\\code{function(reg, jc, ...)}]\\cr\nFunction to submit new jobs. Must return a \\code{\\link{SubmitJobResult}} object.\nThe arguments are \\code{reg} (\\code{\\link{Registry}}) and \\code{jobs} (\\code{\\link{JobCollection}}).}\n\n\\item{killJob}{[\\code{function(reg, batch.id)}]\\cr\nFunction to kill a job on the batch system. Make sure that you definitely kill the job! Return\nvalue is currently ignored. Must have the arguments \\code{reg} (\\code{\\link{Registry}}) and\n\\code{batch.id} (\\code{character(1)} as returned by \\code{submitJob}).\nNote that there is a helper function \\code{\\link{cfKillJob}} to repeatedly try to kill jobs.\nSet \\code{killJob} to \\code{NULL} if killing jobs cannot be supported.}\n\n\\item{listJobsQueued}{[\\code{function(reg)}]\\cr\nList all queued jobs on the batch system for the current user.\nMust return an character vector of batch ids, same format as they\nare returned by \\code{submitJob}.\nSet \\code{listJobsQueued} to \\code{NULL} if listing of queued jobs is not supported.}\n\n\\item{listJobsRunning}{[\\code{function(reg)}]\\cr\nList all running jobs on the batch system for the current user.\nMust return an character vector of batch ids, same format as they\nare returned by \\code{submitJob}. It does not matter if you return a few job ids too many (e.g.\nall for the current user instead of all for the current registry), but you have to include all\nrelevant ones. Must have the argument are \\code{reg} (\\code{\\link{Registry}}).\nSet \\code{listJobsRunning} to \\code{NULL} if listing of running jobs is not supported.}\n\n\\item{array.var}{[\\code{character(1)}]\\cr\nName of the environment variable set by the scheduler to identify IDs of job arrays.\nDefault is \\code{NA} for no array support.}\n\n\\item{store.job.collection}{[\\code{logical(1)}]\\cr\nFlag to indicate that the cluster function implementation of \\code{submitJob} can not directly handle \\code{\\link{JobCollection}} objects.\nIf set to \\code{FALSE}, the \\code{\\link{JobCollection}} is serialized to the file system before submitting the job.}\n\n\\item{store.job.files}{[\\code{logical(1)}]\\cr\nFlag to indicate that job files need to be stored in the file directory.\nIf set to \\code{FALSE} (default), the job file is created in a temporary directory, otherwise (or if the debug mode is enabled) in\nthe subdirectory \\code{jobs} of the \\code{file.dir}.}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n\n\\item{hooks}{[\\code{list}]\\cr\nNamed list of functions which will we called on certain events like \\dQuote{pre.submit} or \\dQuote{post.sync}.\nSee \\link{Hooks}.}\n}\n\\description{\nThis is the constructor used to create \\emph{custom} cluster functions.\nNote that some standard implementations for TORQUE, Slurm, LSF, SGE, etc. ship\nwith the package.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeSubmitJobResult}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctions}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/makeClusterFunctionsDocker.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsDocker.R\n\\name{makeClusterFunctionsDocker}\n\\alias{makeClusterFunctionsDocker}\n\\title{ClusterFunctions for Docker}\n\\usage{\nmakeClusterFunctionsDocker(\n image,\n docker.args = character(0L),\n image.args = character(0L),\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{image}{[\\code{character(1)}]\\cr\nName of the docker image to run.}\n\n\\item{docker.args}{[\\code{character}]\\cr\nAdditional arguments passed to \\dQuote{docker} *before* the command (\\dQuote{run}, \\dQuote{ps} or \\dQuote{kill}) to execute (e.g., the docker host).}\n\n\\item{image.args}{[\\code{character}]\\cr\nAdditional arguments passed to \\dQuote{docker run} (e.g., to define mounts or environment variables).}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for Docker/Docker Swarm (\\url{https://docs.docker.com/engine/swarm/}).\n\nThe \\code{submitJob} function executes\n\\code{docker [docker.args] run --detach=true [image.args] [resources] [image] [cmd]}.\nArguments \\code{docker.args}, \\code{image.args} and \\code{image} can be set on construction.\nThe \\code{resources} part takes the named resources \\code{ncpus} and \\code{memory}\nfrom \\code{\\link{submitJobs}} and maps them to the arguments \\code{--cpu-shares} and \\code{--memory}\n(in Megabytes). The resource \\code{threads} is mapped to the environment variables \\dQuote{OMP_NUM_THREADS}\nand \\dQuote{OPENBLAS_NUM_THREADS}.\nTo reliably identify jobs in the swarm, jobs are labeled with \\dQuote{batchtools=[job.hash]} and named\nusing the current login name (label \\dQuote{user}) and the job hash (label \\dQuote{batchtools}).\n\n\\code{listJobsRunning} uses \\code{docker [docker.args] ps --format=\\{\\{.ID\\}\\}} to filter for running jobs.\n\n\\code{killJobs} uses \\code{docker [docker.args] kill [batch.id]} to filter for running jobs.\n\nThese cluster functions use a \\link{Hook} to remove finished jobs before a new submit and every time the \\link{Registry}\nis synchronized (using \\code{\\link{syncRegistry}}).\nThis is currently required because docker does not remove terminated containers automatically.\nUse \\code{docker ps -a --filter 'label=batchtools' --filter 'status=exited'} to identify and remove terminated\ncontainers manually (or usa a cron job).\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsHyperQueue.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsHyperQueue.R\n\\name{makeClusterFunctionsHyperQueue}\n\\alias{makeClusterFunctionsHyperQueue}\n\\title{ClusterFunctions for HyperQueue}\n\\usage{\nmakeClusterFunctionsHyperQueue(scheduler.latency = 1, fs.latency = 65)\n}\n\\arguments{\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[ClusterFunctions].\n}\n\\description{\nCluster functions for HyperQueue (\\url{https://it4innovations.github.io/hyperqueue/stable/}).\n\nJobs are submitted via the HyperQueue CLI using \\code{hq submit} and executed by calling \\code{Rscript -e \"batchtools::doJobCollection(...)\"}.\nThe job name is set to the job hash and logs are handled internally by batchtools.\nListing jobs uses \\code{hq job list} and cancelling jobs uses \\code{hq job cancel}.\nA running HyperQueue server and workers are required.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsInteractive.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsInteractive.R\n\\name{makeClusterFunctionsInteractive}\n\\alias{makeClusterFunctionsInteractive}\n\\title{ClusterFunctions for Sequential Execution in the Running R Session}\n\\usage{\nmakeClusterFunctionsInteractive(\n external = FALSE,\n write.logs = TRUE,\n fs.latency = 0\n)\n}\n\\arguments{\n\\item{external}{[\\code{logical(1)}]\\cr\nIf set to \\code{TRUE}, jobs are started in a fresh R session instead of currently active but still\nwaits for its termination.\nDefault is \\code{FALSE}.}\n\n\\item{write.logs}{[\\code{logical(1)}]\\cr\nSink the output to log files. Turning logging off can increase the speed of\ncalculations but makes it very difficult to debug.\nDefault is \\code{TRUE}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nAll jobs are executed sequentially using the current R process in which \\code{\\link{submitJobs}} is called.\nThus, \\code{submitJob} blocks the session until the job has finished.\nThe main use of this \\code{ClusterFunctions} implementation is to test and debug programs on a local computer.\n\nListing jobs returns an empty vector (as no jobs can be running when you call this)\nand \\code{killJob} is not implemented for the same reasons.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsLSF.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsLSF.R\n\\name{makeClusterFunctionsLSF}\n\\alias{makeClusterFunctionsLSF}\n\\title{ClusterFunctions for LSF Systems}\n\\usage{\nmakeClusterFunctionsLSF(\n template = \"lsf\",\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for LSF (\\url{https://www.ibm.com/products/hpc-workload-management}).\n\nJob files are created based on the brew template \\code{template.file}. This\nfile is processed with brew and then submitted to the queue using the\n\\code{bsub} command. Jobs are killed using the \\code{bkill} command and the\nlist of running jobs is retrieved using \\code{bjobs -u $USER -w}. The user\nmust have the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).\n\nThe template file can access all resources passed to \\code{\\link{submitJobs}}\nas well as all variables stored in the \\code{\\link{JobCollection}}.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.\n}\n\\note{\nArray jobs are currently not supported.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsMulticore.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsMulticore.R\n\\name{makeClusterFunctionsMulticore}\n\\alias{makeClusterFunctionsMulticore}\n\\title{ClusterFunctions for Parallel Multicore Execution}\n\\usage{\nmakeClusterFunctionsMulticore(ncpus = NA_integer_, fs.latency = 0)\n}\n\\arguments{\n\\item{ncpus}{[\\code{integer(1)}]\\cr\nNumber of CPUs.\nDefault is to use all logical cores. The total number of cores \"available\" can be set via the option \\code{mc.cores}\nand defaults to the heuristic implemented in \\code{\\link[parallel]{detectCores}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nJobs are spawned asynchronously using the functions \\code{mcparallel} and \\code{mccollect} (both in \\pkg{parallel}).\nDoes not work on Windows, use \\code{\\link{makeClusterFunctionsSocket}} instead.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsOpenLava.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsOpenLava.R\n\\name{makeClusterFunctionsOpenLava}\n\\alias{makeClusterFunctionsOpenLava}\n\\title{ClusterFunctions for OpenLava}\n\\usage{\nmakeClusterFunctionsOpenLava(\n template = \"openlava\",\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for OpenLava.\n\nJob files are created based on the brew template \\code{template}. This\nfile is processed with brew and then submitted to the queue using the\n\\code{bsub} command. Jobs are killed using the \\code{bkill} command and the\nlist of running jobs is retrieved using \\code{bjobs -u $USER -w}. The user\nmust have the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).\n\nThe template file can access all resources passed to \\code{\\link{submitJobs}}\nas well as all variables stored in the \\code{\\link{JobCollection}}.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.\n}\n\\note{\nArray jobs are currently not supported.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsSGE.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsSGE.R\n\\name{makeClusterFunctionsSGE}\n\\alias{makeClusterFunctionsSGE}\n\\title{ClusterFunctions for SGE Systems}\n\\usage{\nmakeClusterFunctionsSGE(\n template = \"sge\",\n nodename = \"localhost\",\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n\n\\item{nodename}{[\\code{character(1)}]\\cr\nNodename of the master host. All commands are send via SSH to this host. Only works iff\n\\enumerate{\n \\item{Passwordless authentication (e.g., via SSH public key authentication) is set up.}\n \\item{The file directory is shared across machines, e.g. mounted via SSHFS.}\n \\item{Either the absolute path to the \\code{file.dir} is identical on the machines, or paths are provided relative to the home directory. Symbolic links should work.}\n}}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for Univa Grid Engine / Oracle Grid Engine /\nSun Grid Engine (\\url{https://altair.com/hpc-cloud-applications/}).\n\nJob files are created based on the brew template \\code{template}. This\nfile is processed with brew and then submitted to the queue using the\n\\code{qsub} command. Jobs are killed using the \\code{qdel} command and the\nlist of running jobs is retrieved using \\code{qselect}. The user must have\nthe appropriate privileges to submit, delete and list jobs on the cluster\n(this is usually the case).\n\nThe template file can access all resources passed to \\code{\\link{submitJobs}}\nas well as all variables stored in the \\code{\\link{JobCollection}}.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.\n}\n\\note{\nArray jobs are currently not supported.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsSSH.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsSSH.R\n\\name{makeClusterFunctionsSSH}\n\\alias{makeClusterFunctionsSSH}\n\\title{ClusterFunctions for Remote SSH Execution}\n\\usage{\nmakeClusterFunctionsSSH(workers, fs.latency = 65)\n}\n\\arguments{\n\\item{workers}{[\\code{list} of \\code{\\link{Worker}}]\\cr\nList of Workers as constructed with \\code{\\link{Worker}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nJobs are spawned by starting multiple R sessions via \\code{Rscript} over SSH.\nIf the hostname of the \\code{\\link{Worker}} equals \\dQuote{localhost},\n\\code{Rscript} is called directly so that you do not need to have an SSH client installed.\n}\n\\note{\nIf you use a custom \\dQuote{.ssh/config} file, make sure your\nProxyCommand passes \\sQuote{-q} to ssh, otherwise each output will\nend with the message \\dQuote{Killed by signal 1} and this will break\nthe communication with the nodes.\n}\n\\examples{\n\\dontrun{\n# cluster functions for multicore execution on the local machine\nmakeClusterFunctionsSSH(list(Worker$new(\"localhost\", ncpus = 2)))\n}\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsSlurm.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsSlurm.R\n\\name{makeClusterFunctionsSlurm}\n\\alias{makeClusterFunctionsSlurm}\n\\title{ClusterFunctions for Slurm Systems}\n\\usage{\nmakeClusterFunctionsSlurm(\n template = \"slurm\",\n array.jobs = TRUE,\n nodename = \"localhost\",\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n\n\\item{array.jobs}{[\\code{logical(1)}]\\cr\nIf array jobs are disabled on the computing site, set to \\code{FALSE}.}\n\n\\item{nodename}{[\\code{character(1)}]\\cr\nNodename of the master host. All commands are send via SSH to this host. Only works iff\n\\enumerate{\n \\item{Passwordless authentication (e.g., via SSH public key authentication) is set up.}\n \\item{The file directory is shared across machines, e.g. mounted via SSHFS.}\n \\item{Either the absolute path to the \\code{file.dir} is identical on the machines, or paths are provided relative to the home directory. Symbolic links should work.}\n}}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for Slurm (\\url{https://slurm.schedmd.com/}).\n\nJob files are created based on the brew template \\code{template.file}. This\nfile is processed with brew and then submitted to the queue using the\n\\code{sbatch} command. Jobs are killed using the \\code{scancel} command and\nthe list of running jobs is retrieved using \\code{squeue}. The user must\nhave the appropriate privileges to submit, delete and list jobs on the\ncluster (this is usually the case).\n\nThe template file can access all resources passed to \\code{\\link{submitJobs}}\nas well as all variables stored in the \\code{\\link{JobCollection}}.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.\n\nNote that you might have to specify the cluster name here if you do not want to use the default,\notherwise the commands for listing and killing jobs will not work.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSocket}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsSocket.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsSocket.R\n\\name{makeClusterFunctionsSocket}\n\\alias{makeClusterFunctionsSocket}\n\\title{ClusterFunctions for Parallel Socket Execution}\n\\usage{\nmakeClusterFunctionsSocket(ncpus = NA_integer_, fs.latency = 65)\n}\n\\arguments{\n\\item{ncpus}{[\\code{integer(1)}]\\cr\nNumber of CPUs.\nDefault is to use all logical cores. The total number of cores \"available\" can be set via the option \\code{mc.cores}\nand defaults to the heuristic implemented in \\code{\\link[parallel]{detectCores}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nJobs are spawned asynchronously using the package \\pkg{snow}.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsTORQUE}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeClusterFunctionsTORQUE.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctionsTORQUE.R\n\\name{makeClusterFunctionsTORQUE}\n\\alias{makeClusterFunctionsTORQUE}\n\\title{ClusterFunctions for OpenPBS/TORQUE Systems}\n\\usage{\nmakeClusterFunctionsTORQUE(\n template = \"torque\",\n scheduler.latency = 1,\n fs.latency = 65\n)\n}\n\\arguments{\n\\item{template}{[\\code{character(1)}]\\cr\nEither a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n\\enumerate{\n \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n}}\n\n\\item{scheduler.latency}{[\\code{numeric(1)}]\\cr\nTime to sleep after important interactions with the scheduler to ensure a sane state.\nCurrently only triggered after calling \\code{\\link{submitJobs}}.}\n\n\\item{fs.latency}{[\\code{numeric(1)}]\\cr\nExpected maximum latency of the file system, in seconds.\nSet to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to\naccess files and directories.\nUsually safe to set to \\code{0} to disable the heuristic, e.g. if you are working on a local file system.}\n}\n\\value{\n[\\code{\\link{ClusterFunctions}}].\n}\n\\description{\nCluster functions for TORQUE/PBS (\\url{https://adaptivecomputing.com/cherry-services/torque-resource-manager/}).\n\nJob files are created based on the brew template \\code{template.file}. This file is processed\nwith brew and then submitted to the queue using the \\code{qsub} command. Jobs are killed using\nthe \\code{qdel} command and the list of running jobs is retrieved using \\code{qselect}. The user\nmust have the appropriate privileges to submit, delete and list jobs on the cluster (this is\nusually the case).\n\nThe template file can access all resources passed to \\code{\\link{submitJobs}}\nas well as all variables stored in the \\code{\\link{JobCollection}}.\nIt is the template file's job to choose a queue for the job and handle the desired resource\nallocations.\n}\n\\seealso{\nOther ClusterFunctions: \n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeClusterFunctionsDocker}()},\n\\code{\\link{makeClusterFunctionsHyperQueue}()},\n\\code{\\link{makeClusterFunctionsInteractive}()},\n\\code{\\link{makeClusterFunctionsLSF}()},\n\\code{\\link{makeClusterFunctionsMulticore}()},\n\\code{\\link{makeClusterFunctionsOpenLava}()},\n\\code{\\link{makeClusterFunctionsSGE}()},\n\\code{\\link{makeClusterFunctionsSSH}()},\n\\code{\\link{makeClusterFunctionsSlurm}()},\n\\code{\\link{makeClusterFunctionsSocket}()}\n}\n\\concept{ClusterFunctions}\n" }, { "path": "man/makeExperimentRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/ExperimentRegistry.R\n\\name{makeExperimentRegistry}\n\\alias{makeExperimentRegistry}\n\\alias{ExperimentRegistry}\n\\title{ExperimentRegistry Constructor}\n\\usage{\nmakeExperimentRegistry(\n file.dir = \"registry\",\n work.dir = getwd(),\n conf.file = findConfFile(),\n packages = character(0L),\n namespaces = character(0L),\n source = character(0L),\n load = character(0L),\n seed = NULL,\n make.default = TRUE\n)\n}\n\\arguments{\n\\item{file.dir}{[\\code{character(1)}]\\cr\n Path where all files of the registry are saved.\n Default is directory \\dQuote{registry} in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.\n\n If you pass \\code{NA}, a temporary directory will be used.\n This way, you can create disposable registries for \\code{\\link{btlapply}} or examples.\n By default, the temporary directory \\code{\\link[base]{tempdir}()} will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable \\code{temp.dir}.}\n\n\\item{work.dir}{[\\code{character(1)}]\\cr\n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see \\code{\\link[base]{getwd}}).\n \\code{loadRegistry} uses the stored \\code{work.dir}, but you may also explicitly overwrite it,\n e.g., after switching to another system.\n\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.}\n\n\\item{conf.file}{[\\code{character(1)}]\\cr\n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how \\pkg{batchtools} interacts with the system via \\code{\\link{ClusterFunctions}}.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.\n\n The file lookup is implemented in the internal (but exported) function \\code{findConfFile} which returns the first file found of the following candidates:\n \\enumerate{\n \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n }\n Set to \\code{NA} if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. \\code{default.resources = list(walltime = 3600)} to set default resources or \\dQuote{max.concurrent.jobs} to\n limit the number of jobs allowed to run simultaneously on the system.}\n\n\\item{packages}{[\\code{character}]\\cr\nPackages that will always be loaded on each node.\nUses \\code{\\link[base]{require}} internally.\nDefault is \\code{character(0)}.}\n\n\\item{namespaces}{[\\code{character}]\\cr\nSame as \\code{packages}, but the packages will not be attached.\nUses \\code{\\link[base]{requireNamespace}} internally.\nDefault is \\code{character(0)}.}\n\n\\item{source}{[\\code{character}]\\cr\nFiles which should be sourced on the slaves prior to executing a job.\nCalls \\code{\\link[base]{sys.source}} using the \\code{\\link[base]{.GlobalEnv}}.}\n\n\\item{load}{[\\code{character}]\\cr\nFiles which should be loaded on the slaves prior to executing a job.\nCalls \\code{\\link[base]{load}} using the \\code{\\link[base]{.GlobalEnv}}.}\n\n\\item{seed}{[\\code{integer(1)}]\\cr\nStart seed for jobs. Each job uses the (\\code{seed} + \\code{job.id}) as seed.\nDefault is a random integer between 1 and 32768.\nNote that there is an additional seeding mechanism to synchronize instantiation of\n\\code{\\link{Problem}}s in a \\code{\\link{ExperimentRegistry}}.}\n\n\\item{make.default}{[\\code{logical(1)}]\\cr\nIf set to \\code{TRUE}, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is \\code{TRUE}.}\n}\n\\value{\n[\\code{ExperimentRegistry}].\n}\n\\description{\n\\code{makeExperimentRegistry} constructs a special \\code{\\link{Registry}} which\nis suitable for the definition of large scale computer experiments.\n\nEach experiments consists of a \\code{\\link{Problem}} and an \\code{\\link{Algorithm}}.\nThese can be parametrized with \\code{\\link{addExperiments}} to actually define computational\njobs.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n\n# Definde one problem, two algorithms and add them with some parameters:\naddProblem(reg = tmp, \"p1\",\n fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd))\naddAlgorithm(reg = tmp, \"a1\", fun = function(job, data, instance, ...) mean(instance))\naddAlgorithm(reg = tmp, \"a2\", fun = function(job, data, instance, ...) median(instance))\nids = addExperiments(reg = tmp, list(p1 = data.table::CJ(n = c(50, 100), mean = -2:2, sd = 1:4)))\n\n# Overview over defined experiments:\ntmp$problems\ntmp$algorithms\nsummarizeExperiments(reg = tmp)\nsummarizeExperiments(reg = tmp, by = c(\"problem\", \"algorithm\", \"n\"))\nids = findExperiments(prob.pars = (n == 50), reg = tmp)\nprint(unwrap(getJobPars(ids, reg = tmp)))\n\n# Submit jobs\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\n\n# Reduce the results of algorithm a1\nids.mean = findExperiments(algo.name = \"a1\", reg = tmp)\nreduceResults(ids.mean, fun = function(aggr, res, ...) c(aggr, res), reg = tmp)\n\n# Join info table with all results and calculate mean of results\n# grouped by n and algorithm\nids = findDone(reg = tmp)\npars = unwrap(getJobPars(ids, reg = tmp))\nresults = unwrap(reduceResultsDataTable(ids, fun = function(res) list(res = res), reg = tmp))\ntab = ljoin(pars, results)\ntab[, list(mres = mean(res)), by = c(\"n\", \"algorithm\")]\n}\n\\concept{Registry Experiment}\n" }, { "path": "man/makeRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Registry.R\n\\name{makeRegistry}\n\\alias{makeRegistry}\n\\alias{Registry}\n\\title{Registry Constructor}\n\\usage{\nmakeRegistry(\n file.dir = \"registry\",\n work.dir = getwd(),\n conf.file = findConfFile(),\n packages = character(0L),\n namespaces = character(0L),\n source = character(0L),\n load = character(0L),\n seed = NULL,\n make.default = TRUE\n)\n}\n\\arguments{\n\\item{file.dir}{[\\code{character(1)}]\\cr\n Path where all files of the registry are saved.\n Default is directory \\dQuote{registry} in the current working directory.\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.\n\n If you pass \\code{NA}, a temporary directory will be used.\n This way, you can create disposable registries for \\code{\\link{btlapply}} or examples.\n By default, the temporary directory \\code{\\link[base]{tempdir}()} will be used.\n If you want to use another directory, e.g. a directory which is shared between nodes,\n you can set it in your configuration file by setting the variable \\code{temp.dir}.}\n\n\\item{work.dir}{[\\code{character(1)}]\\cr\n Working directory for R process for running jobs.\n Defaults to the working directory currently set during Registry construction (see \\code{\\link[base]{getwd}}).\n \\code{loadRegistry} uses the stored \\code{work.dir}, but you may also explicitly overwrite it,\n e.g., after switching to another system.\n\n The provided path will get normalized unless it is given relative to the home directory\n (i.e., starting with \\dQuote{~}). Note that some templates do not handle relative paths well.}\n\n\\item{conf.file}{[\\code{character(1)}]\\cr\n Path to a configuration file which is sourced while the registry is created.\n In the configuration file you can define how \\pkg{batchtools} interacts with the system via \\code{\\link{ClusterFunctions}}.\n Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.\n\n The file lookup is implemented in the internal (but exported) function \\code{findConfFile} which returns the first file found of the following candidates:\n \\enumerate{\n \\item{File \\dQuote{batchtools.conf.R} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.}\n \\item{File \\dQuote{batchtools.conf.R} in the current working directory.}\n \\item{File \\dQuote{config.R} in the user configuration directory as reported by \\code{rappdirs::user_config_dir(\"batchtools\", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \\dQuote{~/.config/batchtools/config.R}).}\n \\item{\\dQuote{.batchtools.conf.R} in the home directory (\\dQuote{~}).}\n \\item{\\dQuote{config.R} in the site config directory as reported by \\code{rappdirs::site_config_dir(\"batchtools\")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}\n }\n Set to \\code{NA} if you want to suppress reading any configuration file.\n If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.\n Therefore you can set and overwrite slots, e.g. \\code{default.resources = list(walltime = 3600)} to set default resources or \\dQuote{max.concurrent.jobs} to\n limit the number of jobs allowed to run simultaneously on the system.}\n\n\\item{packages}{[\\code{character}]\\cr\nPackages that will always be loaded on each node.\nUses \\code{\\link[base]{require}} internally.\nDefault is \\code{character(0)}.}\n\n\\item{namespaces}{[\\code{character}]\\cr\nSame as \\code{packages}, but the packages will not be attached.\nUses \\code{\\link[base]{requireNamespace}} internally.\nDefault is \\code{character(0)}.}\n\n\\item{source}{[\\code{character}]\\cr\nFiles which should be sourced on the slaves prior to executing a job.\nCalls \\code{\\link[base]{sys.source}} using the \\code{\\link[base]{.GlobalEnv}}.}\n\n\\item{load}{[\\code{character}]\\cr\nFiles which should be loaded on the slaves prior to executing a job.\nCalls \\code{\\link[base]{load}} using the \\code{\\link[base]{.GlobalEnv}}.}\n\n\\item{seed}{[\\code{integer(1)}]\\cr\nStart seed for jobs. Each job uses the (\\code{seed} + \\code{job.id}) as seed.\nDefault is a random integer between 1 and 32768.\nNote that there is an additional seeding mechanism to synchronize instantiation of\n\\code{\\link{Problem}}s in a \\code{\\link{ExperimentRegistry}}.}\n\n\\item{make.default}{[\\code{logical(1)}]\\cr\nIf set to \\code{TRUE}, the created registry is saved inside the package\nnamespace and acts as default registry. You might want to switch this\noff if you work with multiple registries simultaneously.\nDefault is \\code{TRUE}.}\n}\n\\value{\n[\\code{environment}] of class \\dQuote{Registry} with the following slots:\n \\describe{\n \\item{\\code{file.dir} [path]:}{File directory.}\n \\item{\\code{work.dir} [path]:}{Working directory.}\n \\item{\\code{temp.dir} [path]:}{Temporary directory. Used if \\code{file.dir} is \\code{NA} to create temporary registries.}\n \\item{\\code{packages} [character()]:}{Packages to load on the slaves.}\n \\item{\\code{namespaces} [character()]:}{Namespaces to load on the slaves.}\n \\item{\\code{seed} [integer(1)]:}{Registry seed. Before each job is executed, the seed \\code{seed + job.id} is set.}\n \\item{\\code{cluster.functions} [cluster.functions]:}{Usually set in your \\code{conf.file}. Set via a call to \\code{\\link{makeClusterFunctions}}. See example.}\n \\item{\\code{default.resources} [named list()]:}{Usually set in your \\code{conf.file}. Named list of default resources.}\n \\item{\\code{max.concurrent.jobs} [integer(1)]:}{Usually set in your \\code{conf.file}. Maximum number of concurrent jobs for a single user and current registry on the system.\n \\code{\\link{submitJobs}} will try to respect this setting. The resource \\dQuote{max.concurrent.jobs} has higher precedence.}\n \\item{\\code{defs} [data.table]:}{Table with job definitions (i.e. parameters).}\n \\item{\\code{status} [data.table]:}{Table holding information about the computational status. Also see \\code{\\link{getJobStatus}}.}\n \\item{\\code{resources} [data.table]:}{Table holding information about the computational resources used for the job. Also see \\code{\\link{getJobResources}}.}\n \\item{\\code{tags} [data.table]:}{Table holding information about tags. See \\link{Tags}.}\n \\item{\\code{hash} [character(1)]:}{Unique hash which changes each time the registry gets saved to the file system. Can be utilized to invalidate the cache of \\pkg{knitr}.}\n }\n}\n\\description{\n\\code{makeRegistry} constructs the inter-communication object for all functions in \\code{batchtools}.\nAll communication transactions are processed via the file system:\nAll information required to run a job is stored as \\code{\\link{JobCollection}} in a file in the\na subdirectory of the \\code{file.dir} directory.\nEach jobs stores its results as well as computational status information (start time, end time, error message, ...)\nalso on the file system which is regular merged parsed by the master using \\code{\\link{syncRegistry}}.\nAfter integrating the new information into the Registry, the Registry is serialized to the file system via \\code{\\link{saveRegistry}}.\nBoth \\code{\\link{syncRegistry}} and \\code{\\link{saveRegistry}} are called whenever required internally.\nTherefore it should be safe to quit the R session at any time.\nWork can later be resumed by calling \\code{\\link{loadRegistry}} which de-serializes the registry from\nthe file system.\n\nThe registry created last is saved in the package namespace (unless \\code{make.default} is set to\n\\code{FALSE}) and can be retrieved via \\code{\\link{getDefaultRegistry}}.\n\nCanceled jobs and jobs submitted multiple times may leave stray files behind.\nThese can be swept using \\code{\\link{sweepRegistry}}.\n\\code{\\link{clearRegistry}} completely erases all jobs from a registry, including log files and results,\nand thus allows you to start over.\n}\n\\details{\nCurrently \\pkg{batchtools} understands the following options set via the configuration file:\n\\describe{\n \\item{\\code{cluster.functions}:}{As returned by a constructor, e.g. \\code{\\link{makeClusterFunctionsSlurm}}.}\n \\item{\\code{default.resources}:}{List of resources to use. Will be overruled by resources specified via \\code{\\link{submitJobs}}.}\n \\item{\\code{temp.dir}:}{Path to directory to use for temporary registries.}\n \\item{\\code{sleep}:}{Custom sleep function. See \\code{\\link{waitForJobs}}.}\n \\item{\\code{expire.after}:}{Number of iterations before treating jobs as expired in \\code{\\link{waitForJobs}}.}\n \\item{\\code{compress}:}{Compression algorithm to use via \\code{\\link{saveRDS}}.}\n}\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nprint(tmp)\n\n# Set cluster functions to interactive mode and start jobs in external R sessions\ntmp$cluster.functions = makeClusterFunctionsInteractive(external = TRUE)\n\n# Change packages to load\ntmp$packages = c(\"MASS\")\nsaveRegistry(reg = tmp)\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/makeSubmitJobResult.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{makeSubmitJobResult}\n\\alias{makeSubmitJobResult}\n\\alias{SubmitJobResult}\n\\title{Create a SubmitJobResult}\n\\usage{\nmakeSubmitJobResult(\n status,\n batch.id,\n log.file = NA_character_,\n msg = NA_character_\n)\n}\n\\arguments{\n\\item{status}{[\\code{integer(1)}]\\cr\nLaunch status of job. 0 means success, codes between 1 and 100 are temporary errors and any\nerror greater than 100 is a permanent failure.}\n\n\\item{batch.id}{[\\code{character()}]\\cr\nUnique id of this job on batch system, as given by the batch system.\nMust be globally unique so that the job can be terminated using just this information.\nFor array jobs, this may be a vector of length equal to the number of jobs in the array.}\n\n\\item{log.file}{[\\code{character()}]\\cr\nLog file. If \\code{NA}, defaults to \\code{[job.hash].log}.\nSome cluster functions set this for array jobs.}\n\n\\item{msg}{[\\code{character(1)}]\\cr\nOptional error message in case \\code{status} is not equal to 0. Default is \\dQuote{OK},\n\\dQuote{TEMPERROR}, \\dQuote{ERROR}, depending on \\code{status}.}\n}\n\\value{\n[\\code{\\link{SubmitJobResult}}]. A list, containing\n \\code{status}, \\code{batch.id} and \\code{msg}.\n}\n\\description{\nThis function is only intended for use in your own cluster functions implementation.\n\nUse this function in your implementation of \\code{\\link{makeClusterFunctions}} to create a return\nvalue for the \\code{submitJob} function.\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{runOSCommand}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/reduceResults.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/reduceResults.R\n\\name{reduceResults}\n\\alias{reduceResults}\n\\title{Reduce Results}\n\\usage{\nreduceResults(fun, ids = NULL, init, ..., reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{fun}{[\\code{function}]\\cr\nA function to reduce the results. The result of previous iterations (or\nthe \\code{init}) will be passed as first argument, the result of of the\ni-th iteration as second. See \\code{\\link[base]{Reduce}} for some\nexamples.\nIf the function has the formal argument \\dQuote{job}, the \\code{\\link{Job}}/\\code{\\link{Experiment}}\nis also passed to the function (named).}\n\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findDone}}.\nInvalid ids are ignored.}\n\n\\item{init}{[\\code{ANY}]\\cr\nInitial element, as used in \\code{\\link[base]{Reduce}}.\nIf missing, the reduction uses the result of the first job as \\code{init} and the reduction starts\nwith the second job.}\n\n\\item{...}{[\\code{ANY}]\\cr\nAdditional arguments passed to function \\code{fun}.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\nAggregated results in the same order as provided ids.\n Return type depends on the user function. If \\code{ids}\n is empty, \\code{reduceResults} returns \\code{init} (if available) or \\code{NULL} otherwise.\n}\n\\description{\nA version of \\code{\\link[base]{Reduce}} for \\code{\\link{Registry}} objects\nwhich iterates over finished jobs and aggregates them.\nAll jobs must have terminated, an error is raised otherwise.\n}\n\\note{\nIf you have thousands of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\ncan significantly increase the performance.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(function(a, b) list(sum = a+b, prod = a*b), a = 1:3, b = 1:3, reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\n\n# Extract element sum from each result\nreduceResults(function(aggr, res) c(aggr, res$sum), init = list(), reg = tmp)\n\n# Aggregate element sum via '+'\nreduceResults(function(aggr, res) aggr + res$sum, init = 0, reg = tmp)\n\n# Aggregate element prod via '*' where parameter b < 3\nreduce = function(aggr, res, job) {\n if (job$pars$b >= 3)\n return(aggr)\n aggr * res$prod\n}\nreduceResults(reduce, init = 1, reg = tmp)\n\n# Reduce to data.frame() (inefficient, use reduceResultsDataTable() instead)\nreduceResults(rbind, init = data.frame(), reg = tmp)\n\n# Reduce to data.frame by collecting results first, then utilize vectorization of rbind:\nres = reduceResultsList(fun = as.data.frame, reg = tmp)\ndo.call(rbind, res)\n\n# Reduce with custom combine function:\ncomb = function(x, y) list(sum = x$sum + y$sum, prod = x$prod * y$prod)\nreduceResults(comb, reg = tmp)\n\n# The same with neutral element NULL\ncomb = function(x, y) if (is.null(x)) y else list(sum = x$sum + y$sum, prod = x$prod * y$prod)\nreduceResults(comb, init = NULL, reg = tmp)\n\n# Alternative: Reduce in list, reduce manually in a 2nd step\nres = reduceResultsList(reg = tmp)\nReduce(comb, res)\n}\n\\seealso{\nOther Results: \n\\code{\\link{batchMapResults}()},\n\\code{\\link{loadResult}()},\n\\code{\\link{reduceResultsList}()}\n}\n\\concept{Results}\n" }, { "path": "man/reduceResultsList.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/reduceResults.R\n\\name{reduceResultsList}\n\\alias{reduceResultsList}\n\\alias{reduceResultsDataTable}\n\\title{Apply Functions on Results}\n\\usage{\nreduceResultsList(\n ids = NULL,\n fun = NULL,\n ...,\n missing.val,\n reg = getDefaultRegistry()\n)\n\nreduceResultsDataTable(\n ids = NULL,\n fun = NULL,\n ...,\n missing.val,\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findDone}}.\nInvalid ids are ignored.}\n\n\\item{fun}{[\\code{function}]\\cr\nFunction to apply to each result. The result is passed unnamed as first argument. If \\code{NULL}, the identity is used.\nIf the function has the formal argument \\dQuote{job}, the \\code{\\link{Job}}/\\code{\\link{Experiment}} is also passed to the function.}\n\n\\item{...}{[\\code{ANY}]\\cr\nAdditional arguments passed to to function \\code{fun}.}\n\n\\item{missing.val}{[\\code{ANY}]\\cr\nValue to impute as result for a job which is not finished.\nIf not provided and a result is missing, an exception is raised.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n\\code{reduceResultsList} returns a list of the results in the same order as the provided ids.\n \\code{reduceResultsDataTable} returns a \\code{\\link[data.table]{data.table}} with columns \\dQuote{job.id} and additional result columns\n created via \\code{\\link[data.table]{rbindlist}}, sorted by \\dQuote{job.id}.\n}\n\\description{\nApplies a function on the results of your finished jobs and thereby collects\nthem in a \\code{\\link[base]{list}} or \\code{\\link[data.table]{data.table}}.\nThe later requires the provided function to return a list (or \\code{data.frame}) of scalar values.\nSee \\code{\\link[data.table]{rbindlist}} for features and limitations of the aggregation.\n\nIf not all jobs are terminated, the respective result will be \\code{NULL}.\n}\n\\note{\nIf you have thousands of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\ncan significantly increase the performance.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(2) }\n### Example 1 - reduceResultsList\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(function(x) x^2, x = 1:10, reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\nreduceResultsList(fun = sqrt, reg = tmp)\n\n### Example 2 - reduceResultsDataTable\ntmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)\n\n# add first problem\nfun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)\naddProblem(\"rnorm\", fun = fun, reg = tmp)\n\n# add second problem\nfun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)\naddProblem(\"rexp\", fun = fun, reg = tmp)\n\n# add first algorithm\nfun = function(instance, method, ...) if (method == \"mean\") mean(instance) else median(instance)\naddAlgorithm(\"average\", fun = fun, reg = tmp)\n\n# add second algorithm\nfun = function(instance, ...) sd(instance)\naddAlgorithm(\"deviation\", fun = fun, reg = tmp)\n\n# define problem and algorithm designs\nlibrary(data.table)\nprob.designs = algo.designs = list()\nprob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)\nprob.designs$rexp = data.table(n = 100, lambda = 1:5)\nalgo.designs$average = data.table(method = c(\"mean\", \"median\"))\nalgo.designs$deviation = data.table()\n\n# add experiments and submit\naddExperiments(prob.designs, algo.designs, reg = tmp)\nsubmitJobs(reg = tmp)\n\n# collect results and join them with problem and algorithm paramters\nres = ijoin(\n getJobPars(reg = tmp),\n reduceResultsDataTable(reg = tmp, fun = function(x) list(res = x))\n)\nunwrap(res, sep = \".\")\n}\n\\seealso{\n\\code{\\link{reduceResults}}\n\nOther Results: \n\\code{\\link{batchMapResults}()},\n\\code{\\link{loadResult}()},\n\\code{\\link{reduceResults}()}\n}\n\\concept{Results}\n" }, { "path": "man/removeExperiments.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/removeExperiments.R\n\\name{removeExperiments}\n\\alias{removeExperiments}\n\\title{Remove Experiments}\n\\usage{\nremoveExperiments(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to no job.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{ExperimentRegistry}}]\\cr\nRegistry. If not explicitly passed, uses the last created registry.}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] of removed job ids, invisibly.\n}\n\\description{\nRemove Experiments from an \\code{\\link{ExperimentRegistry}}.\nThis function automatically checks if any of the jobs to reset is either pending or running.\nHowever, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\nUse with care while jobs are running.\n}\n\\seealso{\nOther Experiment: \n\\code{\\link{addExperiments}()},\n\\code{\\link{summarizeExperiments}()}\n}\n\\concept{Experiment}\n" }, { "path": "man/removeRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/removeRegistry.R\n\\name{removeRegistry}\n\\alias{removeRegistry}\n\\title{Remove a Registry from the File System}\n\\usage{\nremoveRegistry(wait = 5, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{wait}{[\\code{numeric(1)}]\\cr\nSeconds to wait before proceeding. This is a safety measure to not\naccidentally remove your precious files. Set to 0 in\nnon-interactive scripts to disable this precaution.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{character(1)}]: Path of the deleted file directory.\n}\n\\description{\nAll files will be erased from the file system, including all results.\nIf you wish to remove only intermediate files, use \\code{\\link{sweepRegistry}}.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nremoveRegistry(0, tmp)\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/resetJobs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/resetJobs.R\n\\name{resetJobs}\n\\alias{resetJobs}\n\\title{Reset the Computational State of Jobs}\n\\usage{\nresetJobs(ids = NULL, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to no job.\nInvalid ids are ignored.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] of job ids which have been reset.\n See \\code{\\link{JoinTables}} for examples on working with job tables.\n}\n\\description{\nResets the computational state of jobs in the \\code{\\link{Registry}}.\nThis function automatically checks if any of the jobs to reset is either pending or running.\nHowever, if the implemented heuristic fails, this can lead to inconsistencies in the data base.\nUse with care while jobs are running.\n}\n\\seealso{\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{getStatus}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{showLog}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/runHook.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Hooks.R\n\\name{runHook}\n\\alias{runHook}\n\\alias{Hooks}\n\\alias{Hook}\n\\title{Trigger Evaluation of Custom Function}\n\\usage{\nrunHook(obj, hook, ...)\n}\n\\arguments{\n\\item{obj}{[\\link{Registry} | \\link{JobCollection}]\\cr\nRegistry which contains the \\link{ClusterFunctions} with element \\dQuote{hooks}\nor a \\link{JobCollection} which holds the subset of functions which are executed\nremotely.}\n\n\\item{hook}{[\\code{character(1)}]\\cr\nID of the hook as string.}\n\n\\item{...}{[ANY]\\cr\nAdditional arguments passed to the function referenced by \\code{hook}.\nSee description.}\n}\n\\value{\nReturn value of the called function, or \\code{NULL} if there is no hook\n with the specified ID.\n}\n\\description{\nHooks allow to trigger functions calls on specific events.\nThey can be specified via the \\code{\\link{ClusterFunctions}} and are triggered on the following events:\n\\describe{\n \\item{\\code{pre.sync}}{\\code{function(reg, fns, ...)}: Run before synchronizing the registry on the master. \\code{fn} is the character vector of paths to the update files.}\n \\item{\\code{post.sync}}{\\code{function(reg, updates, ...)}: Run after synchronizing the registry on the master. \\code{updates} is the data.table of processed updates.}\n \\item{\\code{pre.submit.job}}{\\code{function(reg, ...)}: Run before a job is successfully submitted to the scheduler on the master.}\n \\item{\\code{post.submit.job}}{\\code{function(reg, ...)}: Run after a job is successfully submitted to the scheduler on the master.}\n \\item{\\code{pre.submit}}{\\code{function(reg, ...)}: Run before any job is submitted to the scheduler.}\n \\item{\\code{post.submit}}{\\code{function(reg, ...)}: Run after a jobs are submitted to the schedule.}\n \\item{\\code{pre.do.collection}}{\\code{function(reg, reader, ...)}: Run before starting the job collection on the slave.\n \\code{reader} is an internal cache object.}\n \\item{\\code{post.do.collection}}{\\code{function(reg, updates, reader, ...)}: Run after all jobs in the chunk are terminated on the slave.\n \\code{updates} is a \\code{\\link[data.table]{data.table}} of updates which will be merged with the \\code{\\link{Registry}} by the master.\n \\code{reader} is an internal cache object.}\n \\item{\\code{pre.kill}}{\\code{function(reg, ids, ...)}: Run before any job is killed.}\n \\item{\\code{post.kill}}{\\code{function(reg, ids, ...)}: Run after jobs are killed. \\code{ids} is the return value of \\code{\\link{killJobs}}.}\n}\n}\n" }, { "path": "man/runOSCommand.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/runOSCommand.R\n\\name{runOSCommand}\n\\alias{runOSCommand}\n\\title{Run OS Commands on Local or Remote Machines}\n\\usage{\nrunOSCommand(\n sys.cmd,\n sys.args = character(0L),\n stdin = \"\",\n nodename = \"localhost\"\n)\n}\n\\arguments{\n\\item{sys.cmd}{[\\code{character(1)}]\\cr\nCommand to run.}\n\n\\item{sys.args}{[\\code{character}]\\cr\nArguments for \\code{sys.cmd}.}\n\n\\item{stdin}{[\\code{character(1)}]\\cr\nArgument passed to \\code{\\link[base]{system2}}.}\n\n\\item{nodename}{[\\code{character(1)}]\\cr\nName of the SSH node to run the command on. If set to \\dQuote{localhost} (default), the command\nis not piped through SSH.}\n}\n\\value{\n[\\code{named list}] with \\dQuote{sys.cmd}, \\dQuote{sys.args}, \\dQuote{exit.code} (integer), \\dQuote{output} (character).\n}\n\\description{\nThis is a helper function to run arbitrary OS commands on local or remote machines.\nThe interface is similar to \\code{\\link[base]{system2}}, but it always returns the exit status\n\\emph{and} the output.\n}\n\\examples{\n\\dontrun{\nrunOSCommand(\"ls\")\nrunOSCommand(\"ls\", \"-al\")\nrunOSCommand(\"notfound\")\n}\n}\n\\seealso{\nOther ClusterFunctionsHelper: \n\\code{\\link{cfBrewTemplate}()},\n\\code{\\link{cfHandleUnknownSubmitError}()},\n\\code{\\link{cfKillJob}()},\n\\code{\\link{cfReadBrewTemplate}()},\n\\code{\\link{makeClusterFunctions}()},\n\\code{\\link{makeSubmitJobResult}()}\n}\n\\concept{ClusterFunctionsHelper}\n" }, { "path": "man/saveRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/saveRegistry.R\n\\name{saveRegistry}\n\\alias{saveRegistry}\n\\title{Store the Registy to the File System}\n\\usage{\nsaveRegistry(reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{logical(1)}]: \\code{TRUE} if the registry was saved,\n \\code{FALSE} otherwise (if the registry is read-only).\n}\n\\description{\nStores the registry on the file system in its \\dQuote{file.dir} (specified\nfor construction in \\code{\\link{makeRegistry}}, can be accessed via\n\\code{reg$file.dir}).\nThis function is usually called internally whenever needed.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{sweepRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/showLog.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Logs.R\n\\name{showLog}\n\\alias{showLog}\n\\alias{getLog}\n\\title{Inspect Log Files}\n\\usage{\nshowLog(id, reg = getDefaultRegistry())\n\ngetLog(id, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{id}{[\\code{integer(1)} or \\code{data.table}]\\cr\nSingle integer to specify the job or a \\code{data.table} with column \\code{job.id}\nand exactly one row.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\nNothing.\n}\n\\description{\n\\code{showLog} opens the log in the pager. For customization, see \\code{\\link[base]{file.show}}.\n\\code{getLog} returns the log as character vector.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n\n# Create some dummy jobs\nfun = function(i) {\n if (i == 3) stop(i)\n if (i \\%\\% 2 == 1) warning(\"That's odd.\")\n}\nids = batchMap(fun, i = 1:5, reg = tmp)\nsubmitJobs(reg = tmp)\nwaitForJobs(reg = tmp)\ngetStatus(reg = tmp)\n\nwriteLines(getLog(ids[1], reg = tmp))\n\\dontrun{\nshowLog(ids[1], reg = tmp)\n}\n\ngrepLogs(pattern = \"warning\", ignore.case = TRUE, reg = tmp)\n}\n\\seealso{\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{getStatus}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{testJob}()}\n}\n\\concept{debug}\n" }, { "path": "man/submitJobs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/submitJobs.R\n\\name{submitJobs}\n\\alias{submitJobs}\n\\title{Submit Jobs to the Batch Systems}\n\\usage{\nsubmitJobs(\n ids = NULL,\n resources = list(),\n sleep = NULL,\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findNotSubmitted}}.\nInvalid ids are ignored.}\n\n\\item{resources}{[\\code{named list}]\\cr\nComputational resources for the jobs to submit. The actual elements of this list\n(e.g. something like \\dQuote{walltime} or \\dQuote{nodes}) depend on your template file, exceptions are outlined in the section 'Resources'.\nDefault settings for a system can be set in the configuration file by defining the named list \\code{default.resources}.\nNote that these settings are merged by name, e.g. merging \\code{list(walltime = 300)} into \\code{list(walltime = 400, memory = 512)}\nwill result in \\code{list(walltime = 300, memory = 512)}.\nSame holds for individual job resources passed as additional column of \\code{ids} (c.f. section 'Resources').}\n\n\\item{sleep}{[\\code{function(i)} | \\code{numeric(1)}]\\cr\nParameter to control the duration to sleep between temporary errors.\nYou can pass an absolute numeric value in seconds or a \\code{function(i)} which returns the number of seconds to sleep in the \\code{i}-th\niteration between temporary errors.\nIf not provided (\\code{NULL}), tries to read the value (number/function) from the configuration file (stored in \\code{reg$sleep}) or defaults to\na function with exponential backoff between 5 and 120 seconds.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] with columns \\dQuote{job.id} and \\dQuote{chunk}.\n}\n\\description{\nSubmits defined jobs to the batch system.\n\nAfter submitting the jobs, you can use \\code{\\link{waitForJobs}} to wait for the\ntermination of jobs or call \\code{\\link{reduceResultsList}}/\\code{\\link{reduceResults}}\nto collect partial results.\nThe progress can be monitored with \\code{\\link{getStatus}}.\n}\n\\note{\nIf you a large number of jobs, disabling the progress bar (\\code{options(batchtools.progress = FALSE)})\ncan significantly increase the performance of \\code{submitJobs}.\n}\n\\section{Resources}{\n\nYou can pass arbitrary resources to \\code{submitJobs()} which then are available in the cluster function template.\nSome resources' names are standardized and it is good practice to stick to the following nomenclature to avoid confusion:\n\\describe{\n \\item{walltime:}{Upper time limit in seconds for jobs before they get killed by the scheduler. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n \\item{memory:}{Memory limit in Mb. If jobs exceed this limit, they are usually killed by the scheduler. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n \\item{ncpus:}{Number of (physical) CPUs to use on the slave. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n \\item{omp.threads:}{Number of threads to use via OpenMP. Used to set environment variable \\dQuote{OMP_NUM_THREADS}. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n \\item{pp.size:}{Maximum size of the pointer protection stack, see \\code{\\link[base]{Memory}}.}\n \\item{blas.threads:}{Number of threads to use for the BLAS backend. Used to set environment variables \\dQuote{MKL_NUM_THREADS} and \\dQuote{OPENBLAS_NUM_THREADS}. Can be passed as additional column as part of \\code{ids} to set per-job resources.}\n \\item{measure.memory:}{Enable memory measurement for jobs. Comes with a small runtime overhead.}\n \\item{chunks.as.arrayjobs:}{Execute chunks as array jobs.}\n \\item{pm.backend:}{Start a \\pkg{parallelMap} backend on the slave.}\n \\item{foreach.backend:}{Start a \\pkg{foreach} backend on the slave.}\n \\item{clusters:}{Resource used for Slurm to select the set of clusters to run \\code{sbatch}/\\code{squeue}/\\code{scancel} on.}\n}\n}\n\n\\section{Chunking of Jobs}{\n\nMultiple jobs can be grouped (chunked) together to be executed sequentially on the batch system as a single batch job.\nThis is especially useful to avoid overburding the scheduler by submitting thousands of jobs simultaneously.\nTo chunk jobs together, job ids must be provided as \\code{data.frame} with columns \\dQuote{job.id} and \\dQuote{chunk} (integer).\nAll jobs with the same chunk number will be executed sequentially inside the same batch job.\nThe utility functions \\code{\\link{chunk}}, \\code{\\link{binpack}} and \\code{\\link{lpt}}\ncan assist in grouping jobs.\n}\n\n\\section{Array Jobs}{\n\nIf your cluster supports array jobs, you can set the resource \\code{chunks.as.arrayjobs} to \\code{TRUE} in order\nto execute chunks as job arrays on the cluster.\nFor each chunk of size \\code{n}, \\pkg{batchtools} creates a \\code{\\link{JobCollection}} of (possibly heterogeneous) jobs which is\nsubmitted to the scheduler as a single array job with \\code{n} repetitions.\nFor each repetition, the \\code{JobCollection} is first read from the file system, then subsetted to the \\code{i}-th job using\nthe environment variable \\code{reg$cluster.functions$array.var} (depending on the cluster backend, defined automatically) and finally\nexecuted.\n}\n\n\\section{Order of Submission}{\n\nJobs are submitted in the order of chunks, i.e. jobs which have chunk number\n\\code{sort(unique(ids$chunk))[1]} first, then jobs with chunk number \\code{sort(unique(ids$chunk))[2]}\nand so on. If no chunks are provided, jobs are submitted in the order of \\code{ids$job.id}.\n}\n\n\\section{Limiting the Number of Jobs}{\n\nIf requested, \\code{submitJobs} tries to limit the number of concurrent jobs of the user by waiting until jobs terminate\nbefore submitting new ones.\nThis can be controlled by setting \\dQuote{max.concurrent.jobs} in the configuration file (see \\code{\\link{Registry}})\nor by setting the resource \\dQuote{max.concurrent.jobs} to the maximum number of jobs to run simultaneously.\nIf both are set, the setting via the resource takes precedence over the setting in the configuration.\n}\n\n\\section{Measuring Memory}{\n\nSetting the resource \\code{measure.memory} to \\code{TRUE} turns on memory measurement:\n\\code{\\link[base]{gc}} is called directly before and after the job and the difference is\nstored in the internal database. Note that this is just a rough estimate and does\nneither work reliably for external code like C/C++ nor in combination with threading.\n}\n\n\\section{Inner Parallelization}{\n\nInner parallelization is typically done via threading, sockets or MPI.\nTwo backends are supported to assist in setting up inner parallelization.\n\nThe first package is \\pkg{parallelMap}.\nIf you set the resource \\dQuote{pm.backend} to \\dQuote{multicore}, \\dQuote{socket} or \\dQuote{mpi},\n\\code{\\link[parallelMap]{parallelStart}} is called on the slave before the first job in the chunk is started\nand \\code{\\link[parallelMap]{parallelStop}} is called after the last job terminated.\nThis way, the resources for inner parallelization can be set and get automatically stored just like other computational resources.\nThe function provided by the user just has to call \\code{\\link[parallelMap]{parallelMap}} to start parallelization using the preconfigured backend.\n\nTo control the number of CPUs, you have to set the resource \\code{ncpus}.\nOtherwise \\code{ncpus} defaults to the number of available CPUs (as reported by (see \\code{\\link[parallel]{detectCores}}))\non the executing machine for multicore and socket mode and defaults to the return value of \\code{\\link[Rmpi]{mpi.universe.size}-1} for MPI.\nYour template must be set up to handle the parallelization, e.g. request the right number of CPUs or start R with \\code{mpirun}.\nYou may pass further options like \\code{level} to \\code{\\link[parallelMap]{parallelStart}} via the named list \\dQuote{pm.opts}.\n\nThe second supported parallelization backend is \\pkg{foreach}.\nIf you set the resource \\dQuote{foreach.backend} to \\dQuote{seq} (sequential mode), \\dQuote{parallel} (\\pkg{doParallel}) or\n\\dQuote{mpi} (\\pkg{doMPI}), the requested \\pkg{foreach} backend is automatically registered on the slave.\nAgain, the resource \\code{ncpus} is used to determine the number of CPUs.\n\nNeither the namespace of \\pkg{parallelMap} nor the namespace \\pkg{foreach} are attached.\nYou have to do this manually via \\code{\\link[base]{library}} or let the registry load the packages for you.\n}\n\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(3) }\n### Example 1: Submit subsets of jobs\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n\n# toy function which fails if x is even and an input file does not exists\nfun = function(x, fn) if (x \\%\\% 2 == 0 && !file.exists(fn)) stop(\"file not found\") else x\n\n# define jobs via batchMap\nfn = tempfile()\nids = batchMap(fun, 1:20, reg = tmp, fn = fn)\n\n# submit some jobs\nids = 1:10\nsubmitJobs(ids, reg = tmp)\nwaitForJobs(ids, reg = tmp)\ngetStatus(reg = tmp)\n\n# create the required file and re-submit failed jobs\nfile.create(fn)\nsubmitJobs(findErrors(ids, reg = tmp), reg = tmp)\ngetStatus(reg = tmp)\n\n# submit remaining jobs which have not yet been submitted\nids = findNotSubmitted(reg = tmp)\nsubmitJobs(ids, reg = tmp)\ngetStatus(reg = tmp)\n\n# collect results\nreduceResultsList(reg = tmp)\n\n### Example 2: Using memory measurement\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n\n# Toy function which creates a large matrix and returns the column sums\nfun = function(n, p) colMeans(matrix(runif(n*p), n, p))\n\n# Arguments to fun:\nargs = data.table::CJ(n = c(1e4, 1e5), p = c(10, 50)) # like expand.grid()\nprint(args)\n\n# Map function to create jobs\nids = batchMap(fun, args = args, reg = tmp)\n\n# Set resources: enable memory measurement\nres = list(measure.memory = TRUE)\n\n# Submit jobs using the currently configured cluster functions\nsubmitJobs(ids, resources = res, reg = tmp)\n\n# Retrive information about memory, combine with parameters\ninfo = ijoin(getJobStatus(reg = tmp)[, .(job.id, mem.used)], getJobPars(reg = tmp))\nprint(unwrap(info))\n\n# Combine job info with results -> each job is aggregated using mean()\nunwrap(ijoin(info, reduceResultsDataTable(fun = function(res) list(res = mean(res)), reg = tmp)))\n\n### Example 3: Multicore execution on the slave\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\n\n# Function which sleeps 10 seconds, i-times\nf = function(i) {\n parallelMap::parallelMap(Sys.sleep, rep(10, i))\n}\n\n# Create one job with parameter i=4\nids = batchMap(f, i = 4, reg = tmp)\n\n# Set resources: Use parallelMap in multicore mode with 4 CPUs\n# batchtools internally loads the namespace of parallelMap and then\n# calls parallelStart() before the job and parallelStop() right\n# after the job last job in the chunk terminated.\nres = list(pm.backend = \"multicore\", ncpus = 4)\n\n\\dontrun{\n# Submit both jobs and wait for them\nsubmitJobs(resources = res, reg = tmp)\nwaitForJobs(reg = tmp)\n\n# If successfull, the running time should be ~10s\ngetJobTable(reg = tmp)[, .(job.id, time.running)]\n\n# There should also be a note in the log:\ngrepLogs(pattern = \"parallelMap\", reg = tmp)\n}\n}\n" }, { "path": "man/summarizeExperiments.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/summarizeExperiments.R\n\\name{summarizeExperiments}\n\\alias{summarizeExperiments}\n\\title{Quick Summary over Experiments}\n\\usage{\nsummarizeExperiments(\n ids = NULL,\n by = c(\"problem\", \"algorithm\"),\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to all jobs.\nInvalid ids are ignored.}\n\n\\item{by}{[\\code{character}]\\cr\nSplit the resulting table by columns of \\code{\\link{getJobPars}}.}\n\n\\item{reg}{[\\code{\\link{ExperimentRegistry}}]\\cr\nRegistry. If not explicitly passed, uses the last created registry.}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}] of frequencies.\n}\n\\description{\nReturns a frequency table of defined experiments.\nSee \\code{\\link{ExperimentRegistry}} for an example.\n}\n\\seealso{\nOther Experiment: \n\\code{\\link{addExperiments}()},\n\\code{\\link{removeExperiments}()}\n}\n\\concept{Experiment}\n" }, { "path": "man/sweepRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/sweepRegistry.R\n\\name{sweepRegistry}\n\\alias{sweepRegistry}\n\\title{Check Consistency and Remove Obsolete Information}\n\\usage{\nsweepRegistry(reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\description{\nCanceled jobs and jobs submitted multiple times may leave stray files behind.\nThis function checks the registry for consistency and removes obsolete files\nand redundant data base entries.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{syncRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/syncRegistry.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/syncRegistry.R\n\\name{syncRegistry}\n\\alias{syncRegistry}\n\\title{Synchronize the Registry}\n\\usage{\nsyncRegistry(reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{logical(1)}]: \\code{TRUE} if the state has changed, \\code{FALSE} otherwise.\n}\n\\description{\nParses update files written by the slaves to the file system and updates the\ninternal data base.\n}\n\\seealso{\nOther Registry: \n\\code{\\link{clearRegistry}()},\n\\code{\\link{getDefaultRegistry}()},\n\\code{\\link{loadRegistry}()},\n\\code{\\link{makeRegistry}()},\n\\code{\\link{removeRegistry}()},\n\\code{\\link{saveRegistry}()},\n\\code{\\link{sweepRegistry}()}\n}\n\\concept{Registry}\n" }, { "path": "man/testJob.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/testJob.R\n\\name{testJob}\n\\alias{testJob}\n\\title{Run Jobs Interactively}\n\\usage{\ntestJob(id, external = FALSE, reg = getDefaultRegistry())\n}\n\\arguments{\n\\item{id}{[\\code{integer(1)} or \\code{data.table}]\\cr\nSingle integer to specify the job or a \\code{data.table} with column \\code{job.id}\nand exactly one row.}\n\n\\item{external}{[\\code{logical(1)}]\\cr\n Run the job in an external R session? If \\code{TRUE}, starts a fresh R\n session on the local machine to execute the with \\code{\\link{execJob}}.\n You will not be able to use debug tools like \\code{\\link[base]{traceback}}\n or \\code{\\link[base]{browser}}.\n\n If \\code{external} is set to \\code{FALSE} (default) on the other hand,\n \\code{testJob} will execute the job in the current R session and the usual\n debugging tools work. However, spotting missing variable declarations (as they\n are possibly resolved in the global environment) is impossible.\n Same holds for missing package dependency declarations.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\nReturns the result of the job if successful.\n}\n\\description{\nStarts a single job on the local machine.\n}\n\\examples{\n\\dontshow{ batchtools:::example_push_temp(1) }\ntmp = makeRegistry(file.dir = NA, make.default = FALSE)\nbatchMap(function(x) if (x == 2) xxx else x, 1:2, reg = tmp)\ntestJob(1, reg = tmp)\n\\dontrun{\ntestJob(2, reg = tmp)\n}\n}\n\\seealso{\nOther debug: \n\\code{\\link{getErrorMessages}()},\n\\code{\\link{getStatus}()},\n\\code{\\link{grepLogs}()},\n\\code{\\link{killJobs}()},\n\\code{\\link{resetJobs}()},\n\\code{\\link{showLog}()}\n}\n\\concept{debug}\n" }, { "path": "man/unwrap.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/unwrap.R\n\\name{unwrap}\n\\alias{unwrap}\n\\alias{flatten}\n\\title{Unwrap Nested Data Frames}\n\\usage{\nunwrap(x, cols = NULL, sep = NULL)\n\nflatten(x, cols = NULL, sep = NULL)\n}\n\\arguments{\n\\item{x}{[\\code{\\link{data.frame}} | \\code{\\link[data.table]{data.table}}]\\cr\nData frame to flatten.}\n\n\\item{cols}{[\\code{character}]\\cr\nColumns to consider for this operation. If set to \\code{NULL} (default),\nwill operate on all columns of type \\dQuote{list}.}\n\n\\item{sep}{[\\code{character(1)}]\\cr\nIf \\code{NULL} (default), the column names of the additional columns will re-use the names\nof the nested \\code{list}/\\code{data.frame}.\nThis may lead to name clashes.\nIf you provide \\code{sep}, the variable column name will be constructed as\n\\dQuote{[column name of x][sep][inner name]}.}\n}\n\\value{\n[\\code{\\link[data.table]{data.table}}].\n}\n\\description{\nSome functions (e.g., \\code{\\link{getJobPars}}, \\code{\\link{getJobResources}} or \\code{\\link{reduceResultsDataTable}}\nreturn a \\code{data.table} with columns of type \\code{list}.\nThese columns can be unnested/unwrapped with this function.\nThe contents of these columns will be transformed to a \\code{data.table} and \\code{\\link[base]{cbind}}-ed\nto the input data.frame \\code{x}, replacing the original nested column.\n}\n\\note{\nThere is a name clash with function \\code{flatten} in package \\pkg{purrr}.\nThe function \\code{flatten} is discouraged to use for this reason in favor of \\code{unwrap}.\n}\n\\examples{\nx = data.table::data.table(\n id = 1:3,\n values = list(list(a = 1, b = 3), list(a = 2, b = 2), list(a = 3))\n)\nunwrap(x)\nunwrap(x, sep = \".\")\n}\n" }, { "path": "man/waitForJobs.Rd", "content": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/waitForJobs.R\n\\name{waitForJobs}\n\\alias{waitForJobs}\n\\title{Wait for Termination of Jobs}\n\\usage{\nwaitForJobs(\n ids = NULL,\n sleep = NULL,\n timeout = 604800,\n expire.after = NULL,\n stop.on.error = FALSE,\n stop.on.expire = FALSE,\n reg = getDefaultRegistry()\n)\n}\n\\arguments{\n\\item{ids}{[\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\nA \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\nwith a column named \\dQuote{job.id}.\nAlternatively, you may also pass a vector of integerish job ids.\nIf not set, defaults to the return value of \\code{\\link{findSubmitted}}.\nInvalid ids are ignored.}\n\n\\item{sleep}{[\\code{function(i)} | \\code{numeric(1)}]\\cr\nParameter to control the duration to sleep between queries.\nYou can pass an absolute numeric value in seconds or a \\code{function(i)} which returns\nthe number of seconds to sleep in the \\code{i}-th iteration.\nIf not provided (\\code{NULL}), tries to read the value (number/function) from the configuration file\n(stored in \\code{reg$sleep}) or defaults to a function with exponential backoff between\n5 and 120 seconds.}\n\n\\item{timeout}{[\\code{numeric(1)}]\\cr\nAfter waiting \\code{timeout} seconds, show a message and return\n\\code{FALSE}. This argument may be required on some systems where, e.g.,\nexpired jobs or jobs on hold are problematic to detect. If you don't want\na timeout, set this to \\code{Inf}. Default is \\code{604800} (one week).}\n\n\\item{expire.after}{[\\code{integer(1)}]\\cr\nJobs count as \\dQuote{expired} if they are not found on the system but have not communicated back\ntheir results (or error message). This frequently happens on managed system if the scheduler kills\na job because the job has hit the walltime or request more memory than reserved.\nOn the other hand, network file systems often require several seconds for new files to be found,\nwhich can lead to false positives in the detection heuristic.\n\\code{waitForJobs} treats such jobs as expired after they have not been detected on the system\nfor \\code{expire.after} iterations.\nIf not provided (\\code{NULL}), tries to read the value from the configuration file (stored in \\code{reg$expire.after}),\nand finally defaults to \\code{3}.}\n\n\\item{stop.on.error}{[\\code{logical(1)}]\\cr\nImmediately cancel if a job terminates with an error? Default is\n\\code{FALSE}.}\n\n\\item{stop.on.expire}{[\\code{logical(1)}]\\cr\nImmediately cancel if jobs are detected to be expired? Default is \\code{FALSE}.\nExpired jobs will then be ignored for the remainder of \\code{waitForJobs()}.}\n\n\\item{reg}{[\\code{\\link{Registry}}]\\cr\nRegistry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).}\n}\n\\value{\n[\\code{logical(1)}]. Returns \\code{TRUE} if all jobs terminated\n successfully and \\code{FALSE} if either the timeout is reached or at least\n one job terminated with an exception or expired.\n}\n\\description{\nThis function simply waits until all jobs are terminated.\n}\n" }, { "path": "man-roxygen/expreg.R", "content": "#' @param reg [\\code{\\link{ExperimentRegistry}}]\\cr\n#' Registry. If not explicitly passed, uses the last created registry.\n" }, { "path": "man-roxygen/id.R", "content": "#' @param id [\\code{integer(1)} or \\code{data.table}]\\cr\n#' Single integer to specify the job or a \\code{data.table} with column \\code{job.id}\n#' and exactly one row.\n" }, { "path": "man-roxygen/ids.R", "content": "#' @param ids [\\code{\\link[base]{data.frame}} or \\code{integer}]\\cr\n#' A \\code{\\link[base]{data.frame}} (or \\code{\\link[data.table]{data.table}})\n#' with a column named \\dQuote{job.id}.\n#' Alternatively, you may also pass a vector of integerish job ids.\n#' If not set, defaults to <%= switch(ids.default, \"all\" = \"all jobs\", \"none\" = \"no job\", sprintf(\"the return value of \\\\code{\\\\link{%s}}\", ids.default)) %>.\n#' Invalid ids are ignored.\n" }, { "path": "man-roxygen/missing.val.R", "content": "#' @param missing.val [\\code{ANY}]\\cr\n#' Value to impute as result for a job which is not finished.\n#' If not provided and a result is missing, an exception is raised.\n" }, { "path": "man-roxygen/more.args.R", "content": "#' @param more.args [\\code{list}]\\cr\n#' A list of further arguments passed to \\code{fun}.\n#' Default is an empty list.\n" }, { "path": "man-roxygen/ncpus.R", "content": "#' @param ncpus [\\code{integer(1)}]\\cr\n#' Number of CPUs.\n#' Default is to use all logical cores. The total number of cores \"available\" can be set via the option \\code{mc.cores}\n#' and defaults to the heuristic implemented in \\code{\\link[parallel]{detectCores}}.\n" }, { "path": "man-roxygen/nodename.R", "content": "#' @param nodename [\\code{character(1)}]\\cr\n#' Nodename of the master host. All commands are send via SSH to this host. Only works iff\n#' \\enumerate{\n#' \\item{Passwordless authentication (e.g., via SSH public key authentication) is set up.}\n#' \\item{The file directory is shared across machines, e.g. mounted via SSHFS.}\n#' \\item{Either the absolute path to the \\code{file.dir} is identical on the machines, or paths are provided relative to the home directory. Symbolic links should work.}\n#' }\n" }, { "path": "man-roxygen/reg.R", "content": "#' @param reg [\\code{\\link{Registry}}]\\cr\n#' Registry. If not explicitly passed, uses the default registry (see \\code{\\link{setDefaultRegistry}}).\n" }, { "path": "man-roxygen/template.R", "content": "#' @param template [\\code{character(1)}]\\cr\n#' Either a path to a \\pkg{brew} template file (with extension \\dQuote{tmpl}), or a short descriptive name enabling the following heuristic for the file lookup:\n#' \\enumerate{\n#' \\item \\dQuote{batchtools.[template].tmpl} in the path specified by the environment variable \\dQuote{R_BATCHTOOLS_SEARCH_PATH}.\n#' \\item \\dQuote{batchtools.[template].tmpl} in the current working directory.\n#' \\item \\dQuote{[template].tmpl} in the user config directory (see \\code{\\link[rappdirs]{user_config_dir}}); on linux this is usually \\dQuote{~/.config/batchtools/[template].tmpl}.\n#' \\item \\dQuote{.batchtools.[template].tmpl} in the home directory.\n#' \\item \\dQuote{[template].tmpl} in the package installation directory in the subfolder \\dQuote{templates}.\n#' }\n" }, { "path": "paper/codemeta.json", "content": "{\n \"@context\": \"https://raw.githubusercontent.com/mbjones/codemeta/master/codemeta.jsonld\",\n \"@type\": \"Code\",\n \"author\": [\n {\n \"@id\": \"https://orcid.org/0000-0001-9754-0393\",\n \"@type\": \"Person\",\n \"email\": \"lang@statistik.tu-dortmund.de\",\n \"name\": \"Michel Lang\",\n \"affiliation\": \"TU Dortmund University\"\n },\n {\n \"@id\": \"https://orcid.org/0000-0001-6002-6980\",\n \"@type\": \"Person\",\n \"email\": \"bernd.bischl@stat.uni-muenchen.de\",\n \"name\": \"Bernd Bischl\",\n \"affiliation\": \"LMU Munich\"\n },\n {\n \"@id\": \"https://orcid.org/0000-0003-0873-137X\",\n \"@type\": \"Person\",\n \"email\": \"surmann@statistik.tu-dortmund.de\",\n \"name\": \"Dirk Surmann\",\n \"affiliation\": \"TU Dortmund University\"\n }\n ],\n \"identifier\": \"https://doi.org/10.5281/zenodo.165184\",\n \"codeRepository\": \"https://github.com/mllg/batchtools\",\n \"datePublished\": \"2016-11-10\",\n \"dateModified\": \"2016-11-10\",\n \"dateCreated\": \"2016-11-10\",\n \"description\": \"Tools for computation on batch systems\",\n \"keywords\": \"r, parallelization, high-performance computing, batch systems\",\n \"license\": \"GPL v3.0\",\n \"title\": \"batchtools\",\n \"version\": \"v0.9.0\"\n}\n" }, { "path": "paper/paper.bib", "content": "@Article{batchjobs_2015,\n title = {{BatchJobs} and {BatchExperiments}: Abstraction Mechanisms for Using {R} in Batch Environments},\n author = {Bernd Bischl and Michel Lang and Olaf Mersmann and J{\\\"o}rg Rahnenf{\\\"u}hrer and Claus Weihs},\n journal = {Journal of Statistical Software},\n year = {2015},\n volume = {64},\n number = {11},\n pages = {1--25},\n url = {https://www.jstatsoft.org/v64/i11/},\n doi = {10.18637/jss.v064.i11},\n}\n\n@Manual{R,\n title = {R: A Language and Environment for Statistical Computing},\n author = {{R Core Team}},\n organization = {R Foundation for Statistical Computing},\n address = {Vienna, Austria},\n year = {2016},\n url = {https://www.R-project.org/},\n}\n\n@Manual{snow,\n title = {snow: Simple Network of Workstations},\n author = {Luke Tierney and A. J. Rossini and Na Li and H. Sevcikova},\n year = {2016},\n note = {R package version 0.4-2},\n url = {https://CRAN.R-project.org/package=snow},\n}\n\n@Manual{data_table,\n title = {data.table: Extension of Data.frame},\n author = {M Dowle and A Srinivasan and T Short and S Lianoglou with contributions from R Saporta and E Antonyan},\n year = {2015},\n note = {R package version 1.9.6},\n url = {https://CRAN.R-project.org/package=data.table},\n}\n" }, { "path": "paper/paper.md", "content": "---\ntitle: 'batchtools: Tools for R to work on batch systems'\ntags:\n- R\n - high-performance computing\n - batch systems\n - parallelization\nauthors:\n - name: Michel Lang\n orcid: 0000-0001-9754-0393\n affiliation: 1\n - name: Bernd Bischl\n orcid: 0000-0001-6002-6980\n affiliation: 2\n - name: Dirk Surmann\n orcid: 0000-0003-0873-137X\n affiliation: 1\naffiliations:\n - name: TU Dortmund University\n index: 1\n - name: LMU Munich\n index: 2\ndate: 9 November 2016\nbibliography: paper.bib\n---\n\n# Summary\n\nThe [`R`](https://www.r-project.org/) [@R] package [`batchtools`](https://github.com/mllg/batchtools) is the successor of the [`BatchJobs`](https://github.com/tudo-r/BatchJobs) package [@batchjobs_2015].\nIt provides an implementation of a Map-like operation to define and asynchronously execute jobs on a variety of parallel backends:\n\n* Local (blocking) execution in the current `R` session or in an externally spawned `R` process (intended for debugging and prototyping)\n* Local (non-blocking) parallel execution using `parallel`'s multicore backend [@R] or [`snow`](https://cran.r-project.org/package=snow)'s socket mode [@snow].\n* Execution on loosely connected machines using SSH (including basic resource usage control).\n* [Docker Swarm](https://docs.docker.com/engine/swarm/)\n* [IBM Spectrum LSF](https://www.ibm.com/products/hpc-workload-management)\n* [OpenLava](https://www.openlava.org/)\n* [Univa Grid Engine](https://www.univa.com/) (formerly Oracle Grind Engine and Sun Grid Engine)\n* [Slurm Workload Manager](https://slurm.schedmd.com/)\n* [TORQUE/PBS Resource Manager](https://adaptivecomputing.com/cherry-services/moab-hpc/)\n\nExtensibility and user customization are important features as configuration on high-performance computing clusters is often heavily tailored towards very specific requirements or special hardware.\nHence, the interaction with the schedulers uses a template engine for improved flexibility.\nFurthermore, custom functions can be hooked into the package to be called at certain events.\nAs a last resort, many utility functions simplify the implementation of a custom cluster backend from scratch.\n\nThe communication between the master `R` session and the computational nodes is kept as simple as possible and runs completely on the file system which greatly simplifies the extension to additional parallel platforms.\nThe [`data.table`](https://github.com/Rdatatable/data.table) package [@data_table] acts as an in-memory database to keep track of the computational status of all jobs.\nUnique job seeds ensure reproducibility across systems, log files can conveniently be searched using regular expressions and jobs can be annotated with arbitrary tags.\nJobs can be chunked (i.e., merged into one technical cluster job) to be executed as one virtual job on a node (executed sequentially or using multiple local CPUs) in order to reduce the overhead induced by job management and starting/stopping `R`.\nAll in all, the provided tools allow users to work with many thousands or even millions of jobs in an organized and efficient manner.\n\nThe `batchtools` package also comes with an abstraction mechanism to assist in conducting large-scale computer experiments, especially suited for (but not restricted to) benchmarking and exploration of algorithm performance.\nThe mechanism is similar to [`BatchExperiments`](https://github.com/tudo-r/BatchExperiments) [@batchjobs_2015] which `batchtools` now also supersedes:\nAfter defining the building blocks of most computer experiments, problems and algorithms, both can be parametrized to define jobs which are then in a second step submitted to one of the parallel backends.\n\nImportant changes to its predecessors are summarized in a vignette to help users of [`BatchJobs`](https://github.com/tudo-r/BatchJobs)/[`BatchExperiments`](https://github.com/tudo-r/BatchExperiments) migrating their cluster configuration and aid the transition to `batchtools`.\n\n\n# References\n\n" }, { "path": "src/Makevars", "content": "PKG_CFLAGS=${R_DEBUG_FLAGS}\n" }, { "path": "src/binpack.c", "content": "#include \n#include \n#include \n#include \n\nSEXP attribute_hidden c_binpack(SEXP x_, SEXP order_, SEXP capacity_) {\n const double * x = REAL(x_);\n const R_len_t n = length(x_);\n const int * order = INTEGER(order_);\n const double capacity = REAL(capacity_)[0];\n R_len_t ii = order[0] - 1;\n if (x[ii] > capacity)\n error(\"Capacity not sufficient. Largest item does not fit.\");\n\n SEXP res = PROTECT(allocVector(INTSXP, n));\n int * bin = INTEGER(res);\n double * capacities = malloc(n * sizeof(double));\n\n R_len_t bins = 1;\n bin[ii] = 1;\n capacities[0] = capacity - x[ii];\n\n for (R_len_t i = 1; i < n; i++) {\n ii = order[i] - 1;\n bool packed = false;\n for (R_len_t pos = 0; !packed && pos < bins; pos++) {\n if (capacities[pos] >= x[ii]) {\n packed = true;\n bin[ii] = pos + 1;\n capacities[pos] -= x[ii];\n break;\n }\n }\n\n if (!packed) {\n capacities[bins] = capacity - x[ii];\n bins++;\n bin[ii] = bins;\n }\n }\n\n free(capacities);\n UNPROTECT(1);\n return res;\n}\n" }, { "path": "src/count_not_missing.c", "content": "#include \n#include \n#include \n\nstatic R_len_t count_not_missing_logical(SEXP x) {\n const int * xp = LOGICAL(x);\n const int * const xe = xp + length(x);\n R_len_t count = 0;\n for (; xp != xe; xp++) {\n if (*xp != NA_LOGICAL)\n count++;\n }\n return count;\n}\n\nstatic R_len_t count_not_missing_integer(SEXP x) {\n const int * xp = INTEGER(x);\n const int * const xe = xp + length(x);\n R_len_t count = 0;\n for (; xp != xe; xp++) {\n if (*xp != NA_INTEGER)\n count++;\n }\n return count;\n}\n\nstatic R_len_t count_not_missing_double(SEXP x) {\n const double * xp = REAL(x);\n const double * const xe = xp + length(x);\n R_len_t count = 0;\n for (; xp != xe; xp++) {\n if (!ISNAN(*xp))\n count++;\n }\n return count;\n}\n\nstatic R_len_t count_not_missing_string(SEXP x) {\n const R_len_t nx = length(x);\n R_len_t count = 0;\n for (R_len_t i = 0; i < nx; i++) {\n if (STRING_ELT(x, i) != NA_STRING)\n count++;\n }\n return count;\n}\n\nstatic R_len_t count_not_missing_list(SEXP x) {\n const R_len_t nx = length(x);\n R_len_t count = 0;\n for (R_len_t i = 0; i < nx; i++) {\n if (!isNull(VECTOR_ELT(x, i)))\n count++;\n }\n return count;\n}\n\nSEXP attribute_hidden count_not_missing(SEXP x) {\n switch(TYPEOF(x)) {\n case LGLSXP: return ScalarInteger(count_not_missing_logical(x));\n case INTSXP: return ScalarInteger(count_not_missing_integer(x));\n case REALSXP: return ScalarInteger(count_not_missing_double(x));\n case STRSXP: return ScalarInteger(count_not_missing_string(x));\n case VECSXP: return ScalarInteger(count_not_missing_list(x));\n case NILSXP: return ScalarInteger(0);\n default: error(\"Object of type '%s' not supported\", type2char(TYPEOF(x)));\n }\n}\n" }, { "path": "src/fill_gaps.c", "content": "#include \n#include \n#include \n\n/* similar to last observation carried forward, but resets to NA if the last observation is spotted again */\n/* used in log file reading: jobs have a start and stop marker, the lines in between belong to the job */\nSEXP attribute_hidden fill_gaps(SEXP x) {\n const R_len_t n = length(x);\n int last = NA_INTEGER;\n\n const int *xi = INTEGER(x);\n const int * const xend = xi + n;\n\n SEXP y = PROTECT(allocVector(INTSXP, n));\n int *yi = INTEGER(y);\n\n\n for(; xi != xend; xi++, yi++) {\n if (*xi == NA_INTEGER) {\n *yi = last;\n } else {\n *yi = *xi;\n last = (*xi == last) ? NA_INTEGER : *xi;\n }\n }\n\n UNPROTECT(1);\n return y;\n}\n" }, { "path": "src/init.c", "content": "#include \n#include \n#include // for NULL\n#include \n\n/* .Call calls */\nextern SEXP c_binpack(SEXP, SEXP, SEXP);\nextern SEXP c_lpt(SEXP, SEXP, SEXP);\nextern SEXP count_not_missing(SEXP);\nextern SEXP fill_gaps(SEXP);\n\nstatic const R_CallMethodDef CallEntries[] = {\n {\"c_binpack\", (DL_FUNC) &c_binpack, 3},\n {\"c_lpt\", (DL_FUNC) &c_lpt, 3},\n {\"count_not_missing\", (DL_FUNC) &count_not_missing, 1},\n {\"fill_gaps\", (DL_FUNC) &fill_gaps, 1},\n {NULL, NULL, 0}\n};\n\nvoid R_init_batchtools(DllInfo *dll) {\n R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);\n R_useDynamicSymbols(dll, FALSE);\n}\n" }, { "path": "src/lpt.c", "content": "#include \n#include \n#include \n#define min(a, b) (((a) < (b)) ? (a) : (b))\n\nSEXP attribute_hidden c_lpt(SEXP x_, SEXP order_, SEXP chunks_) {\n const double * x = REAL(x_);\n const R_len_t n = length(x_);\n const int * order = INTEGER(order_);\n const int chunks = min(INTEGER(chunks_)[0], n);\n\n SEXP res = PROTECT(allocVector(INTSXP, n));\n int * bin = INTEGER(res);\n double * sums = malloc(chunks * sizeof(double));\n\n for (R_len_t i = 0; i < chunks; i++) {\n R_len_t ii = order[i] - 1;\n bin[ii] = i + 1;\n sums[i] = x[ii];\n }\n\n for (R_len_t i = chunks; i < n; i++) {\n R_len_t ii = order[i] - 1;\n R_len_t pos = 0;\n for (R_len_t j = 1; j < chunks; j++) {\n if (sums[j] < sums[pos])\n pos = j;\n }\n\n bin[ii] = pos + 1;\n sums[pos] += x[ii];\n }\n\n\n free(sums);\n UNPROTECT(1);\n return res;\n}\n" }, { "path": "tests/testthat/helper.R", "content": "library(\"testthat\")\nlibrary(\"data.table\")\nlibrary(\"checkmate\")\nlibrary(\"stringi\")\nrequireNamespace(\"withr\")\n\noptions(datatable.rbindlist.check=\"error\")\n\nis_on_ci = function() {\n identical(Sys.getenv(\"APPVEYOR\"), \"True\") || identical(Sys.getenv(\"TRAVIS\"), \"true\")\n}\n\ngetSysConf = function() {\n conf.file = findConfFile()\n if (!checkmate::testScalarNA(conf.file)) {\n ee = new.env()\n sys.source(conf.file, envir = ee)\n as.list(ee)\n } else {\n list()\n }\n}\n\nmakeTestRegistry = function(file.dir = NA, make.default = FALSE, ...) {\n reg = makeRegistry(file.dir = file.dir, make.default = make.default, ...)\n # cleanup registry directories if not a subdirectory of R's temp dir\n if ((is.na(file.dir) && !identical(reg$temp.dir, fs::path_temp())))\n reg.finalizer(e = reg, f = function(reg) if (fs::dir_exists(reg$file.dir)) fs::dir_delete(reg$file.dir), onexit = TRUE)\n return(reg)\n}\n\nmakeTestExperimentRegistry = function(file.dir = NA, make.default = FALSE, ...) {\n reg = makeExperimentRegistry(file.dir = file.dir, make.default = make.default, ...)\n # cleanup registry directories if not a subdirectory of R's temp dir\n if ((is.na(file.dir) && !identical(reg$temp.dir, fs::path_temp())))\n reg.finalizer(e = reg, f = function(reg) if (fs::dir_exists(reg$file.dir)) fs::dir_delete(reg$file.dir), onexit = TRUE)\n return(reg)\n}\n\nsilent = function(expr) {\n withr::with_options(list(batchtools.progress = FALSE, batchtools.verbose = FALSE), expr)\n}\n\ns.chunk = function(ids) {\n ids$chunk = 1L\n ids\n}\n\nsubmitAndWait = function(reg, ids = NULL, ..., sleep = 1) {\n ids = if (is.null(ids)) findNotSubmitted(reg = reg) else convertIds(reg, ids, keep.extra = names(ids))\n if (\"chunk\" %chnin% names(ids))\n ids = s.chunk(ids)\n silent({\n ids = submitJobs(ids = ids, ..., reg = reg)\n waitForJobs(ids, expire.after = 10L, reg = reg, sleep = sleep)\n })\n}\n\nsuppressAll = function (expr) {\n silent(capture.output({z = suppressWarnings(suppressMessages(suppressPackageStartupMessages(force(expr))))}))\n invisible(z)\n}\n\ncheckTables = function(reg, ...) {\n checkmate::expect_string(reg$hash)\n checkmate::expect_posixct(reg$mtime, len = 1L)\n\n if (class(reg)[1L] == \"Registry\") {\n cols = c(\"def.id\", \"job.pars\")\n types = c(\"integer\", \"list\")\n } else {\n cols = c(\"def.id\", \"problem\", \"prob.pars\", \"algorithm\", \"algo.pars\", \"pars.hash\")\n types = c(\"integer\", \"character\", \"list\", \"character\", \"list\", \"character\")\n }\n expect_is(reg$defs, \"data.table\")\n checkmate::expect_data_table(reg$defs, ncols = length(cols), ...)\n checkmate::expect_set_equal(colnames(reg$defs), cols)\n expect_equal(as.character(reg$defs[, lapply(.SD, class), .SDcols = cols]), types)\n expect_equal(key(reg$defs), \"def.id\")\n expect_equal(anyDuplicated(reg$defs, by = \"def.id\"), 0L)\n\n if (class(reg)[1L] == \"Registry\") {\n cols = c(\"job.id\", \"def.id\", \"submitted\", \"started\", \"done\", \"error\", \"mem.used\", \"resource.id\", \"batch.id\", \"log.file\", \"job.hash\", \"job.name\")\n types = c(\"integer\", \"integer\", \"numeric\", \"numeric\", \"numeric\", \"character\", \"numeric\", \"integer\", \"character\", \"character\", \"character\", \"character\")\n } else {\n cols = c(\"job.id\", \"def.id\", \"submitted\", \"started\", \"done\", \"error\", \"mem.used\", \"resource.id\", \"batch.id\", \"log.file\", \"job.hash\", \"job.name\", \"repl\")\n types = c(\"integer\", \"integer\", \"numeric\", \"numeric\", \"numeric\", \"character\", \"numeric\", \"integer\", \"character\", \"character\", \"character\", \"character\", \"integer\")\n }\n expect_is(reg$status, \"data.table\")\n checkmate::expect_data_table(reg$status, ncols = length(cols), ...)\n checkmate::expect_set_equal(colnames(reg$status), cols)\n expect_equal(as.character(reg$status[, lapply(.SD, class), .SDcols = cols]), types)\n expect_equal(key(reg$status), \"job.id\")\n expect_equal(anyDuplicated(reg$status, by = \"job.id\"), 0L)\n checkStatusIntegrity(reg)\n\n cols = c(\"resource.id\", \"resource.hash\", \"resources\")\n types = c(\"integer\", \"character\", \"list\")\n checkmate::expect_data_table(reg$resources, ncols = length(cols), ...)\n checkmate::expect_set_equal(colnames(reg$resources), cols)\n expect_equal(as.character(reg$resources[, lapply(.SD, class), .SDcols = cols]), types)\n expect_equal(key(reg$resources), \"resource.id\")\n expect_equal(anyDuplicated(reg$resources, by = \"resource.id\"), 0L)\n\n cols = c(\"job.id\", \"tag\")\n types = c(\"integer\", \"character\")\n checkmate::expect_data_table(reg$tags, ncols = length(cols), ...)\n checkmate::expect_set_equal(colnames(reg$tags), cols)\n expect_equal(as.character(reg$tags[, lapply(.SD, class), .SDcols = cols]), types)\n expect_equal(key(reg$tags), \"job.id\")\n\n if (class(reg)[1L] == \"ExperimentRegistry\") {\n checkmate::expect_character(reg$problems, any.missing = FALSE, unique = TRUE)\n checkmate::expect_character(reg$algorithms, any.missing = FALSE, unique = TRUE)\n checkmate::expect_integer(reg$status$repl, lower = 1L, any.missing = FALSE)\n checkmate::expect_subset(reg$defs$problem, reg$problems)\n checkmate::expect_subset(reg$defs$algorithm, reg$algorithms)\n }\n\n expect_key_set_equal(reg$defs, reg$status, by = \"def.id\")\n expect_key_set_equal(reg$status[!is.na(resource.id)], reg$resources, by = \"resource.id\")\n if (nrow(reg$status) > 0L)\n checkmate::expect_data_table(ajoin(reg$tags, reg$status, by = \"job.id\"), nrow = 0)\n else\n expect_equal(nrow(reg$tags), 0)\n}\n\ncheckStatusIntegrity = function(reg) {\n tab = reg$status[, list(job.id, code = (!is.na(submitted)) + 2L * (!is.na(started)) + 4L * (!is.na(done)) + 8L * (!is.na(error)))]\n\n # submitted started done error\n # 2^0 2^1 2^2 2^3\n # 1 2 4 8\n # ------------------------------------------------------\n # 0 0 0 0 -> 0 (unsubmitted)\n # 1 0 0 0 -> 1 (submitted)\n # 1 1 0 0 -> 3 (started)\n # 1 1 1 0 -> 7 (done)\n # 1 1 1 1 -> 15 (error)\n\n checkmate::expect_subset(tab$code, c(0L, 1L, 3L, 7L, 15L), info = \"Status Integrity\")\n}\n\nexpect_copied = function(x, y) {\n expect_false(data.table:::address(x) == data.table:::address(y))\n}\n\nexpect_key_set_equal = function(x, y, by = NULL) {\n expect_true(nrow(ajoin(x, y, by = by)) == 0 && nrow(ajoin(y, x, by = by)) == 0)\n}\n" }, { "path": "tests/testthat/test_Algorithm.R", "content": "test_that(\"addAlgorithm\", {\n reg = makeTestExperimentRegistry()\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) NULL)\n expect_is(algo, \"Algorithm\")\n expect_equal(algo$name, \"a1\")\n expect_function(algo$fun)\n expect_file_exists(getAlgorithmURI(reg, algo$name))\n\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) nrow(data))\n algo = addAlgorithm(reg = reg, \"a2\", fun = function(...) NULL)\n ids = addExperiments(list(p1 = data.table()), algo.designs = list(a1 = data.table(), a2 = data.table()), repls = 2, reg = reg)\n expect_integer(ids$job.id, len = 4L)\n\n removeAlgorithms(reg = reg, \"a1\")\n expect_integer(reg$status$job.id, len = 2L)\n expect_set_equal(reg$algorithms, \"a2\")\n expect_set_equal(reg$algorithms, \"a2\")\n expect_false(fs::file_exists(getAlgorithmURI(reg, \"a1\")))\n expect_true(fs::file_exists(getAlgorithmURI(reg, \"a2\")))\n expect_set_equal(getJobPars(reg = reg)$algorithm, \"a2\")\n checkTables(reg)\n})\n\n\ntest_that(\"addAlgorithm overwrites old algo\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) 2)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) instance * 2)\n ids = addExperiments(list(p1 = data.table()), list(a1 = data.table()), reg = reg)\n run = function(id) suppressAll(execJob(makeJob(id, reg = reg)))\n\n expect_equal(run(1), 4)\n\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) 4)\n expect_equal(run(1), 8)\n\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) instance * 8)\n expect_equal(run(1), 32)\n})\n" }, { "path": "tests/testthat/test_ClusterFunctionHyperQueue.R", "content": "test_that(\"clusterFunctionsHyperQueue\", {\n skip_if(TRUE)\n skip_on_ci()\n skip_on_cran()\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsHyperQueue()\n saveRegistry(reg)\n fun = function(x) {\n Sys.sleep(5)\n TRUE\n }\n ids = batchMap(fun, x = c(5, 5), reg = reg)\n submitJobs(1:2, reg = reg)\n waitForJobs(ids = ids, reg = reg)\n\n expect_data_table(findJobs(ids = ids, reg = reg), nrow = 2)\n expect_data_table(findRunning(reg = reg), nrow = 0L)\n})\n\ntest_that(\"clusterFunctionsHyperQueue: killJob\", {\n skip_if(TRUE)\n skip_on_ci()\n skip_on_cran()\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsHyperQueue()\n saveRegistry(reg)\n fun = function(x) { Sys.sleep(5); TRUE }\n ids = batchMap(fun, x = c(5, 5), reg = reg)\n submitJobs(1:2, reg = reg)\n Sys.sleep(1)\n expect_data_table(killJobs(1, reg = reg), nrow = 1)\n})\n\ntest_that(\"clusterFunctionsHyperQueue with resources\", {\n skip_if(TRUE)\n skip_on_ci()\n skip_on_cran()\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsHyperQueue()\n saveRegistry(reg)\n fun = function(x) {\n Sys.sleep(5)\n TRUE\n }\n ids = batchMap(fun, x = c(5, 5), reg = reg)\n submitJobs(1:2, reg = reg, resources = list(ncpus = 2, walltime = 10, memory = 5))\n waitForJobs(ids = ids, reg = reg)\n\n expect_data_table(findJobs(ids = ids, reg = reg), nrow = 2)\n expect_data_table(findRunning(reg = reg), nrow = 0L)\n})\n" }, { "path": "tests/testthat/test_ClusterFunctions.R", "content": "test_that(\"clusterFunctions constructor\", {\n check = function(cf) {\n expect_is(cf, \"ClusterFunctions\")\n expect_set_equal(names(cf), c(\"name\", \"submitJob\", \"killJob\", \"listJobsQueued\", \"listJobsRunning\",\n \"store.job.collection\", \"store.job.files\", \"array.var\", \"scheduler.latency\", \"fs.latency\", \"hooks\"))\n expect_output(print(cf), \"ClusterFunctions for mode\")\n }\n reg = makeTestRegistry()\n check(reg$cluster.functions)\n fn = fs::path(fs::path_temp(), \"dummy.tmpl\")\n writeLines(\"foo\", fn)\n check(makeClusterFunctionsInteractive())\n check(makeClusterFunctionsSGE(template = fn))\n check(makeClusterFunctionsTORQUE(template = fn))\n check(makeClusterFunctionsSlurm(template = fn))\n check(makeClusterFunctionsOpenLava(template = fn))\n check(makeClusterFunctionsLSF(template = fn))\n check(makeClusterFunctionsTORQUE(\"torque-lido\"))\n check(makeClusterFunctionsSlurm(\"slurm-dortmund\"))\n check(makeClusterFunctionsDocker(\"image\"))\n expect_error(makeClusterFunctionsLSF(), \"point to a readable template file\")\n\n skip_on_os(c(\"windows\", \"solaris\")) # system2 is broken on solaris\n check(makeClusterFunctionsSSH(workers = list(Worker$new(nodename = \"localhost\", ncpus = 1L))))\n})\n\n\ntest_that(\"submitJobResult\", {\n x = makeSubmitJobResult(0, 99)\n expect_is(x, \"SubmitJobResult\")\n expect_identical(x$status, 0L)\n expect_identical(x$batch.id, 99)\n expect_identical(x$msg, \"OK\")\n\n x = makeSubmitJobResult(1, 99)\n expect_is(x, \"SubmitJobResult\")\n expect_identical(x$msg, \"TEMPERROR\")\n\n x = makeSubmitJobResult(101, 99)\n expect_is(x, \"SubmitJobResult\")\n expect_identical(x$msg, \"ERROR\")\n\n expect_output(print(x), \"submission result\")\n\n x = cfHandleUnknownSubmitError(cmd = \"ls\", exit.code = 42L, output = \"answer to life\")\n expect_is(x, \"SubmitJobResult\")\n expect_true(all(stri_detect_fixed(x$msg, c(\"ls\", \"42\", \"answer to life\"))))\n})\n\ntest_that(\"brew\", {\n fn = fs::file_temp()\n lines = c(\"####\", \" \", \"!!!\", \"foo=<%= job.hash %>\")\n writeLines(lines, fn)\n\n res = stri_split_fixed(cfReadBrewTemplate(fn), \"\\n\")[[1]]\n assertCharacter(res, len = 3)\n expect_equal(sum(stri_detect_fixed(res, \"job.hash\")), 1)\n\n res = stri_split_fixed(cfReadBrewTemplate(fn, comment.string = \"###\"), \"\\n\")[[1]]\n assertCharacter(res, len = 2)\n expect_equal(sum(stri_detect_fixed(res, \"job.hash\")), 1)\n\n reg = makeTestRegistry()\n ids = batchMap(identity, 1:2, reg = reg)\n jc = makeJobCollection(1, reg = reg)\n text = cfReadBrewTemplate(fn, comment.string = \"###\")\n\n fn = cfBrewTemplate(text = text, jc = jc, reg = reg)\n brewed = readLines(fn)\n expect_equal(brewed[1], \"!!!\")\n expect_equal(brewed[2], sprintf(\"foo=%s\", jc$job.hash))\n fs::file_delete(fn)\n})\n\ntest_that(\"Special chars in directory names\", {\n reg = makeTestRegistry()\n base.dir = fs::file_temp(pattern = \"test\", tmp_dir = fs::path_dir(reg$file.dir))\n fs::dir_create(base.dir)\n\n file.dir = fs::path(base.dir, \"test#some_frequently-used chars\")\n reg = makeTestRegistry(file.dir = file.dir)\n batchMap(identity, 1:2, reg = reg)\n submitAndWait(reg = reg)\n Sys.sleep(0.2)\n expect_equal(reduceResultsList(reg = reg), list(1L, 2L))\n expect_equal(testJob(1, external = FALSE, reg = reg), 1L)\n})\n\ntest_that(\"Export of environment variable DEBUGME\", {\n reg = makeTestRegistry()\n if (reg$cluster.functions$name == \"Socket\")\n skip(\"Environment variables not exported for CF socket\")\n batchMap(function(i) Sys.getenv(\"DEBUGME\"), i = 1, reg = reg)\n\n withr::local_envvar(c(\"DEBUGME\" = \"grepme\"))\n submitAndWait(reg, 1)\n\n res = loadResult(1, reg = reg)\n expect_string(res, min.chars = 1, fixed = \"grepme\")\n})\n\ntest_that(\"findTemplateFile\", {\n d = fs::path_temp()\n fn = fs::path(d, \"batchtools.slurm.tmpl\")\n fs::file_create(fn)\n withr::with_envvar(list(R_BATCHTOOLS_SEARCH_PATH = d),\n expect_equal(findTemplateFile(\"slurm\"), fs::path_abs(fn))\n )\n fs::file_delete(fn)\n})\n" }, { "path": "tests/testthat/test_ClusterFunctionsMulticore.R", "content": "test_that(\"cf multicore\", {\n skip_on_os(\"windows\")\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsMulticore(2)\n ids = batchMap(Sys.sleep, time = c(2, 2), reg = reg)\n silent({\n submitJobs(1:2, reg = reg)\n expect_equal(findOnSystem(reg = reg), findJobs(reg = reg))\n expect_true(waitForJobs(sleep = 0.2, expire.after = 1, reg = reg))\n })\n expect_data_table(findOnSystem(reg = reg), nrow = 0)\n expect_equal(findDone(reg = reg), findJobs(reg = reg))\n\n\n # check that max.concurrent.jobs works\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsMulticore(2)\n reg$max.concurrent.jobs = 1\n ids = batchMap(Sys.sleep, time = c(2, 0), reg = reg)\n submitAndWait(1:2, reg = reg)\n tab = getJobStatus(reg = reg)\n expect_true(diff(tab$started) > 1)\n})\n\nif (FALSE) {\n # Multicore cleans up finished processes\n reg = makeTestRegistry()\n batchMap(Sys.sleep, rep(0.8, 8), reg = reg)\n parallel::mccollect()\n p = self = Multicore$new(4)\n\n for (i in 1:4) {\n p$spawn(makeJobCollection(i, reg = reg))\n }\n expect_data_table(p$jobs, ncol = 2)\n expect_integer(p$jobs$pid, len = 4L, any.missing = FALSE, lower = 0L)\n expect_integer(p$jobs$count, len = 4L, any.missing = FALSE, lower = 0L, upper = 1L)\n Sys.sleep(1.5)\n p$spawn(makeJobCollection(5L, reg = reg))\n expect_integer(p$jobs$pid, len = 1L, any.missing = FALSE, lower = 0L)\n p$collect(3)\n p$collect(1)\n x = parallel::mccollect()\n expect_true(is.null(x))\n}\n" }, { "path": "tests/testthat/test_ClusterFunctionsSSH.R", "content": "test_that(\"cf ssh\", {\n skip_on_os(\"windows\")\n skip_on_ci()\n skip_on_cran()\n\n reg = makeTestRegistry()\n if (reg$cluster.functions$name == \"Interactive\") {\n workers = list(Worker$new(\"localhost\", ncpus = 2, max.load = 9999))\n reg$cluster.functions = makeClusterFunctionsSSH(workers)\n saveRegistry(reg)\n fun = function(x) { Sys.sleep(x); is(x, \"numeric\") }\n ids = batchMap(fun, x = c(5, 5), reg = reg)\n silent({\n submitJobs(1:2, reg = reg)\n Sys.sleep(0.2)\n expect_equal(findOnSystem(reg = reg), findJobs(reg = reg))\n expect_true(killJobs(2, reg = reg)$killed)\n expect_true(\n waitForJobs(1, sleep = 0.5, reg = reg)\n )\n })\n expect_equal(findDone(reg = reg), findJobs(ids = 1, reg = reg))\n expect_equal(findNotDone(reg = reg), findJobs(ids = 2, reg = reg))\n expect_true(loadResult(1, reg = reg))\n }\n})\n\nif (FALSE) {\n reg = makeTestRegistry()\n workers = list(Worker$new(\"129.217.207.53\"), Worker$new(\"localhost\", ncpus = 1))\n\n reg$cluster.functions = makeClusterFunctionsSSH(workers)\n expect_string(workers[[1L]]$script)\n expect_string(workers[[2L]]$script)\n expect_equal(workers[[1L]]$ncpus, 4L)\n expect_equal(workers[[2L]]$ncpus, 1L)\n fun = function(x) { Sys.sleep(x); is(x, \"numeric\") }\n ids = batchMap(fun, x = 20 * c(1, 1), reg = reg)\n submitJobs(1:2, reg = reg)\n expect_equal(findOnSystem(reg = reg), findJobs(reg = reg))\n expect_true(killJobs(2, reg = reg)$killed)\n expect_true(waitForJobs(1, reg = reg, sleep = 1))\n expect_equal(findDone(reg = reg), findJobs(ids = 1, reg = reg))\n expect_equal(findNotDone(reg = reg), findJobs(ids = 2, reg = reg))\n expect_true(loadResult(1, reg = reg))\n}\n" }, { "path": "tests/testthat/test_ClusterFunctionsSocket.R", "content": "test_that(\"cf socket\", {\n skip_if_not_installed(\"snow\")\n # skip_on_ci()\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsSocket(2)\n ids = batchMap(Sys.sleep, time = c(5, 5), reg = reg)\n silent({\n submitJobs(1:2, reg = reg)\n expect_equal(findOnSystem(reg = reg), findJobs(reg = reg))\n expect_true(waitForJobs(sleep = 0.5, reg = reg))\n })\n expect_data_table(findOnSystem(reg = reg), nrow = 0)\n expect_equal(findDone(reg = reg), findJobs(reg = reg))\n})\n" }, { "path": "tests/testthat/test_ExperimentRegistry.R", "content": "test_that(\"makeTestExperimentRegistry\", {\n reg = makeTestExperimentRegistry()\n expect_is(reg, \"Registry\")\n expect_is(reg, \"ExperimentRegistry\")\n expect_true(is.environment(reg))\n expect_directory_exists(reg$file.dir, access = \"rw\")\n expect_directory_exists(reg$work.dir, access = \"r\")\n expect_directory_exists(fs::path(reg$file.dir, c(\"jobs\", \"results\", \"updates\", \"logs\")))\n expect_file(fs::path(reg$file.dir, \"registry.rds\"))\n expect_character(reg$packages, any.missing = FALSE)\n expect_character(reg$namespaces, any.missing = FALSE)\n expect_int(reg$seed, na.ok = FALSE)\n expect_true(reg$writeable)\n expect_is(reg$cluster.functions, \"ClusterFunctions\")\n expect_list(reg$default.resources, names = \"strict\")\n checkTables(reg, any.missing = FALSE, nrows = 0L)\n expect_character(reg$problems, len = 0L)\n expect_character(reg$algorithms, len = 0L)\n expect_output(print(reg), \"Experiment Registry\")\n})\n\ntest_that(\"Printer works (#170)\", {\n reg = makeTestExperimentRegistry()\n expect_character(reg$problems, len = 0L)\n expect_character(reg$algorithms, len = 0L)\n expect_output(print(reg), \"Problems[[:space:]]*:[[:space:]]*0\")\n expect_output(print(reg), \"Algorithms[[:space:]]*:[[:space:]]*0\")\n addProblem(\"iris\", data = iris, reg = reg)\n addAlgorithm(\"foo\", fun = function(...) list(...), reg = reg)\n expect_character(reg$problems, len = 1L, any.missing = FALSE)\n expect_character(reg$algorithms, len = 1L, any.missing = FALSE)\n expect_output(print(reg), \"Problems[[:space:]]*:[[:space:]]*1\")\n expect_output(print(reg), \"Algorithms[[:space:]]*:[[:space:]]*1\")\n})\n" }, { "path": "tests/testthat/test_Job.R", "content": "test_that(\"Job\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, reg = reg, more.args = list(x = 1))\n submitAndWait(reg, 1, resources = list(foo = \"bar\"))\n\n job = makeJob(reg = reg, i = 1)\n expect_is(job, \"Job\")\n expect_identical(job$id, 1L)\n expect_equal(job$pars, list(i = 1L, x = 1))\n expect_count(job$seed)\n expect_list(job$resources, names = \"named\")\n expect_equal(job$resources$foo, \"bar\")\n expect_function(job$fun)\n\n jc = makeJobCollection(reg = reg, resources = list(foo = \"bar\"))\n job = getJob(jc, i = 1L)\n expect_is(job, \"Job\")\n expect_identical(job$id, 1L)\n expect_equal(job$pars, list(i = 1L, x = 1))\n expect_count(job$seed)\n expect_list(job$resources, names = \"named\")\n expect_equal(job$resources$foo, \"bar\")\n expect_function(job$fun)\n})\n\ntest_that(\"Experiment\", {\n reg = makeTestExperimentRegistry()\n addProblem(reg = reg, \"p1\", fun = function(job, data, ...) list(data = data, ...))\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) length(instance))\n ids = addExperiments(list(p1 = data.table(i = 1:3)), list(a1 = data.table()), reg = reg)\n\n job = makeJob(1, reg = reg)\n expect_is(job, \"Experiment\")\n expect_identical(job$id, 1L)\n expect_equal(job$pars, list(prob.pars = list(i = 1), algo.pars = list()))\n expect_count(job$repl)\n expect_count(job$seed)\n expect_list(job$resources, names = \"named\")\n expect_is(job$problem, \"Problem\")\n expect_is(job$algorithm, \"Algorithm\")\n expect_identical(job$instance, list(data = NULL, i = 1L))\n\n jc = makeJobCollection(reg = reg)\n job = getJob(jc, i = 1L)\n expect_is(job, \"Experiment\")\n expect_identical(job$id, 1L)\n expect_equal(job$pars, list(prob.pars = list(i = 1), algo.pars = list()))\n expect_count(job$seed)\n expect_list(job$resources, names = \"named\")\n expect_is(job$problem, \"Problem\")\n expect_is(job$algorithm, \"Algorithm\")\n expect_identical(job$instance, list(data = NULL, i = 1L))\n})\n\ntest_that(\"External directory is created\", {\n reg = makeTestRegistry()\n fun = function(..., .job) .job$external.dir\n ids = batchMap(fun, i = 1:3, reg = reg, more.args = list(x = 1))\n submitAndWait(reg)\n expect_directory_exists(unwrap(reduceResultsDataTable(1:3, reg = reg))[[2]])\n\n reg = makeTestExperimentRegistry()\n addProblem(reg = reg, \"p1\", fun = function(job, data, ...) list(data = data, ...))\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) {\n saveRDS(job$id, file = fs::path(job$external.dir, sprintf(\"%s.rds\", job$id)), version = 2L)\n job$external.dir\n })\n ids = addExperiments(list(p1 = data.table(i = 1:3)), list(a1 = data.table()), reg = reg)\n submitAndWait(reg, c(1, 3))\n paths = reduceResultsList(1:3, missing.val = NULL, reg = reg)\n expect_directory_exists(paths[[1]])\n expect_true(fs::file_exists(fs::path(reg$file.dir, \"external\", \"1\", \"1.rds\")))\n expect_null(paths[[2]])\n expect_false(fs::dir_exists(fs::path(reg$file.dir, \"external\", \"2\")))\n expect_directory_exists(paths[[3]])\n expect_true(fs::file_exists(fs::path(reg$file.dir, \"external\", \"3\", \"3.rds\")))\n expect_equal(reduceResultsList(1:3, fun = function(job, ...) job$external.dir, reg = reg, missing.val = NULL), paths)\n resetJobs(3, reg = reg)\n expect_false(fs::dir_exists(fs::path(reg$file.dir, \"external\", \"3\")))\n expect_true(fs::dir_exists(fs::path(reg$file.dir, \"external\", \"1\")))\n\n # directory is persistent between submits?\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) {\n list.files(job$external.dir)\n })\n\n submitAndWait(reg, 1)\n sweepRegistry(reg = reg)\n expect_true(fs::file_exists(fs::path(reg$file.dir, \"external\", \"1\", \"1.rds\")))\n expect_identical(loadResult(1, reg = reg), \"1.rds\")\n})\n" }, { "path": "tests/testthat/test_JobCollection.R", "content": "test_that(\"makeJobCollection\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, reg = reg, more.args = list(x = 1))\n\n jc = makeJobCollection(ids, resources = list(foo = 42), reg = reg)\n expect_environment(jc, c(\"file.dir\", \"job.hash\", \"jobs\", \"log.file\", \"packages\", \"resources\", \"uri\", \"work.dir\"))\n\n expect_directory_exists(jc$file.dir)\n expect_string(jc$job.hash, pattern = \"^job[[:alnum:]]{32}$\")\n expect_data_table(jc$jobs, key = \"job.id\")\n expect_string(jc$log.file)\n expect_character(jc$packages, any.missing = FALSE)\n expect_list(jc$resources, names = \"unique\")\n expect_string(jc$uri)\n expect_directory_exists(jc$work.dir)\n expect_list(jc$jobs$job.pars)\n expect_string(jc$array.var, na.ok = TRUE)\n expect_flag(jc$array.jobs)\n\n expect_output(print(jc), \"Collection\")\n})\n\n\ntest_that(\"makeJobCollection does not expand relative paths\", {\n skip_on_os(\"windows\")\n reg = makeTestRegistry(file.dir = NA, make.default = FALSE)\n batchMap(identity, 1, reg = reg)\n reg$file.dir = fs::path_abs(\"~/foo\")\n reg$work.dir = fs::path_abs(\"~/bar\")\n expect_string(reg$file.dir, pattern = \"^~\")\n expect_string(reg$work.dir, pattern = \"^~\")\n jc = makeJobCollection(1, reg = reg)\n expect_true(stri_startswith_fixed(jc$file.dir, \"~/foo\"))\n expect_true(stri_startswith_fixed(jc$uri, \"~/foo/jobs/\"))\n expect_true(stri_startswith_fixed(jc$log.file, \"~/foo/logs\"))\n expect_true(stri_startswith_fixed(jc$work.dir, \"~/bar\"))\n})\n\ntest_that(\"makeJobCollection.ExperimentCollection\", {\n reg = makeTestExperimentRegistry()\n addProblem(reg = reg, \"p1\", fun = function(job, data, ...) list(data = data, ...))\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) length(instance))\n ids = addExperiments(list(p1 = data.table(i = 1:3)), list(a1 = data.table()), reg = reg)\n\n jc = makeJobCollection(ids, resources = list(foo = 42), reg = reg)\n\n expect_directory_exists(jc$file.dir)\n expect_string(jc$job.hash, pattern = \"^job[[:alnum:]]{32}$\")\n expect_data_table(jc$jobs, key = \"job.id\")\n expect_string(jc$log.file)\n expect_character(jc$packages, any.missing = FALSE)\n expect_list(jc$resources, names = \"unique\")\n expect_string(jc$uri)\n expect_directory_exists(jc$work.dir)\n expect_list(jc$jobs$prob.pars)\n expect_list(jc$jobs$algo.pars)\n expect_character(jc$jobs$problem)\n expect_character(jc$jobs$algorithm)\n expect_string(jc$array.var, na.ok = TRUE)\n expect_flag(jc$array.jobs)\n\n expect_is(jc, \"ExperimentCollection\")\n})\n\ntest_that(\"chunks.as.arrayjobs is stored\", {\n reg = makeTestRegistry(file.dir = NA, make.default = FALSE)\n ids = batchMap(identity, 1:2, reg = reg)\n resources = list(chunks.as.arrayjobs = TRUE)\n jc = makeJobCollection(ids, resources = resources, reg = reg)\n expect_true(jc$array.jobs)\n})\n" }, { "path": "tests/testthat/test_JobNames.R", "content": "test_that(\"setJobNames\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, reg = reg)\n\n expect_null(setJobNames(ids, letters[1:3], reg = reg))\n\n x = getJobNames(reg = reg)\n expect_data_table(x, ncol = 2, nrow = 3, key = \"job.id\")\n expect_identical(x$job.name, letters[1:3])\n expect_identical(reg$status$job.name, letters[1:3])\n\n expect_data_table(ijoin(getJobNames(1:2, reg = reg), getJobPars(reg = reg)),\n ncol = 3, nrow = 2, key = \"job.id\")\n\n jc = makeJobCollection(1, reg = reg)\n expect_identical(jc$job.name, \"a\")\n\n jc = makeJobCollection(1:3, reg = reg)\n expect_identical(jc$job.name, \"a\")\n expect_identical(jc$jobs$job.name, letters[1:3])\n\n expect_null(setJobNames(ids, rep(NA_character_, 3), reg = reg))\n x = getJobNames(reg = reg)\n expect_data_table(x, ncol = 2, nrow = 3, key = \"job.id\")\n expect_identical(x$job.name, rep(NA_character_, 3))\n\n jc = makeJobCollection(1:3, reg = reg)\n expect_identical(jc$job.name, jc$job.hash)\n})\n" }, { "path": "tests/testthat/test_Problem.R", "content": "test_that(\"addProblem / removeProblem\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, ...) nrow(data))\n expect_is(prob, \"Problem\")\n expect_equal(prob$data, iris)\n expect_equal(prob$name, \"p1\")\n expect_function(prob$fun)\n expect_null(prob$seed)\n expect_file_exists(getProblemURI(reg, prob$name))\n expect_false(prob$cache)\n expect_false(fs::dir_exists(getProblemCacheDir(reg, \"p1\")))\n\n prob = addProblem(reg = reg, \"p2\", fun = function(...) NULL, seed = 42, cache = TRUE)\n expect_is(prob, \"Problem\")\n expect_null(prob$data, NULL)\n expect_equal(prob$name, \"p2\")\n expect_function(prob$fun)\n expect_identical(prob$seed, 42L)\n expect_file_exists(getProblemURI(reg, prob$name))\n expect_true(prob$cache)\n expect_directory_exists(getProblemCacheDir(reg, \"p2\"))\n\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) NULL)\n prob.designs = list(p1 = data.table(), p2 = data.table())\n algo.designs = list(a1 = data.table())\n ids = addExperiments(prob.designs, algo.designs, repls = 2, reg = reg)\n expect_integer(ids$job.id, len = 4L)\n\n removeProblems(reg = reg, \"p1\")\n expect_directory_exists(getProblemCacheDir(reg, \"p2\"))\n expect_integer(reg$status$job.id, len = 2L)\n expect_set_equal(\"p2\", reg$problems)\n expect_false(fs::file_exists(getProblemURI(reg, \"p1\")))\n expect_true(fs::file_exists(getProblemURI(reg, \"p2\")))\n expect_set_equal(getJobPars(reg = reg)$problem, \"p2\")\n checkTables(reg)\n\n removeProblems(reg = reg, \"p2\")\n expect_false(fs::dir_exists(getProblemCacheDir(reg, \"p2\")))\n})\n\n\ntest_that(\"instance caching\", {\n reg = makeTestExperimentRegistry()\n addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, param) param * 10 + runif(1), seed = 1, cache = TRUE)\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) list(result = instance))\n addAlgorithm(reg = reg, \"a2\", fun = function(job, data, instance, ...) list(result = instance))\n ids = addExperiments(prob.designs = list(p1 = data.table(param = 1:2)), repls = 2, reg = reg)\n\n j = makeJob(1, reg = reg)\n foo = j$instance\n expect_file_exists(getProblemCacheURI(j))\n\n submitAndWait(reg = reg)\n tab = unwrap(ljoin(getJobTable(reg = reg)[, c(\"job.id\", \"repl\", \"problem\", \"prob.pars\", \"algorithm\")], reduceResultsDataTable(reg = reg)))\n expect_equal(tab[, list(v = var(result)), by = c(\"param\", \"problem\", \"repl\")]$v, rep(0, 4))\n})\n" }, { "path": "tests/testthat/test_Registry.R", "content": "test_that(\"makeRegistry\", {\n reg = makeTestRegistry()\n expect_is(reg, \"Registry\")\n expect_true(is.environment(reg))\n expect_directory_exists(reg$file.dir, access = \"rw\")\n expect_directory_exists(reg$work.dir, access = \"r\")\n expect_directory_exists(fs::path(reg$file.dir, c(\"jobs\", \"results\", \"updates\", \"logs\")))\n expect_file_exists(fs::path(reg$file.dir, \"registry.rds\"))\n expect_character(reg$packages, any.missing = FALSE)\n expect_character(reg$namespaces, any.missing = FALSE)\n expect_int(reg$seed, na.ok = FALSE)\n expect_true(reg$writeable)\n expect_is(reg$cluster.functions, \"ClusterFunctions\")\n expect_list(reg$default.resources, names = \"strict\")\n checkTables(reg, any.missing = FALSE, nrows = 0L)\n\n reg = makeTestRegistry(packages = \"checkmate\", seed = 123)\n expect_equal(reg$packages, \"checkmate\")\n expect_int(reg$seed)\n expect_identical(reg$seed, 123L)\n\n expect_output(print(reg), \"Registry\")\n})\n\ntest_that(\"reading conf file\", {\n fn = fs::file_temp(\"conf\")\n writeLines(con = fn, \"default.resources = list(walltime = 42)\")\n reg = makeTestRegistry(conf.file = fn)\n expect_identical(reg$default.resources, list(walltime = 42))\n fs::file_delete(fn)\n})\n\ntest_that(\"make.default does work\", {\n prev = batchtools$default.registry\n\n setDefaultRegistry(NULL)\n expect_error(getDefaultRegistry(), \"No default\")\n\n reg = makeTestRegistry(make.default = TRUE, seed = 123)\n expect_equal(reg$seed, 123L)\n reg = makeTestRegistry(make.default = FALSE, seed = 124)\n expect_equal(reg$seed, 124L)\n expect_class(getDefaultRegistry(), \"Registry\")\n expect_equal(getDefaultRegistry()$seed, 123L)\n\n batchtools$default.registry = prev\n})\n\ntest_that(\"extra files are loaded\", {\n wd = fs::file_temp()\n fs::dir_create(fs::path(wd, \"subdir\"))\n\n # define some files to source/load\n fn = list(source = fs::path(wd, \"src_file.r\"), load = fs::path(wd, \"subdir\", \"load_file.RData\"))\n writeLines(\"x_from_source = 123\", con = fn$source)\n x_from_load = 321\n save(x_from_load, file = fn$load)\n rm(x_from_load)\n\n reg = makeTestRegistry(work.dir = wd, source = fn$source, load = fn$load)\n expect_identical(get(\"x_from_source\", .GlobalEnv), 123)\n expect_identical(get(\"x_from_load\", .GlobalEnv), 321)\n rm(\"x_from_source\", envir = .GlobalEnv)\n rm(\"x_from_load\", envir = .GlobalEnv)\n\n reg = makeTestRegistry(work.dir = wd, source = fs::path_file(fn$source), load = fs::path(\"subdir\", fs::path_file(fn$load)))\n expect_identical(get(\"x_from_source\", .GlobalEnv), 123)\n expect_identical(get(\"x_from_load\", .GlobalEnv), 321)\n rm(\"x_from_source\", envir = .GlobalEnv)\n rm(\"x_from_load\", envir = .GlobalEnv)\n fs::dir_delete(wd)\n})\n\ntest_that(\"loadRegistry\", {\n regs = list(\n makeTestRegistry(),\n makeTestExperimentRegistry()\n )\n for (reg1 in regs) {\n fd = reg1$file.dir\n setDefaultRegistry(NULL)\n reg2 = loadRegistry(fd, make.default = FALSE, writeable = TRUE)\n checkTables(reg1)\n checkTables(reg2)\n nms = union(ls(reg1, all.names = TRUE), ls(reg2, all.names = TRUE))\n nms = chsetdiff(nms, \"hash\")\n for (nm in nms)\n expect_equal(reg1[[nm]], reg2[[nm]], info = nm)\n\n x = readRDS(fs::path(fd, \"registry.rds\"))\n expect_null(x$cluster.functions)\n expect_null(x$default.resources)\n expect_null(x$temp.dir)\n expect_null(x$mtime)\n expect_null(x$writeable)\n }\n})\n\ntest_that(\"loadRegistry with missing dependencies is still usable (#122)\", {\n expect_warning(reg <- makeTestRegistry(source = fs::file_temp()), \"Failed to source\")\n saveRegistry(reg)\n expect_warning(loadRegistry(reg$file.dir, writeable = TRUE), \"Failed to source\")\n batchMap(identity, 1, reg = reg)\n expect_error(testJob(1, external = FALSE, reg = reg), \"Failed to source file\")\n})\n\ntest_that(\"loadRegistry after early node error still usable (#135)\", {\n reg = makeTestRegistry()\n batchMap(identity, 1:2, reg = reg)\n jc = makeJobCollection(1, reg = reg)\n jc$packages = \"not_existing_package\"\n suppressAll(doJobCollection(jc))\n expect_character(list.files(fs::path(reg$file.dir, \"updates\")), len = 1L)\n expect_true(syncRegistry(reg = reg))\n expect_string(getErrorMessages(reg = reg)$message, fixed = \"not_existing_package\")\n})\n\ntest_that(\"syncRegistry skips broken update files)\", {\n reg = makeTestRegistry()\n p = dir(reg, \"updates\")\n fs::file_create(fs::path(p, \"foo.rds\"))\n fs::dir_ls(p)\n expect_message(syncRegistry(reg = reg), \"Skipping\")\n})\n\ntest_that(\"clearRegistry\", {\n reg = makeTestRegistry()\n reg$foo = TRUE\n ids = batchMap(identity, 1:3, reg = reg)\n addJobTags(1:2, \"bar\", reg = reg)\n ids[, chunk := chunk(job.id, n.chunks = 2)]\n submitAndWait(reg, ids)\n\n clearRegistry(reg)\n checkTables(reg, nrow = 0L)\n\n expect_identical(list.files(dir(reg, \"jobs\")), character(0))\n expect_identical(list.files(dir(reg, \"logs\")), character(0))\n expect_identical(list.files(dir(reg, \"results\")), character(0))\n expect_identical(list.files(dir(reg, \"updates\")), character(0))\n expect_false(fs::file_exists(fs::path(reg$file.dir, \"user.function.rds\")))\n\n expect_identical(batchMap(identity, 1:4, reg = reg), data.table(job.id = 1:4, key = \"job.id\"))\n expect_true(reg$foo)\n})\n\ntest_that(\"read only mode\", {\n f = function(x) if (x == 3) stop(3) else x\n reg = makeTestRegistry()\n batchMap(f, 1:4, reg = reg)\n submitAndWait(ids = 1:3, reg)\n\n # simulate that job 4 has been started but is not terminated yet\n jc = makeJobCollection(4L, reg = reg)\n suppressAll({doJobCollection(jc, jc$log.file)})\n reg$status[job.id == 4L, job.hash := jc$job.hash]\n saveRegistry(reg = reg)\n\n reg = loadRegistry(reg$file.dir, writeable = FALSE)\n\n # query status\n expect_class(getStatus(reg = reg), \"Status\")\n expect_data_table(findDone(reg = reg), nrow = 3)\n expect_data_table(findErrors(reg = reg), nrow = 1)\n expect_character(fs::dir_ls(fs::path(reg$file.dir, \"updates\")), len = 1L)\n\n # load results\n expect_identical(loadResult(1L, reg = reg), 1L)\n expect_identical(reduceResultsList(reg = reg), as.list(c(1:2, 4L)))\n expect_character(fs::dir_ls(fs::path(reg$file.dir, \"updates\")), len = 1L)\n\n # inspect errors\n expect_data_table(getErrorMessages(reg = reg), nrow = 1)\n expect_character(getLog(3L, reg = reg))\n expect_character(getLog(4L, reg = reg))\n\n # try to write\n expect_error(sweepRegistry(reg = reg), \"writeable\")\n expect_error(setJobNames(ids = 1L, reg = reg), \"writeable\")\n expect_error(addJobTags(ids = 1L, \"a\", reg = reg), \"writeable\")\n expect_error(resetJobs(reg = reg), \"writeable\")\n expect_error(clearRegistry(reg = reg), \"writeable\")\n expect_error(removeRegistry(reg = reg), \"writeable\")\n expect_error(killJobs(reg = reg), \"writeable\")\n expect_directory_exists(reg$file.dir)\n expect_character(fs::dir_ls(fs::path(reg$file.dir, \"updates\")), len = 1L)\n\n # same stuff for ExperimentRegistry\n reg = makeTestExperimentRegistry()\n addProblem(\"foo\", data = 1, reg = reg)\n addAlgorithm(\"bar\", function(data, instance, ...) instance, reg = reg)\n addExperiments(reg = reg)\n reg$writeable = FALSE\n\n expect_data_table(summarizeExperiments(reg = reg), nrow = 1L)\n expect_data_table(findExperiments(reg = reg))\n\n expect_error(addProblem(\"foo2\", iris, reg = reg), \"writeable\")\n expect_error(removeProblems(\"foo2\", reg = reg), \"writeable\")\n expect_error(addAlgorithm(\"bar2\", function(data, instance, ...) instance, reg = reg), \"writeable\")\n expect_error(removeAlgorithms(\"bar2\", reg = reg), \"writeable\")\n expect_error(addExperiments(reg = reg), \"writeable\")\n expect_error(removeExperiments(1, reg = reg), \"writeable\")\n})\n\ntest_that(\"xz compression\", {\n fn = fs::file_temp(\"conf\")\n writeLines(con = fn, \"compress = \\\"xz\\\"\")\n reg = makeTestRegistry(conf.file = fn)\n expect_identical(reg$compress, \"xz\")\n fd = file(dir(reg, \"registry.rds\"), \"r\")\n expect_identical(summary(fd)$class, \"xzfile\")\n close(fd)\n\n batchMap(identity, 1:3, reg = reg)\n submitAndWait(reg)\n\n fd = file(getResultFiles(reg, 1), \"r\")\n expect_identical(summary(fd)$class, \"xzfile\")\n close(fd)\n})\n" }, { "path": "tests/testthat/test_addExperiments.R", "content": "test_that(\"addExperiments handles parameters correctly\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, x, y, ...) stopifnot(is.numeric(x) && is.character(y)), seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, a, b, ...) { print(str(a)); checkmate::assertList(a, len = 1, names = \"named\"); checkmate::assertDataFrame(b); } )\n prob.designs = list(p1 = data.table(x = 1:2, y = letters[1:2]))\n algo.designs = list(a1 = data.table(a = list(list(x = 1)), b = list(iris)))\n repls = 2\n ids = addExperiments(prob.designs, algo.designs, repls = repls, reg = reg)\n expect_data_table(ids, nrow = 4, key = \"job.id\")\n ids = addExperiments(prob.designs, algo.designs, repls = repls, reg = reg)\n expect_data_table(ids, nrow = 0, key = \"job.id\")\n ids = addExperiments(prob.designs, algo.designs, repls = repls + 1L, reg = reg)\n expect_data_table(ids, nrow = 2, key = \"job.id\")\n\n submitAndWait(reg, ids)\n expect_true(nrow(findErrors(reg = reg)) == 0)\n})\n\ntest_that(\"addExperiments creates default designs\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris)\n prob = addProblem(reg = reg, \"p2\", data = cars)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance) nrow(data))\n algo = addAlgorithm(reg = reg, \"a2\", fun = function(job, data, instance) ncol(data))\n ids = addExperiments(reg = reg)\n expect_equal(findExperiments(reg = reg)$job.id, 1:4)\n expect_equal(as.character(reg$defs$problem), rep(c(\"p1\", \"p2\"), each = 2))\n expect_equal(as.character(reg$defs$algorithm), rep(c(\"a1\", \"a2\"), times = 2))\n})\n\ntest_that(\"addExperiments / user provided designs\", {\n reg = makeTestExperimentRegistry()\n addProblem(reg = reg, \"p1\", data = iris, fun = function(...) list(...))\n addProblem(reg = reg, \"p2\", data = cars, fun = function(...) list(...))\n addAlgorithm(reg = reg, \"a1\", fun = function(...) list(...))\n addAlgorithm(reg = reg, \"a2\", fun = function(...) ncol(data))\n prob.designs = list(p1 = data.table(a = 1, b = 2:4))\n algo.designs = list(a1 = data.table(c = 3:8), a2 = data.table())\n ids = addExperiments(reg = reg, prob.designs = prob.designs, algo.designs = algo.designs, combine = \"bind\")\n expect_data_table(ids, nrow = 9, key = \"job.id\")\n tab = getJobPars(reg = reg)\n pars = unwrap(getJobPars(reg = reg))\n expect_set_equal(pars$problem, \"p1\")\n expect_set_equal(pars$algorithm, c(\"a1\", \"a2\"))\n expect_equal(pars$a, rep(1L, 9))\n expect_equal(pars$b, rep(2:4, 3))\n expect_equal(pars$c, c(3:8, rep(NA, 3)))\n expect_error(addExperiments(reg = reg, prob.designs = list(p1 = data.table(job = 2))), \"reserved keyword 'job'\")\n expect_error(addExperiments(reg = reg, algo.designs = list(a2 = data.table(instance = \"foo\"))), \"reserved keyword 'instance'\")\n\n prob.designs = c(prob.designs, list(p2 = data.table()))\n ids = addExperiments(reg = reg, prob.designs = prob.designs, algo.designs = algo.designs, combine = \"bind\")\n expect_data_table(ids, nrow = 7, key = \"job.id\")\n expect_data_table(unwrap(getJobPars(reg = reg)), nrow = 16)\n\n ids = addExperiments(reg = reg, prob.designs = prob.designs, algo.designs = algo.designs, combine = \"crossprod\")\n expect_data_table(ids, nrow = 12, key = \"job.id\")\n expect_data_table(unwrap(getJobPars(reg = reg)), nrow = 28)\n\n if (getRversion() < \"4.0.0\") {\n pd = list(p1 = data.frame(foo = letters[1:2]))\n withr::with_options(list(stringsAsFactors = NULL), {\n expect_warning(addExperiments(reg = reg, prob.designs = pd), \"stringsAsFactors\")\n })\n withr::with_options(list(stringsAsFactors = TRUE), {\n expect_warning(addExperiments(reg = reg, prob.designs = pd), \"stringsAsFactors\")\n })\n withr::with_options(list(stringsAsFactors = FALSE), {\n addExperiments(reg = reg, prob.designs = pd)\n })\n }\n})\n\n# reg = makeTestExperimentRegistry()\n# addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, ...) nrow(data))\n# addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) NULL)\n# addAlgorithm(reg = reg, \"a2\", fun = function(job, data, instance, ...) NULL)\n# prob.designs = list(p1 = data.table(x = 1:500))\n# algo.designs = list(a1 = data.table(y = 1:50), a2 = data.table(y = 1:20))\n# repls = 2\n# profvis::profvis(addExperiments(prob.designs, algo.designs = algo.designs, repls = repls, reg = reg))\n# ids = findExperiments(reg = reg)\n# profvis::profvis(submitJobs(ids = s.chunk(ids), reg = reg))\n\n# profvis::profvis(unwrap(getJobPars(reg = reg)))\n" }, { "path": "tests/testthat/test_batchMap.R", "content": "test_that(\"batchMap\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, more.args = list(x = 1), reg = reg)\n expect_data_table(ids, any.missing = FALSE, ncols = 1L, nrow = 3L, key = \"job.id\")\n expect_equal(ids$job.id, 1:3)\n if (getRversion() < \"4.1.0\")\n expect_equal(readRDS(fs::path(reg$file.dir, \"user.function.rds\")), fun)\n expect_equal(readRDS(fs::path(reg$file.dir, \"more.args.rds\")), list(x = 1))\n\n checkTables(reg)\n expect_data_table(reg$defs, nrow = 3L, any.missing = FALSE)\n expect_data_table(reg$status, nrow = 3L)\n expect_data_table(reg$resources, nrow = 0L)\n expect_equal(reg$defs$job.pars[[1L]], list(i = 1))\n\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, 1:3, more.args = list(j = 1), reg = reg)\n expect_equal(readRDS(fs::path(reg$file.dir, \"more.args.rds\")), list(j = 1))\n expect_equal(reg$defs$job.pars, lapply(1:3, list))\n\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, args = list(1:3), more.args = list(j = 1), reg = reg)\n expect_equal(readRDS(fs::path(reg$file.dir, \"more.args.rds\")), list(j = 1))\n expect_equal(reg$defs$job.pars, lapply(1:3, list))\n\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, j = 1L, reg = reg)\n expect_identical(reg$defs$job.pars[[3L]], list(i = 3L, j = 1L))\n\n reg = makeTestRegistry()\n fun = function(...) list(...)\n cj = CJ(a = 1:3, b = letters[1:3])\n ids = batchMap(fun, args = cj, reg = reg)\n expect_data_table(ids, nrow = 9, key = \"job.id\")\n expect_equivalent(unwrap(getJobPars(reg = reg))[, c(\"a\", \"b\")], cj)\n})\n\ntest_that(\"batchMap with unnamed more.args (#267)\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, 1:3, more.args = list(j = 1L, 5L), reg = reg)\n expect_equal(readRDS(fs::path(reg$file.dir, \"more.args.rds\")), list(j = 1L, 5L))\n submitAndWait(reg)\n expect_equal(loadResult(1, reg), list(1L, j = 1L, 5L))\n})\n" }, { "path": "tests/testthat/test_batchReduce.R", "content": "test_that(\"batchReduce\", {\n reg = makeTestRegistry()\n xs = 1:20\n ids = batchReduce(function(aggr, x) aggr+x, xs, init = 0, chunks = chunk(seq_along(xs), n.chunks = 10), reg = reg)\n expect_data_table(ids, nrow = 10, key = \"job.id\")\n\n submitAndWait(ids = ids, reg = reg)\n y = reduceResults(fun = function(aggr, job, res) aggr+res, init = 0, reg = reg)\n expect_equal(y, sum(1:20))\n})\n\n\ntest_that(\"batchReduce w/ more.args\", {\n reg = makeTestRegistry()\n xs = 1:20\n chunks = sort(chunk(seq_along(xs), n.chunks = 10))\n ids = batchReduce(function(aggr, x, y) aggr+x+y, 1:20, init = 100, chunks = chunks, more.args = list(y=1), reg = reg)\n expect_data_table(ids, nrow = 10, key = \"job.id\")\n\n submitAndWait(reg = reg)\n\n expect_equal(as.integer(reduceResultsList(reg = reg)), viapply(split(xs, chunks), function(x) 100L + length(x) + sum(x), use.names = FALSE))\n y = reduceResults(fun=function(aggr, job, res) aggr+res, init = 0, reg = reg)\n expect_equal(y, sum(1:20) + 20 + uniqueN(chunks) * 100)\n})\n" }, { "path": "tests/testthat/test_btlapply.R", "content": "test_that(\"btlapply\", {\n reg = makeTestRegistry()\n fun = function(x, y) x^y\n res = silent(btlapply(1:3, fun, y = 2, n.chunks = 2, resources = list(..dummy = 42), reg = reg))\n expect_equal(res, lapply(1:3, fun, y = 2))\n expect_equal(uniqueN(reg$status$job.hash), 2)\n expect_equal(reg$resources$resources[[1L]]$..dummy, 42)\n})\n\ntest_that(\"btmapply\", {\n fun = function(x, y) paste0(x, y)\n x = 1:3\n y = letters[1:3]\n\n reg = makeTestRegistry()\n res = silent(btmapply(fun, x = x, y = y, chunk.size = 2, use.names = FALSE, reg = reg))\n expect_equal(res, mapply(fun, x = x, y = y, SIMPLIFY = FALSE, USE.NAMES = FALSE))\n expect_equal(uniqueN(reg$status$job.hash), 2)\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsInteractive()\n expect_equal(silent(btmapply(fun, x = x, y = y, n.chunks = 1, use.names = FALSE, simplify = TRUE, reg = reg)), mapply(fun, x = x, y = y, SIMPLIFY = TRUE, USE.NAMES = FALSE))\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsInteractive()\n expect_equal(silent(btmapply(fun, x = x, y = y, n.chunks = 1, use.names = TRUE, simplify = TRUE, reg = reg)), mapply(fun, x = x, y = y, SIMPLIFY = TRUE, USE.NAMES = TRUE))\n})\n" }, { "path": "tests/testthat/test_chunk.R", "content": "test_that(\"chunk\", {\n x = 1:10; n.chunks = 2\n expect_integer(chunk(x, n.chunks = n.chunks), len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n x = 1:10; n.chunks = 1\n expect_integer(chunk(x, n.chunks = n.chunks), len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n x = 1:10; n.chunks = 10\n expect_integer(chunk(x, n.chunks = n.chunks), len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n x = 1:10; n.chunks = 20\n expect_integer(chunk(x, n.chunks = n.chunks), len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n x = integer(0); n.chunks = 20\n expect_integer(chunk(x, n.chunks = n.chunks), len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n\n x = 1:10; chunk.size = 3\n res = chunk(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_integer(table(res), lower = 1, upper = chunk.size, any.missing = FALSE)\n\n x = 1:10; chunk.size = 1\n res = chunk(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_integer(table(res), lower = 1, upper = chunk.size, any.missing = FALSE)\n\n expect_equal(chunk(numeric(0), chunk.size = 1), integer(0))\n expect_equal(chunk(numeric(0), n.chunks = 1), integer(0))\n\n x = 1:10; n.chunks = 2\n res = c(rep(1, 5), rep(2, 5))\n expect_equal(chunk(x, n.chunks = n.chunks, shuffle = FALSE), res)\n})\n\ntest_that(\"binpack\", {\n x = 1:10; chunk.size = 10\n res = binpack(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), lower = min(x), upper = chunk.size, any.missing = FALSE)\n\n x = 1; chunk.size = 10\n res = binpack(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), lower = min(x), upper = chunk.size, any.missing = FALSE)\n\n x = rep(1, 100); chunk.size = 1\n res = binpack(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), lower = min(x), upper = chunk.size, any.missing = FALSE)\n\n x = runif(100); chunk.size = 1\n res = binpack(x, chunk.size = chunk.size)\n expect_integer(res, len = length(x), lower = 1, upper = length(x), any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), lower = min(x), upper = chunk.size, any.missing = FALSE)\n\n expect_equal(binpack(numeric(0), 1), integer(0))\n})\n\ntest_that(\"lpt\", {\n x = 1:10; n.chunks = 2\n res = lpt(x, n.chunks)\n expect_integer(res, len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), len = min(length(x), n.chunks), lower = min(x), any.missing = FALSE)\n\n x = runif(100); n.chunks = 3\n res = lpt(x, n.chunks)\n expect_integer(res, len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), len = min(length(x), n.chunks), lower = min(x), any.missing = FALSE)\n\n x = 1:10; n.chunks = 1\n res = lpt(x, n.chunks)\n expect_integer(res, len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), len = min(length(x), n.chunks), lower = min(x), any.missing = FALSE)\n\n x = 1:10; n.chunks = 12\n res = lpt(x, n.chunks)\n expect_integer(res, len = length(x), lower = 1, upper = n.chunks, any.missing = FALSE)\n expect_numeric(sapply(split(x, res), sum), len = min(length(x), n.chunks), lower = min(x), any.missing = FALSE)\n expect_equal(unname(res), 10:1)\n\n expect_equal(lpt(numeric(0), 1), integer(0))\n})\n\ntest_that(\"caching works\", {\n reg = makeTestExperimentRegistry()\n p1 = addProblem(reg = reg, \"p1\", data = iris)\n p2 = addProblem(reg = reg, \"p2\", data = data.frame(a = 1:10))\n a1 = addAlgorithm(reg = reg, name = \"a1\", fun = function(data, ...) nrow(data))\n a2 = addAlgorithm(reg = reg, name = \"a2\", fun = function(data, ...) 2L * nrow(data))\n\n addExperiments(reg = reg)\n ids = findJobs(reg = reg)\n ids$chunk = 1L\n submitAndWait(reg, ids)\n\n expect_identical(unlist(reduceResultsList(ids, reg = reg)), as.integer(c(150, 300, 10, 20)))\n})\n" }, { "path": "tests/testthat/test_convertIds.R", "content": "test_that(\"convertIds\", {\n reg = makeTestRegistry()\n batchMap(identity, 1:10, reg = reg)\n reg$status = reg$status[-3, ]\n\n tab = convertIds(reg, NULL)\n expect_equal(tab, NULL)\n\n tab = convertIds(reg, 1:10)\n expect_data_table(tab, ncol = 1, nrow = 9, key = \"job.id\")\n expect_copied(tab, reg$status)\n\n tab = convertIds(reg, findJobs(reg = reg))\n expect_data_table(tab, ncol = 1, nrow = 9, key = \"job.id\")\n expect_copied(tab, reg$status)\n\n tab = convertIds(reg, data.table(job.id = 3:4, key = \"job.id\"))\n expect_data_table(tab, ncol = 1, nrow = 1, key = \"job.id\")\n\n tab = convertIds(reg, as.data.frame(findJobs(reg = reg)))\n expect_data_table(tab, ncol = 1, key = \"job.id\")\n expect_copied(tab, reg$status)\n\n tab = convertIds(reg, 10:8)\n expect_data_table(tab, ncol = 1, nrow = 3, key = \"job.id\")\n expect_equal(tab$job.id, 8:10)\n expect_copied(tab, reg$status)\n\n tab = convertIds(reg, 10:8, keep.order = TRUE)\n expect_data_table(tab, ncol = 1, nrow = 3)\n expect_equal(tab$job.id, 10:8)\n\n ids = findJobs(reg = reg)\n ids$chunk = 9:1\n tab = convertIds(reg, ids, keep.order = TRUE, keep.extra = \"chunk\")\n expect_data_table(tab, ncol = 2, nrow = 9, key = \"job.id\") # keep index if possible\n\n setorderv(ids, \"chunk\")\n tab = convertIds(reg, ids, keep.order = TRUE, keep.extra = \"chunk\")\n expect_data_table(tab, ncol = 2, nrow = 9)\n expect_null(key(tab))\n expect_equal(tab$job.id, setdiff(10:1, 3L))\n\n expect_error(convertIds(reg, c(2, 2)), \"Duplicated ids\")\n expect_error(convertIds(reg, as.character(1:3)), \"not recognized\")\n\n # issue #40\n ids = ids[list(5:10), on = \"job.id\"][, \"chunk\" := chunk(job.id, chunk.size = 3)]\n ids = convertIds(reg, ids, keep.extra = c(\"job.id\", \"chunk\"))\n expect_data_table(ids, any.missing = FALSE)\n})\n" }, { "path": "tests/testthat/test_count.R", "content": "test_that(\"count\", {\n expect_identical(count(1:3), 3L)\n expect_identical(count(integer(0L)), 0L)\n expect_identical(count(list()), 0L)\n expect_identical(count(c(TRUE, NA, FALSE)), 2L)\n expect_identical(count(c(1L, NA, 3L)), 2L)\n expect_identical(count(c(1., NA, 3.)), 2L)\n expect_identical(count(c(\"a\", NA, \"c\")), 2L)\n expect_identical(count(list(1, NULL, 3)), 2L)\n})\n" }, { "path": "tests/testthat/test_doJobCollection.R", "content": "test_that(\"doJobCollection handles bulky log output\", {\n N = 1e5\n reg = makeTestRegistry()\n fun = function(N) print(paste(rep(\"a\", N), collapse = \"\"))\n batchMap(fun, N, reg = reg)\n jc = makeJobCollection(1, reg = reg)\n fn = fs::file_temp()\n doJobCollection(jc, output = fn)\n lines = readLines(fn)\n expect_true(any(nchar(lines) >= N))\n fs::file_delete(fn)\n})\n\ntest_that(\"doJobCollection truncates error messages\", {\n N = 5000 # R truncates stop() at 2^13 chars\n reg = makeTestRegistry()\n fun = function(N) stop(paste(rep(\"a\", N), collapse = \"\"))\n batchMap(fun, N, reg = reg)\n jc = makeJobCollection(1, reg = reg)\n fn = fs::file_temp()\n doJobCollection(jc, output = fn)\n syncRegistry(reg = reg)\n msg = getErrorMessages(reg = reg)$message\n expect_true(stri_endswith_fixed(msg, \" [truncated]\"))\n fs::file_delete(fn)\n})\n\ntest_that(\"doJobCollection does not swallow warning messages\", {\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsInteractive(external = TRUE)\n fun = function(x) warning(\"GREPME\")\n batchMap(fun, 1, reg = reg)\n submitAndWait(reg, 1)\n expect_data_table(findErrors(reg = reg), nrow = 0L)\n expect_data_table(grepLogs(pattern = \"GREPME\", reg = reg), nrow = 1L)\n})\n\ntest_that(\"doJobCollection signals slave errors\", {\n reg = makeTestRegistry()\n fn = fs::file_temp(ext = \".R\", tmp_dir = reg$temp.dir)\n reg$source = fn\n saveRegistry(reg)\n assign(\"y_on_master\", 2, envir = .GlobalEnv)\n writeLines(\"x <- y_on_master\", fn)\n rm(y_on_master, envir = .GlobalEnv)\n\n expect_error(withr::with_dir(reg$work.dir, loadRegistryDependencies(reg, must.work = TRUE)), \"y_on_master\")\n batchMap(identity, 1, reg = reg)\n submitAndWait(reg, 1)\n expect_data_table(findErrors(reg = reg), nrow = 1)\n expect_string(getErrorMessages(reg = reg)$message, fixed = \"y_on_master\")\n fs::file_delete(fn)\n})\n" }, { "path": "tests/testthat/test_estimateRuntimes.R", "content": "test_that(\"estimateRuntimes\", {\n reg = makeTestExperimentRegistry()\n addProblem(name = \"iris\", data = iris, fun = function(data, ...) nrow(data), reg = reg)\n addAlgorithm(name = \"nrow\", function(instance, ...) nrow(instance), reg = reg)\n addAlgorithm(name = \"ncol\", function(instance, ...) ncol(instance), reg = reg)\n addExperiments(algo.designs = list(nrow = CJ(x = 1:50, y = letters[1:5])), reg = reg)\n addExperiments(algo.designs = list(ncol = CJ(x = 1:50, y = letters[1:5])), reg = reg)\n tab = unwrap(getJobPars(reg = reg))\n\n ids = tab[, .SD[sample(nrow(.SD), 4)], by = c(\"problem\", \"algorithm\", \"y\")]\n setkeyv(ids, \"job.id\")\n submitAndWait(reg, ids = s.chunk(ids))\n\n # \"simulate\" runtimes\n runtime = function(algorithm, x, y) {\n ifelse(algorithm == \"nrow\", 100L, 500L) + 100000L * (y %in% letters[1:2])\n }\n reg$status[ids, done := done + tab[ids, runtime(algorithm, x, y), on = \"job.id\"], on = \"job.id\"]\n\n res = estimateRuntimes(tab, reg = reg)\n expect_is(res, \"RuntimeEstimate\")\n expect_set_equal(names(res), c(\"runtimes\", \"model\"))\n expect_is(res$model, \"ranger\")\n expect_data_table(res$runtimes, key = \"job.id\", nrow = nrow(reg$status))\n expect_output(print(res), \"Runtime Estimate for\")\n # there is a problem with \"null value passed as symbol address\"\n # this has probably to do with rcpp/ranger\n # NB: only during R CMD check, not interactively\n # expect_output(print(res, n = 5), \"Parallel\")\n\n res = ijoin(res$runtimes, tab)\n res = res[, list(t = mean(runtime)), by = y]\n expect_true(all(res[y == \"a\", t] > res[y %in% c(\"c\", \"d\", \"e\"), t]))\n\n # remaining is suppressed if nothing more to submit, no error\n res = estimateRuntimes(unwrap(getJobPars(findDone(reg = reg), reg = reg)), reg = reg)\n expect_output(print(res, n = 2))\n})\n" }, { "path": "tests/testthat/test_export.R", "content": "test_that(\"export works\", {\n reg = makeTestRegistry()\n x = batchExport(list(exported_obj = 42L), reg = reg)\n expect_data_table(x, nrow = 1, ncol = 2)\n expect_set_equal(names(x), c(\"name\", \"uri\"))\n expect_equal(x$name, \"exported_obj\")\n expect_file_exists(fs::path(reg$file.dir, \"exports\", mangle(\"exported_obj\")))\n withr::with_dir(reg$work.dir, loadRegistryDependencies(reg))\n expect_equal(get(\"exported_obj\", envir = .GlobalEnv), 42L)\n\n x = batchExport(reg = reg)\n expect_data_table(x, nrow = 1, ncol = 2)\n expect_set_equal(names(x), c(\"name\", \"uri\"))\n expect_equal(x$name, \"exported_obj\")\n\n x = batchExport(unexport = \"exported_obj\", reg = reg)\n expect_data_table(x, nrow = 0, ncol = 2)\n expect_set_equal(names(x), c(\"name\", \"uri\"))\n expect_false(fs::file_exists(fs::path(reg$file.dir, \"exports\", mangle(\"exported_obj\"))))\n\n x = batchExport(list(exported_obj = 43L), reg = reg)\n batchMap(function(x) exported_obj + x, 1L, reg = reg)\n submitAndWait(reg)\n expect_equal(loadResult(1, reg = reg), 44L)\n rm(\"exported_obj\", envir = .GlobalEnv)\n})\n\n\ntest_that(\"export works with funny variable names\", {\n reg = makeTestRegistry()\n x = batchExport(list(`%bla%` = function(x, y, ...) 42), reg = reg)\n expect_data_table(x, nrow = 1, ncol = 2)\n expect_set_equal(names(x), c(\"name\", \"uri\"))\n expect_equal(x$name, \"%bla%\")\n expect_file_exists(fs::path(reg$file.dir, \"exports\", mangle(\"%bla%\")))\n withr::with_dir(reg$work.dir, loadRegistryDependencies(reg))\n expect_function(get(\"%bla%\", envir = .GlobalEnv))\n expect_equal(1 %bla% 2, 42)\n\n x = batchExport(unexport = \"%bla%\", reg = reg)\n expect_data_table(x, nrow = 0, ncol = 2)\n expect_set_equal(names(x), c(\"name\", \"uri\"))\n expect_false(fs::file_exists(fs::path(reg$file.dir, \"exports\", mangle(\"%bla%\"))))\n\n rm(\"%bla%\", envir = .GlobalEnv)\n})\n" }, { "path": "tests/testthat/test_findConfFile.R", "content": "test_that(\"findConfFile\", {\n d = fs::path_real(fs::path_temp())\n fn = fs::path(d, \"batchtools.conf.R\")\n fs::file_create(fn)\n withr::with_dir(d,\n expect_equal(findConfFile(), fs::path_abs(fn))\n )\n withr::with_envvar(list(R_BATCHTOOLS_SEARCH_PATH = d),\n expect_equal(findConfFile(), fs::path_abs(fn))\n )\n fs::file_delete(fn)\n})\n" }, { "path": "tests/testthat/test_findJobs.R", "content": "none = noIds()\n\ntest_that(\"find[Status]\", {\n reg = makeTestRegistry()\n\n expect_equal(findJobs(reg = reg), none)\n expect_equal(findSubmitted(reg = reg), none)\n expect_equal(findNotSubmitted(reg = reg), none)\n expect_equal(findStarted(reg = reg), none)\n expect_equal(findNotStarted(reg = reg), none)\n expect_equal(findDone(reg = reg), none)\n expect_equal(findNotDone(reg = reg), none)\n expect_equal(findErrors(reg = reg), none)\n expect_equal(findOnSystem(reg = reg), none)\n expect_equal(findRunning(reg = reg), none)\n expect_equal(findQueued(reg = reg), none)\n expect_equal(findExpired(reg = reg), none)\n\n fun = function(i) if (i == 3) stop(i) else i\n ids = batchMap(fun, i = 1:5, reg = reg)\n all = reg$status[, \"job.id\"]\n\n expect_equal(findJobs(reg = reg), all)\n expect_equal(findSubmitted(reg = reg), none)\n expect_equal(findStarted(reg = reg), none)\n expect_equal(findNotStarted(reg = reg), all)\n expect_equal(findDone(reg = reg), none)\n expect_equal(findNotDone(reg = reg), all)\n expect_equal(findErrors(reg = reg), none)\n expect_equal(findOnSystem(reg = reg), none)\n expect_equal(findRunning(reg = reg), none)\n expect_equal(findQueued(reg = reg), none)\n expect_equal(findExpired(reg = reg), none)\n\n submitAndWait(reg, ids)\n\n expect_equal(findJobs(reg = reg), all)\n expect_equal(findSubmitted(reg = reg), all)\n expect_equal(findNotSubmitted(reg = reg), none)\n expect_equal(findStarted(reg = reg), all)\n expect_equal(findNotStarted(reg = reg), none)\n expect_equal(findDone(reg = reg), all[-3L])\n expect_equal(findNotDone(reg = reg), all[3L])\n expect_equal(findErrors(reg = reg), all[3L])\n expect_equal(findOnSystem(reg = reg), none)\n expect_equal(findRunning(reg = reg), none)\n expect_equal(findQueued(reg = reg), none)\n expect_equal(findExpired(reg = reg), none)\n})\n\ntest_that(\"Subsetting\", {\n reg = makeTestRegistry()\n fun = function(i) if (i == 3) stop(i) else i\n ids = batchMap(fun, i = 1:5, reg = reg)\n submitAndWait(reg, ids)\n all = reg$status[, \"job.id\"]\n\n expect_equal(findJobs(ids = 1:3, reg = reg), all[1:3])\n expect_equal(findDone(ids = 3, reg = reg), none)\n expect_equal(findErrors(ids = 1:2, reg = reg), none)\n expect_equal(findSubmitted(1:5, reg = reg), all)\n expect_data_table(findSubmitted(6, reg = reg), ncol = 1L, nrow = 0L)\n})\n\ntest_that(\"findJobs\", {\n reg = makeTestRegistry()\n fun = function(i, j) i + j\n ids = batchMap(fun, i = 1:5, j = c(2, 2, 3, 4, 4), reg = reg)\n all = reg$status[, \"job.id\"]\n expect_equal(findJobs(i == 1, reg = reg), all[1])\n expect_equal(findJobs(i >= 3, reg = reg), all[3:5])\n expect_equal(findJobs(i >= 3 & j > 3, reg = reg), all[4:5])\n xi = 2\n expect_equal(findJobs(i == xi, reg = reg), all[2])\n})\n\ntest_that(\"findOnSystem\", {\n reg = makeTestRegistry()\n if (is.null(reg$cluster.functions$listJobsRunning))\n skip(\"Test requires listJobsRunning\")\n silent({\n ids = batchMap(reg = reg, Sys.sleep, c(10, 10))\n submitJobs(reg = reg, ids = s.chunk(ids))\n expect_equal(findOnSystem(reg = reg), findJobs(reg = reg))\n expect_equal(findExpired(reg = reg), noIds())\n\n # ensure that the registry is not removed before jobs have finished\n waitForJobs(reg = reg, sleep = 1)\n })\n})\n\ntest_that(\"findExperiments\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", fun = function(job, data, n, ...) mean(runif(n)), seed = 42)\n prob = addProblem(reg = reg, \"p2\", data = iris, fun = function(job, data) nrow(data))\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq) instance^sq)\n prob.designs = list(p1 = data.table(n = c(10, 20)), p2 = data.table())\n algo.designs = list(a1 = data.table(sq = 1:3))\n repls = 10\n addExperiments(prob.designs, algo.designs, repls = repls, reg = reg)\n\n tab = findExperiments(reg = reg)\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.name = \"p1\")\n expect_data_table(tab, nrow = 60, ncol = 1, key = \"job.id\")\n\n expect_error(findExperiments(reg = reg, prob.name = c(\"p1\", \"p2\")), \"length 1\")\n\n tab = findExperiments(reg = reg, prob.pattern = \"p.\")\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.pattern = \"2$\")\n expect_data_table(tab, nrow = 30, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.pattern = \"p1\", algo.pattern = \"a1\", repls = 3:4)\n expect_data_table(tab, nrow = 12, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.pattern = c(\"^p\"))\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.name = \"p2\")\n expect_data_table(tab, nrow = 30, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, ids = 1:10, prob.name = \"p1\")\n expect_data_table(tab, nrow = 10, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.name = \"a1\")\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.name = \"p1\", prob.pars = n == 10)\n expect_data_table(tab, nrow = 30, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.pars = sq == 2)\n expect_data_table(tab, nrow = 30, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.name = \"a1\")\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.pattern = \"a.\")\n expect_data_table(tab, nrow = 90, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.name = \"p\")\n expect_data_table(tab, nrow = 0, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.name = \"a\")\n expect_data_table(tab, nrow = 0, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, prob.name = \"xxx\")\n expect_data_table(tab, nrow = 0, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, algo.name = \"xxx\")\n expect_data_table(tab, nrow = 0, ncol = 1, key = \"job.id\")\n\n tab = findExperiments(reg = reg, repls = 1:2)\n expect_data_table(tab, nrow = 18, ncol = 1, key = \"job.id\")\n})\n" }, { "path": "tests/testthat/test_foreach.R", "content": "test_that(\"foreach/seq\", {\n skip_if_not_installed(\"foreach\")\n reg = makeTestRegistry(packages = \"foreach\")\n fun = function(i) {\n foreach(j = 1:i, .combine = c) %dopar% { j^2 }\n }\n ids = batchMap(fun, i = 1:2, reg = reg)\n submitAndWait(reg, ids = ids, resources = list(foreach.backend = \"seq\", ncpus = 2))\n expect_equal(nrow(findDone(reg = reg)), 2L)\n expect_equal(reduceResultsList(reg = reg), list(1, c(1, 4)))\n})\n\ntest_that(\"foreach/multicore\", {\n skip_if_not_installed(\"foreach\")\n skip_if_not_installed(\"doParallel\")\n reg = makeTestRegistry(packages = \"foreach\")\n if (reg$cluster.functions$name %chin% c(\"Parallel\", \"Socket\"))\n skip(\"Nested local parallelization not supported\")\n\n fun = function(i) {\n foreach(j = 1:2) %dopar% { Sys.sleep(3); i }\n }\n ids = batchMap(fun, i = 1, reg = reg)\n\n submitAndWait(reg, ids = ids, resources = list(foreach.backend = \"parallel\", ncpus = 2))\n expect_equal(nrow(findDone(reg = reg)), 1L)\n status = getJobStatus(reg = reg)\n expect_true(status$time.running < 5.9)\n expect_equal(reduceResultsList(reg = reg), list(as.list(c(1, 1))))\n})\n" }, { "path": "tests/testthat/test_future.R", "content": "test_that(\"futures work\", {\n skip_if_not_installed(\"future.batchtools\")\n path = Sys.getenv(\"R_FUTURE_CACHE_PATH\")\n if (!nzchar(path))\n Sys.setenv(R_FUTURE_CACHE_PATH = fs::path(fs::path_temp(), \".future\"))\n\n library(\"future\")\n library(\"future.batchtools\")\n plan(batchtools_local)\n\n pid %<-% { Sys.getpid() }\n expect_count(pid)\n expect_false(pid == Sys.getpid())\n})\n" }, { "path": "tests/testthat/test_getErrorMessages.R", "content": "test_that(\"getErrorMessages\", {\n reg = makeTestRegistry()\n fun = function(i) if (i == 3) stop(\"foobar\") else i\n ids = batchMap(fun, i = 1:5, reg = reg)\n submitAndWait(reg, 1:4)\n\n tab = getErrorMessages(ids, reg = reg)\n expect_data_table(tab, nrow = 5, ncol = 4, key = \"job.id\")\n expect_set_equal(names(tab), c(\"job.id\", \"terminated\", \"error\", \"message\"))\n expect_identical(tab$job.id, 1:5)\n expect_equal(tab$terminated, c(rep(TRUE, 4), FALSE))\n expect_equal(tab$error, replace(logical(5), 3, TRUE))\n expect_character(tab$message)\n expect_equal(is.na(tab$message), !replace(logical(5), 3, TRUE))\n expect_string(tab$message[3], fixed = \"foobar\")\n\n tab = getErrorMessages(ids, missing.as.error = TRUE, reg = reg)\n expect_data_table(tab, nrow = 5, ncol = 4, key = \"job.id\")\n expect_set_equal(names(tab), c(\"job.id\", \"terminated\", \"error\", \"message\"))\n expect_identical(tab$job.id, 1:5)\n expect_equal(tab$terminated, c(rep(TRUE, 4), FALSE))\n expect_equal(tab$error, replace(logical(5), c(3, 5), TRUE))\n expect_character(tab$message)\n expect_equal(is.na(tab$message), !replace(logical(5), c(3, 5), TRUE))\n expect_string(tab$message[3], fixed = \"foobar\")\n expect_string(tab$message[5], fixed = \"[not terminated]\")\n})\n" }, { "path": "tests/testthat/test_getJobTable.R", "content": "test_that(\"getJobTable.Registry\", {\n reg = makeTestRegistry()\n fun = function(i, j) i + j\n ids = batchMap(fun, i = 1:4, j = rep(1, 4), reg = reg)\n\n tab = getJobTable(reg = reg)\n expect_data_table(tab, nrows = 4, ncols = 15, key = \"job.id\")\n expect_list(tab$job.pars)\n expect_equal(tab$job.pars[[1]], list(i = 1L, j = 1))\n\n tab = unwrap(tab)\n expect_data_table(tab, nrows = 4, ncols = 15, key = \"job.id\")\n expect_null(tab[[\"job.pars\"]])\n expect_equal(tab$i, 1:4)\n expect_equal(tab$j, rep(1, 4))\n\n expect_is(tab$submitted, \"POSIXct\")\n expect_is(tab$started, \"POSIXct\")\n expect_is(tab$done, \"POSIXct\")\n expect_is(tab$time.queued, \"difftime\")\n expect_numeric(tab$time.queued, lower = 0)\n expect_is(tab$time.running, \"difftime\")\n expect_numeric(tab$time.running, lower = 0)\n expect_character(tab$tags)\n expect_true(allMissing(tab$tags))\n\n tab = unwrap(getJobTable(reg = reg), sep = \".\")\n expect_null(tab[[\"job.pars\"]])\n expect_equal(tab$job.pars.i, 1:4)\n expect_equal(tab$job.pars.j, rep(1, 4))\n\n # be sure that the original tables are untouched\n checkTables(reg)\n\n submitAndWait(reg = reg, ids = s.chunk(ids), resources = list(my.walltime = 42L))\n addJobTags(2:3, \"my_tag\", reg = reg)\n\n tab = getJobTable(reg = reg)\n expect_data_table(tab, key = \"job.id\")\n expect_copied(tab, reg$status)\n expect_is(tab$submitted, \"POSIXct\")\n expect_is(tab$started, \"POSIXct\")\n expect_is(tab$done, \"POSIXct\")\n expect_is(tab$time.queued, \"difftime\")\n expect_numeric(tab$time.queued, lower = 0)\n expect_is(tab$time.running, \"difftime\")\n expect_numeric(tab$time.running, lower = 0)\n expect_character(tab$tags, min.len = 1L)\n\n tab = getJobResources(reg = reg)\n expect_data_table(tab, nrow = 4, ncols = 2, key = \"job.id\")\n expect_copied(tab, reg$resources)\n expect_set_equal(tab$resource.hash[1], tab$resource.hash)\n expect_list(tab$resources)\n expect_true(all(vlapply(tab$resources, function(r) r$my.walltime == 42)))\n\n tab = unwrap(getJobResources(reg = reg))\n expect_null(tab[[\"resources\"]])\n expect_integer(tab$my.walltime, any.missing = FALSE)\n})\n\ntest_that(\"getJobPars\", {\n reg = makeTestRegistry()\n fun = function(i, j) i + j\n ids = batchMap(fun, i = 1:4, j = rep(1, 4), reg = reg)\n tab = getJobPars(reg = reg)\n expect_data_table(tab, nrow = 4, ncol = 2, key = \"job.id\")\n tab = unwrap(tab)\n expect_copied(tab, reg$defs)\n expect_null(tab$job.pars)\n expect_equal(tab$i, 1:4)\n expect_equal(tab$j, rep(1, 4))\n tab = unwrap(getJobPars(reg = reg, ids = 1:2))\n expect_data_table(tab, nrow = 2, ncol = 3, key = \"job.id\")\n tab = unwrap(getJobPars(reg = reg), sep = \".\")\n expect_data_table(tab, nrow = 4, ncol = 3, key = \"job.id\")\n expect_equal(tab$job.pars.i, 1:4)\n expect_equal(tab$job.pars.j, rep(1, 4))\n})\n\ntest_that(\"getJobPars with repls\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(\"prob\", data = iris, fun = function(data, job) nrow(data), reg = reg)\n algo = addAlgorithm(\"algo\", fun = function(job, data, instance, i, ...) instance, reg = reg)\n prob.designs = list(prob = data.table())\n algo.designs = list(algo = data.table(i = 1:2))\n ids = addExperiments(prob.designs, algo.designs, repls = 3, reg = reg)\n waitForJobs(reg = reg, sleep = 1)\n ids[, chunk := chunk(job.id, chunk.size = 2)]\n submitAndWait(ids = ids, reg = reg)\n expect_equal(nrow(getJobPars(reg = reg)), nrow(ids))\n})\n\ntest_that(\"getJobTable.ExperimentRegistry\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) nrow(data), seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq) instance^sq)\n ids = addExperiments(list(p1 = data.table(k = 1)), list(a1 = data.table(sq = 1:3)), reg = reg)\n\n tab = getJobTable(reg = reg)\n expect_data_table(tab, nrows = 3, ncols = 19, key = \"job.id\")\n expect_copied(tab, reg$status)\n expect_null(tab$job.pars)\n expect_list(tab$prob.pars)\n expect_list(tab$algo.pars)\n for (i in 1:3) {\n expect_equal(tab$prob.pars[[i]], list(k = 1))\n expect_equal(tab$algo.pars[[i]], list(sq = i))\n }\n expect_equal(tab$problem[1], \"p1\")\n expect_equal(tab$algorithm[1], \"a1\")\n\n tab = unwrap(getJobTable(ids = 1:3, reg = reg), c(\"prob.pars\", \"algo.pars\"))\n expect_data_table(tab, nrows = 3, ncols = 19, key = \"job.id\")\n expect_null(tab[[\"job.pars\"]])\n expect_set_equal(tab$k, rep(1, 3))\n expect_set_equal(tab$sq, 1:3)\n\n tab = unwrap(getJobPars(reg = reg), sep = \".\")\n expect_null(tab[[\"job.pars\"]])\n expect_set_equal(tab$prob.pars.k, rep(1, 3))\n expect_set_equal(tab$algo.pars.sq, 1:3)\n})\n\n\ntest_that(\"experiment registry with vector parameters\", {\n tmp = makeTestExperimentRegistry()\n fun = function(job, data, n, mean, sd, ...) rnorm(sum(n), mean = mean, sd = sd)\n addProblem(\"rnorm\", fun = fun, reg = tmp)\n fun = function(instance, ...) sd(instance)\n addAlgorithm(\"deviation\", fun = fun, reg = tmp)\n\n prob.designs = algo.designs = list()\n prob.designs$rnorm = data.table(expand.grid(n = list(100, 1:4), mean = 0, sd = 1:2))\n algo.designs$deviation = data.table()\n addExperiments(prob.designs, algo.designs, reg = tmp)\n submitAndWait(reg = tmp)\n\n res = getJobPars(reg = tmp)\n expect_data_table(res, ncol = 5)\n expect_list(res$prob.pars, len = 4)\n res = unwrap(res)\n expect_data_table(res, ncol = 6, nrow = 4, col.names = \"unique\")\n expect_list(res$n, len = 4)\n expect_numeric(res$mean, len = 4, any.missing = FALSE)\n expect_numeric(res$sd, len = 4, any.missing = FALSE)\n\n res = unwrap(res)\n expect_data_table(res, ncol = 9, nrow = 4, col.names = \"unique\")\n})\n" }, { "path": "tests/testthat/test_getStatus.R", "content": "test_that(\"getStatus\", {\n reg = makeTestRegistry()\n fun = function(i) if (i == 4) stop(\"4!\") else i\n ids = batchMap(fun, i = 1:10, reg = reg)\n submitAndWait(reg, 1:5)\n\n stat = getStatus(reg = reg)\n expect_data_table(stat, any.missing = FALSE, types = \"integer\", nrows = 1L)\n\n expect_equal(stat$defined, 10L)\n expect_equal(stat$submitted, 5L)\n expect_equal(stat$started, 5L)\n expect_equal(stat$done, 4L)\n expect_equal(stat$error, 1L)\n expect_equal(stat$queued, 0L)\n expect_equal(stat$running, 0L)\n expect_equal(stat$system, 0L)\n expect_equal(stat$expired, 0L)\n\n expect_output(print(stat), \"Status for 10 jobs\")\n})\n" }, { "path": "tests/testthat/test_grepLogs.R", "content": "silent({\n reg = makeTestRegistry()\n ids = batchMap(reg = reg, function(x) {\n if (x == 1) {\n print(\"FOOBAR: AAA\")\n } else if (x == 2) {\n cat(\"FOOBAR: BBB\")\n } else {\n if (identical(Sys.getenv(\"TESTTHAT\"), \"true\")) {\n # testthat uses muffle restarts which breaks our internal\n # sink() somehow.\n # https://github.com/r-lib/testthat/issues/460\n cat(\"FOOBAR: CCC\", file = stderr())\n } else {\n message(\"FOOBAR: CCC\")\n }\n }\n invisible(NULL)\n }, x = 1:5)\n ids$chunk = as.integer(c(1, 1, 2, 3, 4))\n submitAndWait(reg, ids[1:4])\n})\n\ntest_that(\"grepLogs\", {\n expect_true(any(grepl(\"AAA\", getLog(1, reg = reg))))\n expect_true(any(grepl(\"BBB\", getLog(2, reg = reg))))\n expect_true(any(grepl(\"CCC\", getLog(3, reg = reg))))\n expect_false(any(grepl(\"AAA\", getLog(2, reg = reg))))\n\n expect_data_table(grepLogs(pattern = \"FOOBAR\", reg = reg), ncol = 2, key = \"job.id\")\n expect_equal(grepLogs(pattern = \"FOOBAR\", reg = reg)$job.id, 1:4)\n expect_equal(grepLogs(pattern = \"XXX\", reg = reg)$job.id, integer(0L))\n expect_error(grepLogs(pattern = \"\", reg = reg), \"at least\")\n expect_error(grepLogs(pattern = NA, reg = reg), \"not be NA\")\n expect_equal(grepLogs(pattern = \"AAA\", reg = reg)$job.id, 1L)\n expect_equal(grepLogs(pattern = \"BBB\", reg = reg)$job.id, 2L)\n expect_equal(grepLogs(pattern = \"CCC\", reg = reg)$job.id, 3:4)\n\n expect_equal(grepLogs(pattern = \"aaa\", reg = reg)$job.id, integer(0L))\n expect_equal(grepLogs(pattern = \"aaa\", ignore.case = TRUE, reg = reg)$job.id, 1L)\n\n expect_data_table(grepLogs(pattern = \"F..BAR\", reg = reg), ncol = 2, nrow = 4, key = \"job.id\")\n expect_data_table(grepLogs(pattern = \"F..BAR\", fixed = TRUE, reg = reg), ncol = 2, nrow = 0, key = \"job.id\")\n\n expect_data_table(grepLogs(1:2, pattern = \"CCC\", reg = reg), nrow = 0, ncol = 2)\n expect_data_table(grepLogs(5, pattern = \"CCC\", reg = reg), nrow = 0, ncol = 2)\n})\n" }, { "path": "tests/testthat/test_hooks.R", "content": "test_that(\"hooks\", {\n reg = makeTestRegistry()\n if (!is.null(reg$cluster.functions$hooks$pre.do.collection) || !is.null(reg$cluster.functions$hooks$post.sync))\n skip(\"Hooks already defined by Cluster Functions\")\n reg$cluster.functions$hooks = insert(reg$cluster.functions$hooks, list(\n \"pre.do.collection\" = function(jc, ...) cat(jc$job.hash, \"\\n\", sep = \"\"),\n \"post.sync\" = function(reg, ...) cat(\"post.syn\", file = fs::path(reg$file.dir, \"post.sync.txt\"))\n ))\n\n jc = makeJobCollection(1, reg = reg)\n expect_function(jc$hooks$pre.do.collection, args = \"jc\")\n\n fn.ps = fs::path(reg$file.dir, \"post.sync.txt\")\n expect_false(fs::file_exists(fn.ps))\n\n batchMap(identity, 1, reg = reg)\n submitAndWait(reg, 1)\n syncRegistry(reg = reg)\n expect_true(fs::file_exists(fn.ps))\n\n lines = getLog(1, reg = reg)\n expect_true(reg$status[1]$job.hash %chin% lines)\n})\n" }, { "path": "tests/testthat/test_joins.R", "content": "test_that(\"joins\", {\n reg = makeTestRegistry()\n batchMap(identity, x = 1:6, reg = reg)\n x = unwrap(getJobPars(reg = reg)[1:5])\n y = findJobs(x >= 2 & x <= 5, reg = reg)\n y$extra.col = head(letters, nrow(y))\n\n res = ijoin(x, y)\n expect_data_table(res, key = \"job.id\", ncol = 3, any.missing = FALSE)\n expect_identical(res$job.id, 2:5)\n expect_copied(res, x)\n\n res = ljoin(as.data.frame(x), y)\n expect_data_table(res, key = \"job.id\", ncol = 3)\n expect_identical(res$job.id, 1:5)\n expect_true(anyMissing(res$extra.col))\n expect_copied(res, x)\n\n res = rjoin(as.data.frame(x), y)\n expect_data_table(res, key = \"job.id\", ncol = 3, any.missing = FALSE)\n expect_identical(res$job.id, 2:5)\n expect_copied(res, x)\n\n res = rjoin(y, x)\n expect_data_table(res, key = \"job.id\", ncol = 3)\n expect_identical(res$job.id, 1:5)\n expect_true(anyMissing(res$extra.col))\n expect_copied(res, x)\n\n res = ojoin(x, y)\n expect_data_table(res, key = \"job.id\", ncol = 3)\n expect_identical(res$job.id, 1:5)\n expect_true(anyMissing(res$extra.col))\n expect_copied(res, x)\n\n res = sjoin(x, y)\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 2:5)\n expect_copied(res, x)\n\n res = sjoin(y, x)\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 2:5)\n expect_copied(res, x)\n\n res = ajoin(x, y)\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 1L)\n expect_copied(res, x)\n\n res = ijoin(x, data.frame(job.id = 2:4))\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 2:4)\n expect_copied(res, x)\n\n res = ijoin(data.frame(job.id = 2:4), x)\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 2:4)\n expect_copied(res, x)\n\n res = ajoin(as.data.frame(x), y)\n expect_data_table(res, key = \"job.id\", ncol = 2, any.missing = FALSE)\n expect_identical(res$job.id, 1L)\n expect_copied(res, x)\n\n res = ujoin(x, y)\n expect_equivalent(res, x)\n expect_copied(res, x)\n\n yy = copy(y)\n yy$x = 10:13\n res = ujoin(x, yy)\n expect_data_table(res, key = \"job.id\", ncol = ncol(x), any.missing = FALSE)\n expect_identical(res$job.id, 1:5)\n expect_identical(res$x, c(1L, 10:13))\n expect_copied(res, x)\n\n res = ujoin(x, yy, all.y = TRUE)\n expect_data_table(res, key = \"job.id\", ncol = 3)\n expect_identical(res$job.id, 1:5)\n expect_identical(res$x, c(1L, 10:13))\n expect_identical(res$extra.col, c(NA, letters[1:4]))\n expect_copied(res, x)\n})\n\ntest_that(\"guessBy\", {\n x = data.frame(id = 1:3, x = 1:3)\n y = data.frame(jid = 1:3, y = 3:1)\n\n expect_error(guessBy(x, y), \"explicitly\")\n expect_error(guessBy(x, y, by = \"id\"), \"subset of\")\n\n by = guessBy(x, y, by = c(id = \"jid\"))\n expect_equal(unname(by), \"jid\")\n expect_equal(names(by), \"id\")\n\n y$id = y$jid\n by = guessBy(x, y, by = \"id\")\n expect_equal(unname(by), \"id\")\n expect_equal(names(by), NULL)\n})\n" }, { "path": "tests/testthat/test_killJobs.R", "content": "test_that(\"killJobs\", {\n reg = makeTestRegistry()\n if (is.null(reg$cluster.functions$killJob))\n skip(\"Test requires killJobs\")\n\n ids = batchMap(Sys.sleep, time = 60, reg = reg)\n silent(submitJobs(1, reg = reg))\n expect_equal(findOnSystem(1, reg = reg), findJobs(reg = reg))\n\n batch.id = reg$status[1, batch.id]\n silent({\n res = killJobs(1, reg = reg)\n })\n expect_equal(res$job.id, 1L)\n expect_equal(res$batch.id, batch.id)\n expect_true(res$killed)\n})\n" }, { "path": "tests/testthat/test_manual.R", "content": "test_that(\"rscimark\", {\n skip(\"manual test\")\n\n reg = makeTestRegistry(package = \"rscimark\")\n reg$cluster.functions = makeClusterFunctionsMulticore(4)\n batchMap(rscimark, minimum.time = rep(1, 5), reg = reg)\n submitJobs(reg = reg)\n waitForJobs(reg = reg, sleep = 1)\n tab = getJobTable(reg = reg)\n expect_true(tab$started[5] >= min(tab$done[1:4]))\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsMulticore(4)\n batchMap(Sys.sleep, rep(3, 4), reg = reg)\n submitJobs(reg = reg)\n waitForJobs(reg = reg, sleep = 1)\n tab = getJobTable(reg = reg)\n expect_true(all(as.numeric(diff(range(tab$started))) <= 2))\n expect_true(all(as.numeric(diff(range(tab$done))) <= 2))\n})\n" }, { "path": "tests/testthat/test_memory.R", "content": "test_that(\"memory measurements work\", {\n skip_on_os(\"windows\")\n skip_on_ci()\n skip_on_cran()\n\n reg = makeTestRegistry()\n reg$cluster.functions = makeClusterFunctionsSSH(list(Worker$new(\"localhost\")))\n ids = batchMap(function(n) { m = matrix(runif(n), nrow = 10); m %*% t(m) }, n = c(100, 1e7), reg = reg)\n submitAndWait(reg, 1:2, resources = list(measure.memory = TRUE))\n\n expect_true(any(stri_detect_fixed(readLog(1L, reg = reg)$lines, \"Memory measurement enabled\")))\n expect_numeric(reg$status$mem.used, any.missing = FALSE)\n expect_true(reg$status$mem.used[2] > reg$status$mem.used[1])\n})\n" }, { "path": "tests/testthat/test_mergeRegistries.R", "content": "test_that(\"mergeRegistries\", {\n target = makeTestRegistry()\n f = function(.job, x) { if (x %in% c(2, 7)) fs::file_create(fs::path(.job$external.dir, \"foo\")); x^2 }\n batchMap(f, 1:10, reg = target)\n\n td = fs::path(target$temp.dir, fs::path_file(fs::file_temp()))\n fs::dir_create(td)\n file.copy(target$file.dir, td, recursive = TRUE)\n file.dir = fs::path(td, fs::path_file(target$file.dir)) # FIXME: dir_copy?\n source = loadRegistry(file.dir, writeable = TRUE, make.default = FALSE)\n\n submitAndWait(target, data.table(job.id = 1:4, chunk = 1L))\n submitAndWait(source, data.table(job.id = 6:9, chunk = c(1L, 1L, 1L, 2L)))\n expect_data_table(findDone(reg = source), nrow = 4)\n expect_data_table(findDone(reg = target), nrow = 4)\n\n mergeRegistries(source, target)\n expect_data_table(findDone(reg = source), nrow = 4)\n expect_data_table(findDone(reg = target), nrow = 8)\n\n checkTables(target)\n\n expect_set_equal(list.files(dir(target, \"external\")), as.character(c(2, 7)))\n expect_equal(unwrap(reduceResultsDataTable(reg = target))$result.1, c(1,2,3,4,6,7,8,9)^2)\n expect_file_exists(fs::path(target$file.dir, \"external\", c(\"2\", \"7\"), \"foo\"))\n\n fs::dir_delete(td)\n})\n" }, { "path": "tests/testthat/test_parallelMap.R", "content": "silent({\n reg = makeTestRegistry()\n fun = function(i) { fun = function(i) i^2; parallelMap::parallelMap(fun, 1:i)}\n ids = batchMap(fun, i = 1:4, reg = reg)\n})\n\ntest_that(\"pm/multicore\", {\n skip_on_os(\"windows\")\n skip_if_not_installed(\"parallelMap\")\n skip_on_ci()\n if (reg$cluster.functions$name %chin% c(\"Parallel\", \"Socket\"))\n skip(\"Nested local parallelization not supported\")\n\n submitAndWait(reg, ids = ids, resources = list(pm.backend = \"multicore\", ncpus = 2))\n expect_equal(nrow(findDone(reg = reg)), 4L)\n})\n\ntest_that(\"pm/socket\", {\n skip_if_not_installed(\"parallelMap\")\n skip_if_not_installed(\"snow\")\n skip_on_ci()\n if (reg$cluster.functions$name %chin% c(\"Parallel\", \"Socket\"))\n skip(\"Nested local parallelization not supported\")\n\n submitAndWait(reg, ids = ids, resources = list(pm.backend = \"socket\", ncpus = 2))\n expect_equal(nrow(findDone(reg = reg)), 4L)\n})\n\ntest_that(\"parallelMap works with batchtools\", {\n skip_if_not_installed(\"parallelMap\")\n skip_if_not(packageVersion(\"parallelMap\") >= \"1.4\")\n requireNamespace(\"parallelMap\")\n\n dir = reg$temp.dir %??% fs::path_temp()\n parallelMap::parallelStartBatchtools(storagedir = dir, show.info = FALSE)\n dir = getOption(\"parallelMap.bt.reg.filedir\")\n res = parallelMap::parallelMap(function(x, y) x + y, x = 1:2, y = 1)\n parallelMap::parallelStop()\n if (fs::dir_exists(dir))\n fs::dir_delete(dir)\n expect_equal(res, list(2, 3))\n})\n\n# test_that(\"pm/mpi\", {\n# skip_on_os(\"mac\")\n# skip_on_cran()\n# skip_if_not_installed(\"parallelMap\")\n# skip_if_not_installed(\"snow\")\n# skip_if_not_installed(\"Rmpi\")\n# skip_on_ci()\n# if (reg$cluster.functions$name %chin% c(\"Parallel\", \"Socket\"))\n# skip(\"Nested local parallelization not supported\")\n\n# submitAndWait(reg, ids = ids, resources = list(pm.backend = \"mpi\", ncpus = 2))\n# expect_equal(nrow(findDone(reg = reg)), 4)\n# })\n" }, { "path": "tests/testthat/test_reduceResults.R", "content": "suppressMessages({\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, a = 1:4, b = 4:1, reg = reg)\n submitAndWait(reg, 1:3)\n})\n\ntest_that(\"loadResult\", {\n expect_equal(loadResult(reg = reg, 1), list(a = 1, b = 4))\n expect_equal(loadResult(reg = reg, 2), list(a = 2, b = 3))\n expect_error(loadResult(reg = reg, 4), \"not terminated\")\n})\n\ntest_that(\"batchMapResults\", {\n target = makeTestRegistry()\n x = batchMapResults(target = target, function(x, c, d) x$a+x$b + c + d, c = 11:13, source = reg, more.args = list(d = 2))\n expect_data_table(x, nrow = 3, key = \"job.id\")\n expect_data_table(target$status, nrow = 3)\n submitAndWait(target)\n res = unwrap(reduceResultsDataTable(reg = target))\n expect_equal(res[[2L]], 11:13 + rep(5, 3) + 2)\n})\n\ntest_that(\"reduceResults\", {\n silent({\n expect_equal(reduceResults(fun = function(aggr, res, ...) c(aggr, res$a), init = integer(0), reg = reg), 1:3)\n expect_equal(reduceResults(ids = 1, fun = c, reg = reg), list(a = 1, b = 4))\n expect_equal(reduceResults(ids = 1, fun = c, list(c = 1), reg = reg)$c, 1)\n expect_equal(reduceResults(fun = function(aggr, res, extra.arg, ...) aggr + res$a + extra.arg, init = 0, extra.arg = 1, reg = reg), sum(1:3 + 1))\n expect_equal(reduceResults(fun = function(job, aggr, res) c(aggr, job$id), init = integer(0), ids = 2:3, reg = reg), 2:3)\n expect_list(reduceResults(fun = function(job, aggr, res) c(aggr, list(job)), init = list(), ids = 2:3, reg = reg), types = \"Job\", len = 2)\n\n expect_equal(\n reduceResults(fun = function(aggr, res, ...) c(aggr, res$a), ids = 3:1, init = integer(0), reg = reg),\n rev(reduceResults(fun = function(aggr, res, ...) c(aggr, res$a), ids = 1:3, init = integer(0), reg = reg))\n )\n expect_error(reduceResults(fun = function(aggr, res, ...) c(aggr, res$a), ids = 1:4, init = integer(0), reg = reg),\n \"successfully computed\")\n })\n})\n\ntest_that(\"reduceResultsList\", {\n silent({\n expect_equal(reduceResultsList(reg = reg), Map(fun, a = 1:3, b = 4:2))\n expect_equal(reduceResultsList(reg = reg, fun = function(x) x$a), as.list(1:3))\n expect_equal(reduceResultsList(reg = reg, fun = function(x, y) x$a + y, y = 1), as.list(1:3 + 1))\n expect_list(reduceResultsList(reg = reg, fun = function(job, ...) job), types = \"Job\", len = 3)\n expect_equal(reduceResultsList(ids = 2:1, reg = reg), rev(reduceResultsList(ids = 1:2, reg = reg)))\n })\n})\n\ntest_that(\"reduceResultsDataTable\", {\n silent({\n tab = unwrap(reduceResultsDataTable(reg = reg))\n expect_data_table(tab, nrow = 3, ncol = 3, key = \"job.id\")\n expect_null(tab$result)\n expect_equal(tab$a, 1:3)\n\n tab = unwrap(reduceResultsDataTable(reg = reg, fun = function(x) list(a = x$a)))\n expect_data_table(tab, nrow = 3, ncol = 2, key = \"job.id\")\n expect_equal(tab$a, 1:3)\n\n tab = unwrap(reduceResultsDataTable(reg = reg, ids = 3:2, fun = function(x) list(a = x$a)))\n expect_data_table(tab, nrow = 2, ncol = 2, key = \"job.id\")\n expect_equal(tab$a, 2:3)\n\n tab = unwrap(reduceResultsDataTable(reg = reg, fun = function(x) x$a))\n expect_data_table(tab, nrow = 3, ncol = 2, key = \"job.id\")\n expect_equal(tab$result.1, 1:3)\n\n tab = unwrap(reduceResultsDataTable(reg = reg, fun = function(x, y) x$a + y, y = 1))\n expect_data_table(tab, nrow = 3, ncol = 2, key = \"job.id\")\n expect_equal(tab$result.1, 1:3 + 1L)\n })\n})\n\ntest_that(\"reduceResultsDataTable/unwrap simple\", {\n silent({\n tab = reduceResultsDataTable(reg = reg)\n expect_data_table(tab, nrow = 3, ncol = 2, key = \"job.id\")\n expect_set_equal(names(tab), c(\"job.id\", \"result\"))\n expect_list(tab$result[[1]], types = \"numeric\", len = 2)\n tab = unwrap(tab)\n expect_data_table(tab, ncol = 3)\n expect_equal(tab$job.id, 1:3)\n expect_equal(tab$a, 1:3)\n expect_equal(tab$b, 4:2)\n })\n})\n\nsuppressMessages({\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", fun = function(job, data, ...) 2, seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq) instance^sq)\n ids = addExperiments(list(p1 = data.table()), list(a1 = data.table(sq = 1:3)), reg = reg)\n submitAndWait(reg = reg)\n})\n\ntest_that(\"reduceResults/BatchExperiments\", {\n silent({\n expect_equal(reduceResults(fun = function(aggr, res, ...) c(aggr, res), init = integer(0), reg = reg), 2^(1:3))\n expect_equal(reduceResults(ids = 2:3, fun = function(aggr, job, res, ...) c(aggr, job$id), init = integer(0), reg = reg), 2:3)\n expect_list(reduceResults(fun = function(job, aggr, res) c(aggr, list(job)), init = list(), ids = 2:3, reg = reg), types = \"Experiment\", len = 2)\n })\n})\n\ntest_that(\"reduceResultsList/BatchExperiments\", {\n silent({\n expect_equal(reduceResultsList(reg = reg), as.list(2^(1:3)))\n expect_equal(reduceResultsList(fun = function(job, ...) job$prob.name, reg = reg), as.list(rep(\"p1\", 3)))\n expect_equal(reduceResultsList(fun = function(job, ...) job$algo.name, reg = reg), as.list(rep(\"a1\", 3)))\n expect_equal(reduceResultsList(fun = function(job, ...) job$instance, reg = reg), as.list(rep(2, 3)))\n })\n})\n\ntest_that(\"reduceResults with no results reg\", {\n silent({\n reg = makeTestRegistry()\n\n expect_equal(reduceResults(fun = c, reg = reg), NULL)\n expect_equal(reduceResults(fun = c, reg = reg, init = 42), 42)\n expect_equal(reduceResultsList(reg = reg), list())\n\n fun = function(...) list(...)\n ids = batchMap(fun, a = 1:3, b = 3:1, reg = reg)\n\n expect_equal(reduceResults(fun = c, reg = reg), NULL)\n expect_equal(reduceResults(fun = c, reg = reg, init = 42), 42)\n expect_equal(reduceResultsList(reg = reg), list())\n })\n})\n\ntest_that(\"reduceResultsList/NULL\", {\n reg = makeTestRegistry()\n f = function(...) NULL\n ids = batchMap(f, 1:3, reg = reg)\n submitAndWait(ids, reg = reg)\n res = reduceResultsList(ids = ids, reg = reg)\n expect_equal(res, replicate(3, NULL, simplify = FALSE))\n})\n\ntest_that(\"reduceResultsDataTable/multiRowResults\", {\n silent({\n reg = makeTestRegistry()\n fun = function(a) data.frame(y1 = rep(a, 3), y2 = rep(a/2, 3))\n ids = batchMap(fun, a = c(10, 100), reg = reg)\n submitAndWait(reg, ids)\n res = reduceResultsDataTable(reg = reg)\n expect_data_frame(res, ncol = 2, nrow = 2)\n expect_list(res$result, types = \"data.frame\", len = 2L)\n })\n})\n\ntest_that(\"reduceResultsDataTable/unwrap objects\", {\n silent({\n reg = makeTestRegistry()\n fun = function(...) iris\n ids = batchMap(fun, i = 1:2, reg = reg)\n submitAndWait(reg, 1:2)\n tab = reduceResultsDataTable(reg = reg)\n expect_data_table(tab, nrow = 2, ncol = 2, key = \"job.id\")\n expect_set_equal(names(tab), c(\"job.id\", \"result\"))\n expect_list(tab$result, types = \"data.frame\", names = \"unnamed\")\n })\n})\n" }, { "path": "tests/testthat/test_removeExperiments.R", "content": "test_that(\"removeExperiments\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) nrow(data), seed = 42)\n prob = addProblem(reg = reg, \"p2\", data = iris, fun = function(job, data) nrow(data), seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq) instance^sq)\n algo = addAlgorithm(reg = reg, \"a2\", fun = function(job, data, instance, sq) instance^sq)\n ids = addExperiments(list(p1 = data.table(), p2 = data.table(x = 1:2)), list(a1 = data.table(sq = 1:3), a2 = data.table(sq = 1:2)), reg = reg)\n N = nrow(findExperiments(reg = reg))\n\n expect_data_table(removeExperiments(1, reg = reg), nrow = 1, key = \"job.id\")\n expect_equal(findExperiments(reg = reg)$job.id, 2:N)\n expect_data_table(removeExperiments(1, reg = reg), nrow = 0, key = \"job.id\")\n expect_equal(findExperiments(reg = reg)$job.id, 2:N)\n\n ids = findExperiments(prob.name = \"p1\", reg = reg)\n expect_data_table(removeExperiments(ids, reg = reg), nrow = 4, key = \"job.id\")\n expect_equal(findExperiments(reg = reg)$job.id, 6:N)\n expect_true(fs::file_exists(getProblemURI(reg, \"p1\")))\n expect_set_equal(c(\"p1\", \"p2\"), reg$problems)\n\n ids = findExperiments(algo.name = \"a2\", reg = reg)\n expect_data_table(removeExperiments(ids, reg = reg), nrow = 4, key = \"job.id\")\n expect_equal(findExperiments(reg = reg)$job.id, 6:(N-nrow(ids)))\n expect_true(fs::file_exists(getAlgorithmURI(reg, \"a2\")))\n expect_set_equal(c(\"a1\", \"a2\"), reg$algorithms)\n\n checkTables(reg)\n})\n" }, { "path": "tests/testthat/test_removeRegistry.R", "content": "test_that(\"removeRegistry\", {\n reg = makeTestRegistry()\n expect_directory_exists(reg$file.dir)\n res = removeRegistry(0.01, reg = reg)\n expect_string(res)\n expect_false(fs::dir_exists(reg$file.dir))\n})\n\ntest_that(\"removeRegistry resets default registry\", {\n prev = batchtools$default.registry\n reg = makeTestExperimentRegistry(make.default = TRUE)\n expect_is(batchtools$default.registry, \"Registry\")\n res = removeRegistry(0, reg = reg)\n expect_false(fs::dir_exists(reg$file.dir))\n expect_null(batchtools$default.registry)\n batchtools$default.registry = prev\n})\n" }, { "path": "tests/testthat/test_resetJobs.R", "content": "test_that(\"resetJobs\", {\n reg = makeTestRegistry()\n f = function(x, .job) if (x == 2) stop(2) else .job$external.dir\n batchMap(f, 1:3, reg = reg)\n before = list(\n status = copy(reg$status),\n defs = copy(reg$defs)\n )\n submitAndWait(reg, 1:3)\n\n expect_file_exists(getLogFiles(reg, 3))\n expect_false(identical(reg$status$submitted, before$status$submitted))\n expect_file_exists(getResultFiles(reg, 1))\n expect_equal(unname(fs::dir_exists(fs::path(reg$file.dir, \"external\", 1:3))), c(TRUE, FALSE, TRUE))\n\n resetJobs(1, reg = reg)\n expect_true(all.equal(before$status[1], reg$status[1]))\n expect_false(fs::file_exists(getResultFiles(reg, 1)))\n expect_true(fs::file_exists(getResultFiles(reg, 3)))\n expect_file_exists(getLogFiles(reg, 3))\n expect_equal(unname(fs::dir_exists(fs::path(reg$file.dir, \"external\", 1:3))), c(FALSE, FALSE, TRUE))\n expect_false(fs::file_exists(getResultFiles(reg, 1)))\n expect_file_exists(getResultFiles(reg, 3))\n\n resetJobs(2:3, reg = reg)\n expect_data_table(reg$status, key = \"job.id\")\n expect_data_table(reg$defs, key = \"def.id\")\n expect_equivalent(before$status, reg$status)\n expect_false(fs::file_exists(getLogFiles(reg, 3)))\n expect_false(fs::file_exists(getResultFiles(reg, 3)))\n expect_equal(unname(fs::dir_exists(fs::path(reg$file.dir, \"external\", 1:3))), c(FALSE, FALSE, FALSE))\n})\n\n\ntest_that(\"functions produce error after resetting jobs\", {\n reg = makeTestRegistry()\n f = function(x, .job) if (x == 2) stop(2) else .job$external.dir\n batchMap(f, 1:3, reg = reg)\n submitAndWait(reg, 1:3)\n\n resetJobs(1, reg = reg)\n expect_error(getLog(1, reg = reg), \"not available\")\n expect_error(loadResult(1, reg = reg), \"not terminated\")\n})\n" }, { "path": "tests/testthat/test_runOSCommand.R", "content": "test_that(\"runOSCommand\", {\n skip_on_os(c(\"windows\", \"solaris\")) # system2 is broken on solaris\n x = runOSCommand(\"ls\", find.package(\"batchtools\"))\n expect_list(x, names = \"named\", len = 4L)\n expect_names(names(x), permutation.of = c(\"sys.cmd\", \"sys.args\", \"exit.code\", \"output\"))\n expect_identical(x$exit.code, 0L)\n expect_true(all(c(\"DESCRIPTION\", \"NAMESPACE\", \"NEWS.md\") %chin% x$output))\n})\n\ntest_that(\"command not found\", {\n skip_on_os(\"solaris\") # system2 is broken on solaris\n res = runOSCommand(\"notfoundcommand\")\n expect_list(res, len = 4)\n expect_identical(res$exit.code, 127L)\n expect_identical(res$output, \"command not found\")\n expect_error(OSError(\"Command not found\", res), pattern = \"Command not found\")\n expect_error(OSError(\"Command not found\", res), pattern = \"'notfoundcommand'\")\n expect_error(OSError(\"Command not found\", res), pattern = \"exit code 127\")\n})\n\ntest_that(\"stdin\", {\n skip_on_os(c(\"windows\", \"solaris\")) # system2 is broken on solaris\n\n tf = fs::file_temp()\n lines = letters\n writeLines(letters, con = tf)\n res = runOSCommand(\"cat\", stdin = tf)\n expect_identical(res$exit.code, 0L)\n expect_identical(res$output, letters)\n fs::file_delete(tf)\n})\n" }, { "path": "tests/testthat/test_seed.R", "content": "test_that(\"with_seed\", {\n set.seed(1)\n x.1 = runif(5)\n set.seed(42)\n x.42 = runif(5)\n x.next = runif(5)\n\n set.seed(42)\n y.1 = withr::with_seed(1, runif(5))\n y.42 = runif(5)\n y.next = runif(5)\n\n expect_identical(x.1, y.1)\n expect_identical(x.42, y.42)\n expect_identical(x.next, y.next)\n expect_error(withr::with_seed(1, print(state)))\n})\n\ntest_that(\"Problem and Algorithm seed\", {\n reg = makeTestExperimentRegistry(seed = 42)\n addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, ...) runif(1), seed = 1L)\n addProblem(reg = reg, \"p2\", data = iris, fun = function(job, data, ...) runif(1))\n addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) list(instance = instance, res = runif(1)))\n addAlgorithm(reg = reg, \"a2\", fun = function(job, data, instance, ...) list(instance = instance, res = runif(1)))\n prob.designs = list(p1 = data.table(), p2 = data.table())\n algo.designs = list(a1 = data.table(), a2 = data.table())\n repls = 3\n ids = addExperiments(prob.designs, algo.designs, repls = repls, reg = reg)\n\n submitAndWait(reg, ids)\n\n set.seed(1); p1 = runif(1)\n set.seed(2); p2 = runif(1)\n set.seed(3); p3 = runif(1)\n set.seed(43); a1 = runif(1)\n set.seed(44); a2 = runif(1)\n set.seed(45); a3 = runif(1)\n silent({\n ids = findExperiments(algo.name = \"a1\", prob.name = \"p1\", reg = reg)\n results = rbindlist(reduceResultsList(ids, reg = reg), use.names = TRUE)\n })\n expect_true(all(results$instance == c(p1, p2, p3)))\n expect_true(all(results$res == c(a1, a2, a3)))\n\n silent({\n ids = findExperiments(prob.name = \"p1\", repl = 2, reg = reg)\n results = rbindlist(reduceResultsList(ids, reg = reg), use.names = TRUE)\n })\n expect_true(all(results$instance == p2))\n\n silent({\n ids = findExperiments(prob.name = \"p2\", reg = reg)\n results = rbindlist(reduceResultsList(ids, reg = reg), use.names = TRUE)\n })\n expect_numeric(results$instance, unique = TRUE)\n expect_numeric(results$res, unique = TRUE)\n})\n\ntest_that(\"Seed is correctly reported (#203)\", {\n reg = makeTestRegistry(seed = 1)\n batchMap(function(x, .job) list(seed = .job$seed), x = 1:3, reg = reg)\n submitAndWait(reg)\n res = unwrap(reduceResultsDataTable(reg = reg))\n expect_data_table(res, nrow = 3, ncol = 2)\n expect_identical(res$seed, 2:4)\n\n expect_true(any(stri_detect_fixed(getLog(1, reg = reg), \"Setting seed to 2\")))\n expect_true(any(stri_detect_fixed(getLog(2, reg = reg), \"Setting seed to 3\")))\n expect_true(any(stri_detect_fixed(getLog(3, reg = reg), \"Setting seed to 4\")))\n\n\n reg = makeTestExperimentRegistry(seed = 1)\n addProblem(reg = reg, \"p1\", fun = function(job, ...) job$seed, seed = 100L)\n addAlgorithm(reg = reg, \"a1\", fun = function(job, instance, ...) list(instance = instance, seed = job$seed))\n ids = addExperiments(repls = 2, reg = reg)\n getStatus(reg = reg)\n submitAndWait(reg)\n\n res = unwrap(reduceResultsDataTable(reg = reg))\n expect_data_table(res, nrow = 2, ncol = 3)\n expect_identical(res$instance, 2:3)\n expect_identical(res$seed, 2:3)\n\n expect_true(any(stri_detect_fixed(getLog(1, reg = reg), \"seed = 2\")))\n expect_true(any(stri_detect_fixed(getLog(2, reg = reg), \"seed = 3\")))\n})\n" }, { "path": "tests/testthat/test_showLog.R", "content": "test_that(\"showLog/getLog\", {\n reg = makeTestRegistry()\n batchMap(function(x) print(\"GREPME\"), 1:2, reg = reg)\n expect_error(showLog(id = 1, reg = reg), \"not available\")\n expect_error(readLog(id = data.table(job.id = 1L), reg = reg), \"not available\")\n submitAndWait(reg)\n\n lines = getLog(id = 1, reg = reg)\n expect_character(lines, min.len = 3L, any.missing = FALSE)\n expect_equal(sum(stri_detect_fixed(lines, \"GREPME\")), 1L)\n expect_true(any(stri_startswith_fixed(lines, \"### [bt\")))\n expect_identical(sum(stri_endswith_fixed(lines, \"[batchtools job.id=1]\")), 2L)\n expect_false(any(stri_endswith_fixed(lines, \"[batchtools job.id=2]\")))\n\n lines = getLog(id = 2, reg = reg)\n expect_false(any(stri_endswith_fixed(lines, \"[batchtools job.id=1]\")))\n\n withr::with_options(list(pager = function(files, header, title, delete.file) files), {\n x = showLog(id = 2, reg = reg)\n expect_equal(fs::path_file(x), \"2.log\")\n expect_equal(sum(stri_detect_fixed(readLines(x), \"GREPME\")), 1L)\n })\n\n expect_error(getLog(id = 1:2, reg = reg), \"exactly\")\n expect_error(getLog(id = 3, reg = reg), \"exactly\")\n})\n\ntest_that(\"empty log files\", {\n reg = makeTestRegistry()\n batchMap(identity, 1, reg = reg)\n submitAndWait(reg)\n\n # overwrite log file\n log.file = getLogFiles(reg, 1)\n fs::file_delete(log.file)\n fs::file_create(log.file)\n\n x = readLog(data.table(job.id = 1), reg = reg)\n expect_data_table(x, ncol = 2, nrow = 0, index = \"job.id\")\n\n expect_equal(getLog(1, reg = reg), character(0L))\n})\n" }, { "path": "tests/testthat/test_sleep.R", "content": "test_that(\"getSleepFunction\", {\n reg = makeTestRegistry()\n f = getSleepFunction(reg, NULL)\n expect_function(f)\n expect_true(any(grepl(\"Sys.sleep\", as.character(body(f)))))\n\n f = getSleepFunction(reg, 99)\n expect_function(f)\n expect_true(any(grepl(\"Sys.sleep\", as.character(body(f)))))\n\n f = getSleepFunction(reg, function(x) x^2)\n expect_function(f)\n expect_true(any(grepl(\"Sys.sleep\", as.character(body(f)))))\n})\n" }, { "path": "tests/testthat/test_submitJobs.R", "content": "test_that(\"submitJobs\", {\n reg = makeTestRegistry()\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, reg = reg)\n\n submitAndWait(reg, 1:2, resources = list(foo = \"bar\"))\n checkTables(reg)\n\n expect_integer(reg$status[1:2, resource.id], any.missing = FALSE)\n expect_character(reg$status[1:2, batch.id], any.missing = FALSE)\n expect_numeric(reg$status[1:2, submitted], any.missing = FALSE)\n expect_true(is.na(reg$status[3, submitted]))\n x = reg$resources[1, resources][[1L]]\n y = insert(reg$default.resources, list(foo = \"bar\"))\n if (isTRUE(y$chunks.as.arrayjobs) && is.na(reg$cluster.functions$array.var))\n y$chunks.as.arrayjobs = NULL\n expect_equal(x[order(names2(x))], y[order(names2(y))])\n\n submitAndWait(reg, 3, resources = list(walltime = 100, memory = 500))\n res = reg$resources[2, resources][[1L]]\n expect_equal(res$walltime, 100)\n expect_equal(res$memory, 500)\n\n # should be 2 chunks?\n expect_equal(uniqueN(reg$status$job.hash), 2)\n})\n\ntest_that(\"per job resources\", {\n reg = makeTestRegistry()\n\n fun = function(...) list(...)\n ids = batchMap(fun, i = 1:3, reg = reg)\n ids$walltime = as.integer(c(180, 120, 180))\n ids$chunk = 1:3\n\n submitAndWait(reg, ids = ids)\n\n res = reg$resources\n expect_data_table(res, nrow = 2)\n expect_equal(uniqueN(res, by = \"resource.hash\"), 2L)\n expect_set_equal(rbindlist(res$resources)$walltime, c(120L, 180L))\n\n ids$chunk = 1L\n expect_error(submitJobs(ids, reg = reg), \"per-job\")\n})\n" }, { "path": "tests/testthat/test_summarizeExperiments.R", "content": "test_that(\"summarizeExperiments\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data) nrow(data), seed = 42)\n prob = addProblem(reg = reg, \"p2\", data = iris, fun = function(job, data) nrow(data), seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq) instance^sq)\n ids = addExperiments(list(p1 = data.table(), p2 = data.table(x = 1:2)), list(a1 = data.table(sq = 1:3)), reg = reg)\n s = summarizeExperiments(reg = reg)\n expect_data_table(s, nrows = 2, ncols = 3)\n expect_equal(s$.count, c(3, 6))\n expect_equal(s$problem, c(\"p1\", \"p2\"))\n expect_equal(s$algorithm, c(\"a1\", \"a1\"))\n\n s = summarizeExperiments(reg = reg, by = c(\"problem\", \"algorithm\", \"x\"))\n expect_data_table(s, nrows = 3, ncols = 4)\n expect_equal(s$.count, c(3, 3, 3))\n expect_equal(s$x, c(NA, 1, 2))\n})\n" }, { "path": "tests/testthat/test_sweepRegistry.R", "content": "test_that(\"sweepRegistry\", {\n reg = makeTestRegistry()\n array.jobs = isTRUE(reg$default.resources$chunks.as.arrayjobs)\n batchMap(identity, 1, reg = reg)\n\n submitAndWait(reg, 1, resources = list(foo = 1))\n submitAndWait(reg, 1, resources = list(foo = 2))\n writeRDS(makeJobCollection(1, reg = reg), fs::path(reg$file.dir, \"jobs\", \"test.rds\"))\n\n expect_data_table(reg$resources, nrow = 2)\n expect_character(list.files(dir(reg, \"logs\")), len = 2L)\n expect_character(list.files(fs::path(reg$file.dir, \"jobs\"), pattern = \"\\\\.rds$\"), len = 1L + (array.jobs && reg$cluster.functions$store.job.collection) * 2L)\n expect_character(list.files(fs::path(reg$file.dir, \"jobs\"), pattern = \"\\\\.job$\"), len = (batchtools$debug && array.jobs) * 2L)\n\n expect_true(sweepRegistry(reg))\n\n expect_data_table(reg$resources, nrow = 1)\n expect_character(list.files(dir(reg, \"logs\")), len = 1L)\n if (reg$cluster.functions$store.job.collection)\n expect_character(list.files(fs::path(reg$file.dir, \"jobs\")), len = 0L)\n checkTables(reg)\n\n\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, ...) nrow(data))\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, ...) NULL)\n addExperiments(prob.designs = list(p1 = data.table(i = 1:10)), reg = reg)\n addJobTags(6:10, \"foo\", reg = reg)\n expect_data_table(reg$tags, nrow = 5, any.missing = FALSE)\n removeExperiments(ids = 6:10, reg = reg)\n expect_data_table(reg$tags, nrow = 0)\n\n checkTables(reg)\n})\n\ntest_that(\"relative paths work (#113)\", {\n skip_on_cran()\n skip_on_os(\"windows\")\n skip_if_not(is.null(getSysConf()$temp.dir)) # we are probably on a system where home is not shared\n fd = sprintf(\"~/batchtools-test-%s\", fs::path_file(fs::file_temp(\"\")))\n reg = makeTestExperimentRegistry(file.dir = fd)\n\n problems = list(\"a\", \"b\")\n pdes = lapply(problems, function(p) {\n addProblem(name = p, data = p, fun = function(...) list(...), reg = reg)\n res = data.frame(fold = 1:3)\n })\n names(pdes) = problems\n\n algo.rep1 = function(job, data, instance, x) { rep(paste(data, x), instance$fold) }\n algo.rep2 = function(job, data, instance, x) { rep(paste(data, x), instance$fold) }\n addAlgorithm(name = \"rep1\", fun = algo.rep1, reg = reg)\n addAlgorithm(name = \"rep2\", fun = algo.rep2, reg = reg)\n\n ades = list(\n rep1 = data.table(x = LETTERS[1:3]),\n rep2 = data.table(x = letters[1:3])\n )\n\n addExperiments(pdes, ades, reg = reg)\n submitAndWait(reg = reg)\n\n ids.rep1 = findExperiments(algo.name = \"rep1\", reg = reg)\n ids.rep2 = findExperiments(algo.name = \"rep2\", reg = reg)\n removeExperiments(ids.rep2, reg = reg)\n\n expect_character(getLog(ids.rep1[1], reg = reg), min.len = 1, any.missing = FALSE)\n expect_list(reduceResultsList(ids = ids.rep1, reg = reg), len = 18)\n\n checkTables(reg)\n\n fs::dir_delete(fs::path_expand(fd))\n})\n" }, { "path": "tests/testthat/test_tags.R", "content": "test_that(\"tags work\", {\n reg = makeTestRegistry()\n batchMap(identity, 1:10, reg = reg)\n expect_equal(getUsedJobTags(reg = reg), character())\n expect_data_table(findTagged(tag = \"foo\", reg = reg), nrow = 0, ncol = 1)\n expect_data_table(removeJobTags(reg = reg, tags = \"foo\"), nrow = 0, ncol = 1)\n\n expect_data_table(addJobTags(1:4, \"walltime\", reg = reg), nrow = 4, key = \"job.id\")\n expect_data_table(addJobTags(3:7, \"broken\", reg = reg), nrow = 5, key = \"job.id\")\n expect_set_equal(getUsedJobTags(reg = reg), c(\"walltime\", \"broken\"))\n expect_set_equal(getUsedJobTags(1:2, reg = reg), c(\"walltime\"))\n\n addJobTags(tags = c(\"foo\", \"bar\"), reg = reg)\n x = getJobTags(reg = reg)\n expect_true(all(stri_detect_fixed(x$tags, \"foo\")))\n expect_true(all(stri_detect_fixed(x$tags, \"bar\")))\n x = removeJobTags(tags = c(\"foo\", \"bar\"), reg = reg)\n expect_data_table(x, ncol = 1, nrow = 10, key = \"job.id\")\n x = getJobTags(reg = reg)\n expect_false(any(stri_detect_fixed(x$tags, \"foo\"), na.rm = TRUE))\n expect_false(any(stri_detect_fixed(x$tags, \"bar\"), na.rm = TRUE))\n\n x = getJobTags(reg = reg)\n expect_data_table(x, nrow = 10, ncol = 2, key = \"job.id\")\n expect_character(x$tags, min.len = 1L)\n\n x = findTagged(tags = \"broken\", reg = reg)\n expect_data_table(x, nrow = 5, ncol = 1, key = \"job.id\")\n expect_equal(x$job.id, 3:7)\n\n x = findTagged(tags = \"whoops\", reg = reg)\n expect_data_table(x, nrow = 0, ncol = 1, key = \"job.id\")\n\n x = removeJobTags(9:3, \"walltime\", reg = reg)\n expect_data_table(x, ncol = 1, nrow = 2, key = \"job.id\")\n expect_equal(x$job.id, 3:4)\n x = getJobTags(reg = reg)\n expect_equal(x$tags, c(rep(\"walltime\", 2), rep(\"broken\", 5), rep(NA_character_, 3)))\n\n checkTables(reg)\n})\n" }, { "path": "tests/testthat/test_testJob.R", "content": "test_that(\"testJob\", {\n reg = makeTestRegistry()\n f = function(x) if (x %% 2 == 0) stop(\"foo\") else x^2\n batchMap(reg = reg, f, 1:3)\n expect_equal(testJob(reg = reg, id = 1), 1)\n expect_equal(testJob(reg = reg, id = 3), 9)\n expect_error(testJob(reg = reg, id = 2), \"foo\")\n\n expect_equal(suppressAll(testJob(reg = reg, id = 1, external = TRUE)), 1)\n expect_error(suppressAll(testJob(reg = reg, id = 2, external = TRUE)), \"re-run\")\n\n expect_equal(findSubmitted(reg = reg), data.table(job.id = integer(0L), key = \"job.id\"))\n expect_equal(findDone(reg = reg), data.table(job.id = integer(0L), key = \"job.id\"))\n expect_equal(findErrors(reg = reg), data.table(job.id = integer(0L), key = \"job.id\"))\n})\n\ntest_that(\"testJob.ExperimentRegistry\", {\n reg = makeTestExperimentRegistry()\n prob = addProblem(reg = reg, \"p1\", data = iris, fun = function(job, data, ...) nrow(data), seed = 42)\n algo = addAlgorithm(reg = reg, \"a1\", fun = function(job, data, instance, sq, ...) instance^sq)\n ids = addExperiments(prob.designs = list(p1 = data.table()), algo.designs = list(a1 = data.table(sq = 1:3)), reg = reg)\n\n suppressAll(x <- testJob(id = 1, reg = reg))\n expect_equal(x, 150)\n suppressAll(x <- testJob(id = 2, reg = reg, external = TRUE))\n expect_equal(x, 150^2)\n})\n\ntest_that(\"traceback works in external session\", {\n reg = makeTestRegistry()\n f = function(x) {\n g = function(x) findme(x)\n findme = function(x) h(x)\n h = function(x) stop(\"Error in h\")\n g(x)\n }\n batchMap(f, 1, reg = reg)\n expect_output(expect_error(testJob(1, external = TRUE, reg = reg), \"external=FALSE\"), \"findme\")\n})\n" }, { "path": "tests/testthat/test_unwrap.R", "content": "test_that(\"unwrap behaves\", {\n x = data.table(\n id = 1:3,\n nested.list = list(list(a = 1), list(a = 2), list(a = 33)),\n nested.2dlist = list(list(a = 1, b = 2), list(a = 1), list(b = 2)),\n nested.df = list(data.frame(a = 1, b = 2), data.frame(a = 1), data.frame(b = 2)),\n multi.row = list(data.frame(a = 1:2, b = 1:2), data.frame(a = 3:4, b = 3:4), data.frame(a = 1:2, b = 3:4)),\n empty = list(NULL, NULL, NULL)\n )\n\n cols = \"nested.list\"\n res = unwrap(x, cols)\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x), col.names = \"unique\", any.missing = FALSE)\n expect_equal(names(res), c(\"id\", \"nested.2dlist\", \"nested.df\", \"multi.row\", \"empty\", \"a\"))\n expect_numeric(res[[\"a\"]])\n\n cols = \"nested.list\"\n res = unwrap(x, cols, sep = \".\")\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x), col.names = \"unique\", any.missing = FALSE)\n expect_equal(names(res), c(\"id\", \"nested.2dlist\", \"nested.df\", \"multi.row\", \"empty\", \"nested.list.a\"))\n expect_numeric(res[[\"nested.list.a\"]])\n\n cols = \"nested.2dlist\"\n res = unwrap(x, cols)\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x) + 1L, col.names = \"unique\", any.missing = TRUE)\n expect_equal(names(res), c(\"id\", \"nested.list\", \"nested.df\", \"multi.row\", \"empty\", \"a\", \"b\"))\n expect_numeric(res[[\"a\"]])\n expect_numeric(res[[\"b\"]])\n\n cols = \"nested.2dlist\"\n res = unwrap(x, cols, sep = \"_\")\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x) + 1L, col.names = \"unique\", any.missing = TRUE)\n expect_set_equal(names(res), c(\"id\", \"nested.list\", \"nested.2dlist_a\", \"nested.2dlist_b\", \"nested.df\", \"multi.row\", \"empty\"))\n expect_numeric(res[[\"nested.2dlist_a\"]])\n expect_numeric(res[[\"nested.2dlist_b\"]])\n\n cols = \"nested.df\"\n res = unwrap(x, cols, sep = \"_\")\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x) + 1L, col.names = \"unique\", any.missing = TRUE)\n expect_set_equal(names(res), c(\"id\", \"nested.list\", \"nested.2dlist\", \"nested.df_a\", \"nested.df_b\", \"multi.row\", \"empty\"))\n expect_numeric(res[[\"nested.df_a\"]])\n expect_numeric(res[[\"nested.df_b\"]])\n\n cols = \"empty\"\n res = unwrap(x, cols)\n expect_data_table(res, nrow = nrow(x), ncol = ncol(x) - 1L, col.names = \"unique\", any.missing = TRUE)\n expect_equal(names(res), c(\"id\", \"nested.list\", \"nested.2dlist\", \"nested.df\", \"multi.row\"))\n\n expect_error(unwrap(x), \"Name clash\")\n x = data.table(x = list(2, 3, 5), y = 1:3)\n res = unwrap(x)\n expect_data_table(res, nrow = 3, ncol = 2, col.names = \"unique\", any.missing = FALSE)\n expect_set_equal(names(res), c(\"y\", \"x.1\"))\n})\n" }, { "path": "tests/testthat/test_waitForJobs.R", "content": "test_that(\"waitForJobs\", {\n reg = makeTestRegistry()\n fun = function(x) if (x == 2) stop(x) else x\n ids = batchMap(reg = reg, fun, 1:2)\n silent({\n submitJobs(ids, reg = reg)\n expect_true(waitForJobs(ids = ids[1], reg = reg, sleep = 1))\n expect_false(waitForJobs(ids = ids, stop.on.error = TRUE, sleep = 1, expire.after = 3, reg = reg))\n })\n})\n\ntest_that(\"waitForJobs: detection of expired jobs\", {\n reg = makeTestRegistry()\n if (is.null(reg$cluster.functions$killJob))\n skip(\"Test requires killJobs\")\n ids = batchMap(reg = reg, Sys.sleep, c(20, 20))\n ids$chunk = 1L\n\n silent({\n submitJobs(ids, reg = reg)\n batch.ids = reg$status$batch.id\n reg$cluster.functions$killJob(reg, batch.ids[1])\n expect_warning(waitForJobs(ids, reg = reg, sleep = 1, stop.on.expire = TRUE), \"disappeared\")\n })\n})\n\ntest_that(\"waitForJobs: filter out unsubmitted jobs\", {\n reg = makeTestRegistry()\n ids = batchMap(identity, 1:2, reg = reg)\n silent({\n submitJobs(ids = 1, reg = reg)\n expect_warning(res <- waitForJobs(ids = ids, reg = reg, sleep = 1), \"unsubmitted\")\n expect_true(res)\n })\n})\n" }, { "path": "tests/testthat.R", "content": "# setting R_TESTS to empty string because of\n# https://github.com/hadley/testthat/issues/144\n# revert this when that issue in R is fixed.\nSys.setenv(\"R_TESTS\" = \"\")\n\nlibrary(testthat)\nlibrary(batchtools)\n\ntest_check(\"batchtools\")\n" }, { "path": "vignettes/batchtools.Rmd", "content": "---\ntitle: \"batchtools\"\noutput:\n html_document:\n toc: true\nurlcolor: blue\nlinkcolor: blue\nvignette: >\n %\\VignetteIndexEntry{batchtools}\n %\\VignetteEngine{knitr::rmarkdown}\n %\\VignetteEncoding{UTF-8}\n---\n\n```{r,include = FALSE, cache = FALSE}\nlibrary(batchtools)\nlibrary(data.table) # for %chin%, data.table\noptions(batchtools.progress = FALSE, datatable.print.class = TRUE, batchtools.timestamps = FALSE)\nif (identical(Sys.getenv(\"IN_PKGDOWN\"), \"true\")) {\n tmp_dir = fs::path(dirname(tempdir()), \"batchtools-vignette\")\n if (fs::dir_exists(tmp_dir))\n fs::dir_delete(tmp_dir)\n fs::file_temp_push(fs::path(tmp_dir, letters))\n}\n```\n\n# Setup\n\n## Cluster Functions\n\nThe communication with the batch system is managed via so-called cluster functions.\nThey are created with the constructor `makeClusterFunctions()` which defines how jobs are submitted on your system.\nFurthermore, you may provide functions to list queued/running jobs and to kill jobs.\n\nUsually you do not have to start from scratch but can just use one of the cluster functions which ship with the package:\n\n\n* Interactive Cluster Functions (default): `makeClusterFunctionsInteractive()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsInteractive.R)\n* Multicore Cluster Functions: `makeClusterFunctionsMulticore()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsMulticore.R)\n* Socket Cluster Functions: `makeClusterFunctionsSocket()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsSocket.R)\n* Makeshift SSH cluster: `makeClusterFunctionsSSH()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsSSH.R)\n* Docker Swarm: `makeClusterFunctionsDocker()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsDocker.R)\n* IBM Spectrum Load Sharing Facility (LSF): `makeClusterFunctionsLSF()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsLSF.R)\n* OpenLava: `makeClusterFunctionsOpenLava()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsOpenLava.R)\n* Univa Grid Engine / Oracle Grid Engine (OGE) / Sun Grid Engine (SGE): `makeClusterFunctionsSGE()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsSGE.R)\n* Slurm: `makeClusterFunctionsSlurm()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsSlurm.R)\n* TORQUE/OpenPBS: `makeClusterFunctionsTORQUE()`, [implementation](https://github.com/mlr-org/batchtools/blob/master/R/clusterFunctionsTORQUE.R)\n\n\nTo use the package with the socket cluster functions, you would call the respective constructor `makeClusterFunctionsSocket()`:\n```{r, message=FALSE}\nreg = makeRegistry(NA)\nreg$cluster.functions = makeClusterFunctionsSocket(2)\n```\nTo make this selection permanent for this registry, save the `?Registry` with `saveRegistry()`.\nTo make your cluster function selection permanent for a specific system across R sessions for all new Registries, you can set up a configuration file (see below).\n\nIf you have trouble debugging your cluster functions, you can enable the debug mode for extra output.\nTo do so, install the [debugme package](https://cran.r-project.org/package=debugme) and set the environment variable `DEBUGME` to `batchtools` before you load the `batchtools` package:\n```{r,eval=FALSE}\nSys.setenv(DEBUGME = \"batchtools\")\nlibrary(batchtools)\n```\n\n\n## Template Files\n\nMany cluster functions require a template file as argument.\nThese templates are used to communicate with the scheduler and contain placeholders to evaluate arbitrary R expressions.\nInternally, the [brew package](https://cran.r-project.org/package=brew) is used for this purpose.\nSome exemplary template files can be found [here](https://github.com/mlr-org/batchtools/tree/master/inst/templates).\nIt would be great if you would help expand this collection to cover more exotic configurations.\nTo do so, open a new pull request.\n\nNote that all variables defined in a `?JobCollection` can be used inside the template.\nIf you need to pass extra variables, you can set them via the argument `resources` of `submitJobs()`.\n\nIf the flexibility which comes with templating is not sufficient, you can still construct a custom cluster function implementation yourself using the provided `makeClusterFunctions()`.\n\n\n## Configuration File\n\nThe configuration file can be used to set system specific options.\nIts default location depends on the operating system (see `makeRegistry()`), but for the first time setup you can put one in the current working directory (as reported by `getwd()`).\nIn order to set the cluster function implementation, you would generate a file with the following content:\n```{r,eval = FALSE}\ncluster.functions = makeClusterFunctionsInteractive()\n```\nThe configuration file is parsed whenever you create or load a `?Registry`.\nIt is sourced inside of your registry which has the advantage that you can (a) access all of the parameters which are passed to makeRegistry() and (b) you can also directly change them.\nLets say you always want your working directory in your home directory and you always want to load the `checkmate` package on the nodes, you can just append these lines:\n```{r, eval = FALSE}\nwork.dir = \"~\"\npackages = union(packages, \"checkmate\")\n```\nSee the documentation on `?Registry` for a more complete list of supported configuration options.\n\n\n# Migration from `BatchJobs`/`Batchexperiments`\n\nThe development of [BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) is discontinued because of the following reasons:\n\n* Maintainability: The packages [BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) are tightly connected which makes maintaining difficult. Changes have to be synchronized and tested against the current CRAN versions for compatibility. Furthermore, BatchExperiments violates CRAN policies by calling internal functions of BatchJobs.\n* Data base issues: Although we invested weeks to mitigate issues with locks of the SQLite data base or file system (staged queries, file system timeouts, ...), BatchJobs kept working unreliable on some systems with high latency or specific file systems. This made BatchJobs unusable for many users.\n\n[BatchJobs](https://github.com/tudo-r/BatchJobs/) and [BatchExperiments](https://github.com/tudo-r/Batchexperiments) will remain on CRAN or Github, but new features are unlikely to be ported back.\n\n\n## Internal Changes\n* batchtools does not use SQLite anymore.\n Instead, all the information is stored directly in the registry using [data.tables](https://cran.r-project.org/package=data.table) acting as an in-memory database. As a side effect, many operations are much faster.\n* Nodes do not have to access the registry.\n `submitJobs()` stores a temporary object of type `doJobCollection()` on the node.\n This avoids file system locks because each job accesses only one file exclusively.\n* `ClusterFunctionsMulticore` now uses the parallel package for multicore execution.\n* `ClusterFunctionsSSH` can still be used to emulate a scheduler-like system which respects the work load on the local machine.\n Setting the hostname to `\"localhost\"` just strips out `ssh` of the command issued.\n\n## Interface Changes\n* batchtools remembers the last created or loaded Registry and sets it as default registry.\n This way, you do not need to pass the registry around anymore.\n If you need to work with multiple registries simultaneously on the other hand, you can still do so by explicitly passing registries to the functions.\n* Most functions now return a [data.table](https://cran.r-project.org/package=data.table) which is keyed with the `job.id`.\n This way, return values can be joined together easily and efficient (see `?JoinTables`) for some examples).\n* The building blocks of a problem has been renamed from `static` and `dynamic` to the more intuitive `data` and `fun`.\n Thus, algorithm function should have the formal arguments `job`, `data` and `instance`.\n* The function `makeDesign` has been removed.\n Parameters can be defined by just passing a `data.frame` or `data.table` to `addExperiments()`.\n For exhaustive designs, use `data.table::CJ()`.\n\n## Template changes\n* The scheduler should directly execute the command:\n\n```\nRscript -e 'batchtools::doJobCollection()'\n```\n\n There is no intermediate R source file like there was in `BatchJobs`.\n* All information stored in the object `?JobCollection` can be accessed while brewing the template.\n* Extra variables may be passed via the argument `resoures` of `submitJobs()`.\n\n## New features\n* Support for Docker Swarm via `?ClusterFunctionsDocker`.\n* Jobs can now be tagged and untagged to provide an easy way to group them.\n* Some resources like the number of CPUs are now optionally passed to [parallelMap](https://cran.r-project.org/package=parallelMap).\n This eases nested parallelization, e.g. to use multicore parallelization on the slave by just setting a resource on the master.\n See `submitJobs()` for an example.\n* `?ClusterFunctions` are now more flexible in general as they can define hook functions which will be called at certain events.\n `?ClusterFunctionsDocker` is an example use case which implements a housekeeping routine.\n This routine is called every time before a job is about to get submitted to the scheduler (in the case: the Docker Swarm) via the hook `pre.submit` and every time directly after the registry synchronized jobs stored on the file system via the hook `post.sync`.\n\n\n## Porting to `batchtools`\n\nThe following table assists in porting to batchtools by mapping BatchJobs/BatchExperiments functions to their counterparts in batchtools.\nThe table does not cover functions which are (a) used only internally in BatchJobs and (b) functions which have not been renamed.\n\n| BatchJobs | batchtools |\n| ------------- | :-------------: |\n| `addRegistryPackages` | Set `reg$packages` or `reg$namespaces`, call saveRegistry()] |\n| `addRegistrySourceDirs` | - |\n| `addRegistrySourceFiles` | Set `reg$source`, call saveRegistry() |\n| `batchExpandGrid` | `batchMap(..., args = CJ(x = 1:3, y = 1:10))` |\n| `batchMapQuick` | `btmapply()` |\n| `batchReduceResults` | - |\n| `batchUnexport` | `batchExport()` |\n| `filterResults` | - |\n| `getJobIds` | `findJobs()` |\n| `getJobInfo` | `getJobStatus()` |\n| `getJob` | `makeJob()` |\n| `getJobParamDf` | `getJobPars()` |\n| `loadResults` | `reduceResultsList()` |\n| `reduceResultsDataFrame` | `reduceResultsDataTable()` |\n| `reduceResultsMatrix` | `reduceResultsList()` + `do.call(rbind, res)` |\n| `reduceResultsVector` | `reduceResultsDataTable()` |\n| `setJobFunction` | - |\n| `setJobNames` | - |\n| `showStatus` | `getStatus()` |\n\n\n# Example 1: Approximation of $\\pi$\nTo get a first insight into the usage of `batchtools`, we start with an exemplary Monte Carlo simulation to approximate $\\pi$.\nFor background information, see [Wikipedia](https://en.wikipedia.org/wiki/Monte_Carlo_method).\n\nFirst, a so-called registry object has to be created, which defines a directory where all relevant information, files and results of the computational jobs will be stored.\nThere are two different types of registry objects: First, a regular `?Registry` which we will use in this example.\nSecond, an `?ExperimentRegistry` which provides an alternative way to define computational jobs and thereby is tailored for a broad range of large scale computer experiments.\nHere, we use a temporary registry which is stored in the temp directory of the system and gets automatically deleted if you close the R session.\n\n```{r, message = FALSE}\nreg = makeRegistry(file.dir = NA, seed = 1)\n```\n\nFor a permanent registry, set the `file.dir` to a valid path.\nIt can then be reused later, e.g., when you login to the system again, by calling the function `loadRegistry(file.dir)`.\n\nWhen a registry object is created or loaded, it is stored for the active R session as the default.\nTherefore the argument `reg` will be ignored in functions calls of this example, assuming the correct registry is set as default.\nTo get the current default registry, `getDefaultRegistry()` can be used.\nTo switch to another registry, use `setDefaultRegistry()`.\n\nFirst, we create a function which samples $n$ points $(x_i, y_i)$ whereas $x_i$ and $y_i$ are distributed uniformly, i.e. $x_i, y_i \\sim \\mathcal{U}(0,1)$.\nNext, the distance to the origin $(0, 0)$ is calculated and the fraction of points in the unit circle ($d \\leq 1$) is returned.\n```{r}\npiApprox = function(n) {\n nums = matrix(runif(2 * n), ncol = 2)\n d = sqrt(nums[, 1]^2 + nums[, 2]^2)\n 4 * mean(d <= 1)\n}\nset.seed(42)\npiApprox(1000)\n```\nWe now parallelize `piApprox()` with `batchtools`: We create 10 jobs, each doing a MC simulation with $10^5$ jobs.\nWe use `batchMap()` to define the jobs (note that this does not yet start the calculation):\n```{r}\nbatchMap(fun = piApprox, n = rep(1e5, 10))\n```\nThe length of the vector or list defines how many different jobs are created, while the elements itself are used as arguments for the function.\nThe function `batchMap(fun, ...)` works analogously to `Map(f, ...)` of the base package.\nAn overview over the jobs and their IDs can be retrieved with `getJobTable()` which returns a data.frame with all relevant information:\n```{r}\nnames(getJobTable())\n```\nNote that a unique job ID is assigned to each job.\nThese IDs can be used to restrict operations to subsets of jobs.\nTo actually start the calculation, call `submitJobs()`.\nThe registry and the selected job IDs can be taken as arguments as well as an arbitrary list of resource requirements, which are to be handled by the cluster back end.\n```{r}\nsubmitJobs(resources = list(walltime = 3600, memory = 1024))\n```\nIn this example, a cap for the execution time (so-called walltime) and for the maximum memory requirements are set.\nThe progress of the submitted jobs can be checked with `getStatus()`.\n```{r}\ngetStatus()\n```\nThe resulting output includes the number of jobs in the registry, how many have been submitted, have started to execute on the batch system, are currently running, have successfully completed, and have terminated due to an R exception.\nAfter jobs have successfully terminated, we can load their results on the master.\nThis can be done in a simple fashion by using either `loadResult()`, which returns a single result exactly in the form it was calculated during mapping, or by using `reduceResults()`, which is a version of `Reduce()` from the base package for registry objects.\n```{r}\nwaitForJobs()\nmean(sapply(1:10, loadResult))\nreduceResults(function(x, y) x + y) / 10\n```\n\nIf you are absolutely sure that your function works, you can take a shortcut and use *batchtools* in an `lapply` fashion using `btlapply()`.\nThis function creates a temporary registry (but you may also pass one yourself), calls `batchMap()`, wait for the jobs to terminate with `waitForJobs()` and then uses `reduceResultsList()` to return the results.\n\n```{r, R.options=list(batchtools.verbose=FALSE)}\nres = btlapply(rep(1e5, 10), piApprox)\nmean(unlist(res))\n```\n\n# Example 2: Machine Learning\n\nWe stick to a rather simple, but not unrealistic example to explain some further functionalities: Applying two classification learners to the famous iris data set (Anderson 1935), vary a few hyperparameters and evaluate the effect on the classification performance.\n\nFirst, we create a registry, the central meta-data object which records technical details and the setup of the experiments.\nWe use an `?ExperimentRegistry` where the job definition is split into creating problems and algorithms.\nSee the paper on [BatchJobs and BatchExperiments](https://www.jstatsoft.org/article/view/v064i11) for a detailed explanation.\nAgain, we use a temporary registry and make it the default registry.\n```{r, message = FALSE}\nlibrary(batchtools)\nreg = makeExperimentRegistry(file.dir = NA, seed = 1)\n```\n\n## Problems and Algorithms\n\nBy adding a problem to the registry, we can define the data on which certain computational jobs shall work.\nThis can be a matrix, data frame or array that always stays the same for all subsequent experiments.\nBut it can also be of a more dynamic nature, e.g., subsamples of a dataset or random numbers drawn from a probability distribution .\nTherefore the function `addProblem()` accepts static parts in its `data` argument, which is passed to the argument `fun` which generates a (possibly stochastic) problem instance.\nFor `data`, any R object can be used.\nIf only `data` is given, the generated instance is `data`.\nThe argument `fun` has to be a function with the arguments `data` and `job` (and optionally other arbitrary parameters).\nThe argument `job` is an object of type `?Job` which holds additional information about the job.\n\nWe want to split the iris data set into a training set and test set.\nIn this example we use use subsampling which just randomly takes a fraction of the observations as training set.\nWe define a problem function which returns the indices of the respective training and test set for a split with `100 * ratio`% of the observations being in the test set:\n```{r}\nsubsample = function(data, job, ratio, ...) {\n n = nrow(data)\n train = sample(n, floor(n * ratio))\n test = setdiff(seq_len(n), train)\n list(test = test, train = train)\n}\n```\n`addProblem()` files the problem to the file system and the problem gets recorded in the registry.\n```{r}\ndata(\"iris\", package = \"datasets\")\naddProblem(name = \"iris\", data = iris, fun = subsample, seed = 42)\n```\nThe function call will be evaluated at a later stage on the workers.\nIn this process, the `data` part will be loaded and passed to the function.\nNote that we set a problem seed to synchronize the experiments in the sense that the same resampled training and test sets are used for the algorithm comparison in each distinct replication.\n\nThe algorithms for the jobs are added to the registry in a similar manner.\nWhen using `addAlgorithm()`, an identifier as well as the algorithm to apply to are required arguments.\nThe algorithm must be given as a function with arguments `job`, `data` and `instance`.\nFurther arbitrary arguments (e.g., hyperparameters or strategy parameters) may be defined analogously as for the function in `addProblem`.\nThe objects passed to the function via `job` and `data` are here the same as above, while via `instance` the return value of the evaluated problem function is passed.\nThe algorithm can return any R object which will automatically be stored on the file system for later retrieval.\nFirstly, we create an algorithm which applies a support vector machine:\n```{r}\nsvm.wrapper = function(data, job, instance, ...) {\n library(\"e1071\")\n mod = svm(Species ~ ., data = data[instance$train, ], ...)\n pred = predict(mod, newdata = data[instance$test, ], type = \"class\")\n table(data$Species[instance$test], pred)\n}\naddAlgorithm(name = \"svm\", fun = svm.wrapper)\n```\n\nSecondly, a random forest of classification trees:\n```{r}\nforest.wrapper = function(data, job, instance, ...) {\n library(\"ranger\")\n mod = ranger(Species ~ ., data = data[instance$train, ], write.forest = TRUE)\n pred = predict(mod, data = data[instance$test, ])\n table(data$Species[instance$test], pred$predictions)\n}\naddAlgorithm(name = \"forest\", fun = forest.wrapper)\n```\nBoth algorithms return a confusion matrix for the predictions on the test set, which will later be used to calculate the misclassification rate.\n\nNote that using the `...` argument in the wrapper definitions allows us to circumvent naming specific design parameters for now.\nThis is an advantage if we later want to extend the set of algorithm parameters in the experiment.\nThe algorithms get recorded in the registry and the corresponding functions are stored on the file system.\n\nDefined problems and algorithms can be queried with:\n```{r}\nreg$problems\nreg$algorithms\n```\nThe flow to define experiments is summarized in the following figure:\n\n```{r,echo=FALSE}\nknitr::include_graphics(\"tikz_prob_algo_simple.png\", auto_pdf = TRUE)\n```\n\n## Creating jobs\n`addExperiments()` is used to parametrize the jobs and thereby define computational jobs.\nTo do so, you have to pass named lists of parameters to `addExperiments()`.\nThe elements of the respective list (one for problems and one for algorithms) must be named after the problem or algorithm they refer to.\nThe data frames contain parameter constellations for the problem or algorithm function where columns must have the same names as the target arguments.\nWhen the problem design and the algorithm design are combined in `addExperiments()`, each combination of the parameter sets of the two designs defines a distinct job.\nHow often each of these jobs should be computed can be determined with the argument `repls`.\n\n```{r}\n# problem design: try two values for the ratio parameter\npdes = list(iris = data.table(ratio = c(0.67, 0.9)))\n\n# algorithm design: try combinations of kernel and epsilon exhaustively,\n# try different number of trees for the forest\nades = list(\n svm = CJ(kernel = c(\"linear\", \"polynomial\", \"radial\"), epsilon = c(0.01, 0.1)),\n forest = data.table(ntree = c(100, 500, 1000))\n)\n\naddExperiments(pdes, ades, repls = 5)\n```\nThe jobs are now available in the registry with an individual job ID for each.\nThe function `summarizeExperiments()` returns a table which gives a quick overview over all defined experiments.\n\n```{r}\nsummarizeExperiments()\nsummarizeExperiments(by = c(\"problem\", \"algorithm\", \"ratio\"))\n```\n\n## Before Submitting\nBefore submitting all jobs to the batch system, we encourage you to test each algorithm individually.\nOr sometimes you want to submit only a subset of experiments because the jobs vastly differ in runtime.\nAnother reoccurring task is the collection of results for only a subset of experiments.\nFor all these use cases, `findExperiments()` can be employed to conveniently select a particular subset of jobs.\nIt returns the IDs of all experiments that match the given criteria.\nYour selection can depend on substring matches of problem or algorithm IDs using `prob.name` or `algo.name`, respectively.\nYou can also pass R expressions, which will be evaluated in your problem parameter setting (`prob.pars`) or algorithm parameter setting (`algo.pars`).\nThe expression is then expected to evaluate to a Boolean value.\nFurthermore, you can restrict the experiments to specific replication numbers.\n\nTo illustrate `findExperiments()`, we will select two experiments, one with a support vector machine and the other with a random forest and the parameter `ntree = 1000`.\nThe selected experiment IDs are then passed to testJob.\n```{r}\nid1 = head(findExperiments(algo.name = \"svm\"), 1)\nprint(id1)\nid2 = head(findExperiments(algo.name = \"forest\", algo.pars = (ntree == 1000)), 1)\nprint(id2)\ntestJob(id = id1)\ntestJob(id = id2)\n```\n\nIf something goes wrong, `batchtools` comes with a bunch of useful debugging utilities (see separate vignette on error handling).\nIf everything turns out fine, we can proceed with the calculation.\n\n## Submitting and Collecting Results\n\nTo submit the jobs, we call `submitJobs()` and wait for all jobs to terminate using `waitForJobs()`.\n```{r}\nsubmitJobs()\nwaitForJobs()\n```\nAfter jobs are finished, the results can be collected with `reduceResultsDataTable()` where we directly extract the mean misclassification error:\n```{r}\nreduce = function(res) list(mce = (sum(res) - sum(diag(res))) / sum(res))\nresults = unwrap(reduceResultsDataTable(fun = reduce))\nhead(results)\n```\n\nNext, we merge the results table with the table of job parameters using one of the join helpers (see `?JoinTables`) provided by `batchtools` (here, we use an inner join):\n```{r}\npars = unwrap(getJobPars())\ntab = ijoin(pars, results)\nhead(tab)\n```\n\nWe now aggregate the results group-wise.\nYou can use [`data.table`](https://cran.r-project.org/package=data.table), `base::aggregate()`, or the [`dplyr`](https://cran.r-project.org/package=dplyr) package for this purpose.\nHere, we use [`data.table`](https://cran.r-project.org/package=data.table) to subset the table to jobs where the ratio is `0.67` and group by algorithm the algorithm hyperparameters:\n```{r}\ntab[ratio == 0.67, list(mmce = mean(mce)),\n by = c(\"algorithm\", \"kernel\", \"epsilon\", \"ntree\")]\n```\n\n# Example: Error Handling\n\nIn any large scale experiment many things can and will go wrong.\nThe cluster might have an outage, jobs may run into resource limits or crash, subtle bugs in your code could be triggered or any other error condition might arise.\nIn these situations it is important to quickly determine what went wrong and to recompute only the minimal number of required jobs.\n\nTherefore, before you submit anything you should use `testJob()` to catch errors that are easy to spot because they are raised in many or all jobs.\nIf `external` is set, this function runs the job without side effects in an independent R process on your local machine via `Rscript` similar as on the slave, redirects the output of the process to your R console, loads the job result and returns it.\nIf you do not set `external`, the job is executed is in the currently running R session, with the drawback that you might be unable to catch missing variable declarations or missing package dependencies.\n\nBy way of illustration here is a small example.\nFirst, we create a temporary registry.\n```{r, message = FALSE}\nlibrary(batchtools)\nreg = makeRegistry(file.dir = NA, seed = 1)\n```\nTen jobs are created, one will trow a warning and two of them will raise an exception.\n```{r}\nflakeyFunction <- function(value) {\n if (value == 5) warning(\"Just a simple warning\")\n if (value %in% c(2, 9)) stop(\"Ooops.\")\n value^2\n}\nbatchMap(flakeyFunction, 1:10)\n```\n\nNow that the jobs are defined, we can test jobs independently:\n```{r}\ntestJob(id = 1)\n```\nIn this case, testing the job with ID = 1 provides the appropriate result but testing the job with ID = 2 leads to an error:\n```{r}\nas.character(try(testJob(id = 2)))\n```\nWe ignore the error here, and just assume everything looks fine and submit all jobs.\n```{r}\nsubmitJobs()\nwaitForJobs()\n```\nAfter you have submitted jobs and suspect that something is going wrong, the first thing to do is to run `getStatus()` to display a summary of the current state of the system.\n```{r}\ngetStatus()\n```\nThe status message shows that two of the jobs could not be executed successfully.\nTo get the IDs of all jobs that failed due to an error we can use `findErrors()` and to retrieve the actual error message, we can use `getErrorMessages()`.\n```{r}\nfindErrors()\ngetErrorMessages()\n```\nIf we want to peek into the R log file of a job to see more context for the error we can use `showLog()` which opens a pager or use `getLog()` to get the log as character vector:\n```{r}\ntail(getLog(id = 9))\n```\n\nYou can also grep for messages (output suppressed in this vignette for technical reasons):\n```{r,eval=FALSE}\ngrepLogs(pattern = \"simple\", ignore.case = TRUE)\n```\n\n\n# Workflow\n\n## On the Local System\n\n1. Create a Registry with `makeRegistry()` / `makeExperimentRegistry()` or load an existing from the file system with `loadRegistry()`.\n2. Define computational jobs with `batchMap()` / `batchReduce()` if you used `makeRegistry()` or define an experiment with `addAlgorithm()`, `addProblem()` and `addExperiments()` if you started with `makeExperimentRegistry()`.\n It is advised to test some jobs with `testJob()` in the interactive session and with `testJob(external = TRUE)` in a separate R process.\n Note that you can add additional jobs if you are using an `?ExperimentRegistry`.\n3. If required, query the data base for job ids depending on their status, parameters or tags (see `findJobs()`).\n The returned tables can easily be combined in a set-like fashion with data base verbs: union (`ojoin()` for outer join, intersect `ijoin()` for inner join, difference `ajoin()` for anti join).\n4. Submit jobs with `submitJobs()`. You can specify job resources here.\n If you have thousands of fast terminating jobs, you want to `chunk()` them first.\n If some jobs already terminated, you can estimate the runtimes with `estimateRuntimes()` and chunk jobs into heterogeneous groups with `lpt()` and `binpack()`.\n5. Monitor jobs. `getStatus()` gives a summarizing overview.\n Use `showLog()` and `grepLogs()` to investigate log file.\n Run jobs in the currently running session with `testJob()` to get a `traceback()`.\n6. Collect (partial) results. `loadResult()` retrieves a single result from the file system.\n `reduceResults()` mimics `Reduce()` and allows to apply a function to many files in an iterative fashion.\n `reduceResultsList()` and `reduceResultsDataTable()` collect results into a `list` or `data.table`, respectively.\n\n```{r,echo=FALSE}\nknitr::include_graphics(\"function_overview.png\", auto_pdf = TRUE)\n```\n\n## On Multiple Systems\n\nMost users develop and prototype their experiments on a desktop box in their preferred IDE and later deploy to a large computing cluster.\nThis can be done by prototyping locally (`testJob()`) or submit subsets via `submitJobs()`.\nTo deploy to the cluster, just copy the file directory (as reported by `reg$file.dir`) to the remote system.\nNext, log in on the cluster (typically via `ssh`), `cd` to the copied directory and call `loadRegistry(\", \"\", writeable = TRUE)`.\nThis function will (a) source the local configuration file so that you can talk to the cluster (verify by checking the output of `reg$cluster.functions`) and (b) adjust the paths to the new system if argument `update.paths` is set.\nAfter loading the Registry, it is advised to test some jobs again with `testJob()` before submitting all of them with `submitJobs(resources = list())` (remember you now need to set resources!).\nAfter some jobs are finished, the `file.dir` can be copied back (do not merge with the previous directory!) and loaded again with `loadRegistry()`.\n\nThis approach is totally viable as long as some general rules are followed:\n\n1. Make sure you have all packages installed. Package versions can be synchronized across machines with, e.g., [`checkpoint`](https://cran.r-project.org/package=checkpoint) or [`packrat`](https://cran.r-project.org/package=packrat).\n2. Test jobs on the remote system prior to submit to ensure that paths are resolved correctly.\n3. Make sure you have set the cluster functions in a configuration file, and stick to one backend as long as jobs are running.\n4. The status can only be monitored on the remote system (for obvious reasons).\n5. Partial results can be inspected both on the remote system and on the local system.\n For the latter, you need to copy over the **complete** `file.dir` first.\n Overwriting/merging directories is not advised as this may lead to inconsistencies if you added or removed experiments on the remote.\n If you have to merge, use `rsync` with option `--delete`.\n Load the registry locally with `loadRegistry()` and collect results. Do not copy back and forth.\n6. Avoid accessing the `file.dir` with multiple sessions simultaneously.\n This includes accessing the registry via a mount!\n Simultaneous access may lead to inconsistencies and missing results.\n" }, { "path": "vignettes/function_overview.tex", "content": "\\documentclass[crop,tikz,convert]{standalone}\n\\usetikzlibrary{shapes,matrix,positioning,chains,arrows,shadows,decorations.pathmorphing,fit,backgrounds}\n\\begin{document}\n\\begin{tikzpicture}[auto]\n \\tikzstyle{box} = [rectangle, drop shadow, draw=black, fill=white, thick, minimum width=4cm, rounded corners, align=center,font=\\ttfamily\\large]\n \\tikzstyle{chead} = [font=\\large\\bfseries]\n \\tikzstyle{rhead} = [chead,align=left, minimum width=4cm]\n \\tikzstyle{bg} = [rectangle, fill=gray!10, inner sep=0.2cm, rounded corners=5mm]\n \\tikzstyle{hl} = [rectangle, draw=red, inner sep=0.2cm, rounded corners=5mm]\n\n \\matrix [row sep=10mm, column sep=5mm] (mat) {\n \\node (chead0) [minimum width=4cm] {}; \\pgfmatrixnextcell\n \\node (chead1) [chead] {Regular Registry}; \\pgfmatrixnextcell\n \\node (chead2) [chead] {Common}; \\pgfmatrixnextcell\n \\node (chead3) [chead] {Experiment Registry}; \\\\\n\n \\node (registry0) [rhead] {(1) Create Registry}; \\pgfmatrixnextcell\n \\node (registry1) [box] {makeRegistry}; \\pgfmatrixnextcell\n \\node (registry2) {}; \\pgfmatrixnextcell\n \\node (registry3) [box] {makeExperimentRegistry}; \\\\\n\n \\node (define0) [rhead] {(2) Define Jobs}; \\pgfmatrixnextcell\n \\node (define1) [box] {batchMap \\\\ batchReduce}; \\pgfmatrixnextcell\n \\node (define2) [box] {batchMapResults}; \\pgfmatrixnextcell\n \\node (define3) [box] {addProblem \\\\ addAlgorithm \\\\ addExperiments}; \\\\\n\n \\node (subsetting0) [rhead] {(3) Subset Jobs}; \\pgfmatrixnextcell\n \\node (subsetting1) [box] {findJobs}; \\pgfmatrixnextcell\n \\node (subsetting2) [box] {findDone\\\\ findErrors \\\\\\ldots}; \\pgfmatrixnextcell\n \\node (subsetting3) [box] {findExperiments}; \\\\\n\n \\node (submit0) [rhead] {(4) Submit Jobs}; \\pgfmatrixnextcell\n \\node (submit1) {}; \\pgfmatrixnextcell\n \\node (submit2) [box] {submitJobs}; \\pgfmatrixnextcell\n \\node (submit3) {}; \\\\\n\n \\node (status0) [rhead] {(5) Monitor \\& Debug}; \\pgfmatrixnextcell\n \\node (status1) {}; \\pgfmatrixnextcell\n \\node (status2) [box] {getStatus \\\\ testJob \\\\ showLog \\\\ grepLogs}; \\pgfmatrixnextcell\n \\node (status3) [box] {summarizeExperiments}; \\\\\n\n \\node (collect0) [rhead] {(6) Collect Results}; \\pgfmatrixnextcell\n \\node (collect1) {}; \\pgfmatrixnextcell\n \\node (collect2) [box] {loadResult \\\\ reduceResults \\\\ reduceResults[List|DataTable]}; \\pgfmatrixnextcell\n \\node (collect3) {}; \\\\\n };\n \\begin{pgfonlayer}{background}\n \\node [bg, fit=(chead0) (collect0)] {};\n \\node [bg, fit=(chead0) (chead3)] {};\n \\end{pgfonlayer}\n\\end{tikzpicture}\n\\end{document}\n" }, { "path": "vignettes/tikz_prob_algo_simple.tex", "content": "\\documentclass[crop,tikz,convert]{standalone}\n\\usetikzlibrary{shapes,matrix,positioning,chains,arrows,shadows,decorations.pathmorphing,fit,backgrounds}\n\\begin{document}\n\n\\begin{tikzpicture}[auto]\n \\tikzstyle{userinput}=[rectangle, drop shadow, draw=black, fill=black!10, thick, minimum width=4cm, align=center]\n \\tikzstyle{internal}=[rectangle, drop shadow, draw=black, fill=white, thick, minimum width=4cm, rounded corners, align=center]\n \\tikzstyle{result}=[ellipse, drop shadow, draw=black, fill=white, thick, align=center, minimum width=3cm]\n \\tikzstyle{line} = [draw, thick, -latex']\n \\tikzstyle{sline} = [draw, thick, -latex',decorate, decoration={snake, segment length=2mm,post length=2mm}]\n\n \\matrix [row sep=10mm, column sep=20mm] {\n % first row\n \\node {}; &\n \\node {}; &\n \\node [result] (result) {result}; \\\\\n\n % second row\n \\node [userinput] (static_problem_part) {\n static problem part\\\\\\texttt{data}\n }; &\n \\node [userinput] (dynamic_problem_part) {\n dynamic problem function\\\\\n \\texttt{fun(data, ...)}\n\t}; &\n\t\\node [userinput] (algorithm) {\n algorithm function\\\\\n \\texttt{fun(data, instance, ...)}\n\t}; \\\\\n\n\t% third row\n\t\\node {}; &\n \\node [userinput] (problem_design) {\n problem design\\\\\n (\\texttt{addExperiments})\n }; &\n\t\\node [userinput] (algorithm_design) {\n algorithm design\\\\\n (\\texttt{addExperiments})\n\t}; \\\\\n };\n\n \\draw [sline] (algorithm) to (result) ;\n \\draw [line] (static_problem_part) to (dynamic_problem_part);\n \\draw [sline] (dynamic_problem_part) to node {\\texttt{instance}} (algorithm) ;\n \\draw [line] (static_problem_part) to [out=0, in=0, bend left=20] (algorithm);\n\n \\draw [line] (problem_design) to node {\\texttt{...}} (dynamic_problem_part);\n \\draw [line] (algorithm_design) to node {\\texttt{...}} (algorithm);\n\\end{tikzpicture}\n\\end{document}\n" } ]