Showing preview only (1,740K chars total). Download the full file or copy to clipboard to get everything.
Repository: mllg/batchtools
Branch: main
Commit: ee7080fc31de
Files: 320
Total size: 1.6 MB
Directory structure:
gitextract_lqp2x_vj/
├── .Rbuildignore
├── .aspell/
│ ├── batchtools.rds
│ └── defaults.R
├── .editorconfig
├── .github/
│ ├── dependabot.yaml
│ └── workflows/
│ ├── pkgdown.yml
│ └── r-cmd-check.yml
├── .gitignore
├── .ignore
├── .lintr
├── DESCRIPTION
├── LICENSE
├── NAMESPACE
├── NEWS.md
├── R/
│ ├── Algorithm.R
│ ├── ExperimentRegistry.R
│ ├── Export.R
│ ├── Hooks.R
│ ├── Job.R
│ ├── JobCollection.R
│ ├── JobNames.R
│ ├── JobTables.R
│ ├── Joins.R
│ ├── Logs.R
│ ├── Problem.R
│ ├── RDSReader.R
│ ├── Registry.R
│ ├── Tags.R
│ ├── Worker.R
│ ├── addExperiments.R
│ ├── batchMap.R
│ ├── batchMapResults.R
│ ├── batchReduce.R
│ ├── btlapply.R
│ ├── chunkIds.R
│ ├── clearRegistry.R
│ ├── clusterFunctions.R
│ ├── clusterFunctionsDocker.R
│ ├── clusterFunctionsHyperQueue.R
│ ├── clusterFunctionsInteractive.R
│ ├── clusterFunctionsLSF.R
│ ├── clusterFunctionsMulticore.R
│ ├── clusterFunctionsOpenLava.R
│ ├── clusterFunctionsSGE.R
│ ├── clusterFunctionsSSH.R
│ ├── clusterFunctionsSlurm.R
│ ├── clusterFunctionsSocket.R
│ ├── clusterFunctionsTORQUE.R
│ ├── config.R
│ ├── doJobCollection.R
│ ├── estimateRuntimes.R
│ ├── execJob.R
│ ├── files.R
│ ├── findJobs.R
│ ├── getDefaultRegistry.R
│ ├── getErrorMessages.R
│ ├── getStatus.R
│ ├── helpers.R
│ ├── ids.R
│ ├── killJobs.R
│ ├── loadRegistry.R
│ ├── loadResult.R
│ ├── mergeRegistries.R
│ ├── reduceResults.R
│ ├── removeExperiments.R
│ ├── removeRegistry.R
│ ├── resetJobs.R
│ ├── runOSCommand.R
│ ├── saveRegistry.R
│ ├── sleep.R
│ ├── submitJobs.R
│ ├── summarizeExperiments.R
│ ├── sweepRegistry.R
│ ├── syncRegistry.R
│ ├── testJob.R
│ ├── unwrap.R
│ ├── updateRegisty.R
│ ├── waitForFiles.R
│ ├── waitForJobs.R
│ └── zzz.R
├── README.Rmd
├── README.md
├── _pkgdown.yml
├── docs/
│ ├── 404.html
│ ├── CNAME
│ ├── LICENSE-text.html
│ ├── articles/
│ │ ├── batchtools.html
│ │ ├── batchtools_files/
│ │ │ └── header-attrs-2.4/
│ │ │ └── header-attrs.js
│ │ └── index.html
│ ├── authors.html
│ ├── bootstrap-toc.css
│ ├── bootstrap-toc.js
│ ├── docsearch.css
│ ├── docsearch.js
│ ├── index.html
│ ├── news/
│ │ └── index.html
│ ├── pkgdown.css
│ ├── pkgdown.js
│ ├── pkgdown.yml
│ └── reference/
│ ├── JobCollection.html
│ ├── JobExperiment.html
│ ├── JobNames.html
│ ├── JoinTables.html
│ ├── Tags.html
│ ├── Worker.html
│ ├── addAlgorithm.html
│ ├── addExperiments.html
│ ├── addProblem.html
│ ├── assertRegistry.html
│ ├── batchExport.html
│ ├── batchMap.html
│ ├── batchMapResults.html
│ ├── batchReduce.html
│ ├── batchtools-deprecated.html
│ ├── batchtools-package.html
│ ├── btlapply.html
│ ├── cfBrewTemplate.html
│ ├── cfHandleUnknownSubmitError.html
│ ├── cfKillJob.html
│ ├── cfReadBrewTemplate.html
│ ├── chunk.html
│ ├── chunkIds.html
│ ├── clearRegistry.html
│ ├── doJobCollection.html
│ ├── estimateRuntimes.html
│ ├── execJob.html
│ ├── findConfFile.html
│ ├── findJobs.html
│ ├── findTemplateFile.html
│ ├── getDefaultRegistry.html
│ ├── getErrorMessages.html
│ ├── getJobTable.html
│ ├── getStatus.html
│ ├── grepLogs.html
│ ├── index.html
│ ├── killJobs.html
│ ├── loadRegistry.html
│ ├── loadResult.html
│ ├── makeClusterFunctions.html
│ ├── makeClusterFunctionsDocker.html
│ ├── makeClusterFunctionsInteractive.html
│ ├── makeClusterFunctionsLSF.html
│ ├── makeClusterFunctionsMulticore.html
│ ├── makeClusterFunctionsOpenLava.html
│ ├── makeClusterFunctionsSGE.html
│ ├── makeClusterFunctionsSSH.html
│ ├── makeClusterFunctionsSlurm.html
│ ├── makeClusterFunctionsSocket.html
│ ├── makeClusterFunctionsTORQUE.html
│ ├── makeExperimentRegistry.html
│ ├── makeRegistry.html
│ ├── makeSubmitJobResult.html
│ ├── reduceResults.html
│ ├── reduceResultsList.html
│ ├── removeExperiments.html
│ ├── removeRegistry.html
│ ├── resetJobs.html
│ ├── runHook.html
│ ├── runOSCommand.html
│ ├── saveRegistry.html
│ ├── showLog.html
│ ├── submitJobs.html
│ ├── summarizeExperiments.html
│ ├── sweepRegistry.html
│ ├── syncRegistry.html
│ ├── testJob.html
│ ├── unwrap.html
│ └── waitForJobs.html
├── inst/
│ ├── CITATION
│ ├── bin/
│ │ └── linux-helper
│ └── templates/
│ ├── lsf-simple.tmpl
│ ├── openlava-simple.tmpl
│ ├── sge-simple.tmpl
│ ├── slurm-dortmund.tmpl
│ ├── slurm-lido3.tmpl
│ ├── slurm-simple.tmpl
│ ├── testJob.tmpl
│ └── torque-lido.tmpl
├── man/
│ ├── JobCollection.Rd
│ ├── JobExperiment.Rd
│ ├── JobNames.Rd
│ ├── JoinTables.Rd
│ ├── Tags.Rd
│ ├── Worker.Rd
│ ├── addAlgorithm.Rd
│ ├── addExperiments.Rd
│ ├── addProblem.Rd
│ ├── assertRegistry.Rd
│ ├── batchExport.Rd
│ ├── batchMap.Rd
│ ├── batchMapResults.Rd
│ ├── batchReduce.Rd
│ ├── batchtools-package.Rd
│ ├── btlapply.Rd
│ ├── cfBrewTemplate.Rd
│ ├── cfHandleUnknownSubmitError.Rd
│ ├── cfKillJob.Rd
│ ├── cfReadBrewTemplate.Rd
│ ├── chunk.Rd
│ ├── clearRegistry.Rd
│ ├── doJobCollection.Rd
│ ├── estimateRuntimes.Rd
│ ├── execJob.Rd
│ ├── findConfFile.Rd
│ ├── findJobs.Rd
│ ├── findTemplateFile.Rd
│ ├── getDefaultRegistry.Rd
│ ├── getErrorMessages.Rd
│ ├── getJobTable.Rd
│ ├── getStatus.Rd
│ ├── grepLogs.Rd
│ ├── killJobs.Rd
│ ├── loadRegistry.Rd
│ ├── loadResult.Rd
│ ├── makeClusterFunctions.Rd
│ ├── makeClusterFunctionsDocker.Rd
│ ├── makeClusterFunctionsHyperQueue.Rd
│ ├── makeClusterFunctionsInteractive.Rd
│ ├── makeClusterFunctionsLSF.Rd
│ ├── makeClusterFunctionsMulticore.Rd
│ ├── makeClusterFunctionsOpenLava.Rd
│ ├── makeClusterFunctionsSGE.Rd
│ ├── makeClusterFunctionsSSH.Rd
│ ├── makeClusterFunctionsSlurm.Rd
│ ├── makeClusterFunctionsSocket.Rd
│ ├── makeClusterFunctionsTORQUE.Rd
│ ├── makeExperimentRegistry.Rd
│ ├── makeRegistry.Rd
│ ├── makeSubmitJobResult.Rd
│ ├── reduceResults.Rd
│ ├── reduceResultsList.Rd
│ ├── removeExperiments.Rd
│ ├── removeRegistry.Rd
│ ├── resetJobs.Rd
│ ├── runHook.Rd
│ ├── runOSCommand.Rd
│ ├── saveRegistry.Rd
│ ├── showLog.Rd
│ ├── submitJobs.Rd
│ ├── summarizeExperiments.Rd
│ ├── sweepRegistry.Rd
│ ├── syncRegistry.Rd
│ ├── testJob.Rd
│ ├── unwrap.Rd
│ └── waitForJobs.Rd
├── man-roxygen/
│ ├── expreg.R
│ ├── id.R
│ ├── ids.R
│ ├── missing.val.R
│ ├── more.args.R
│ ├── ncpus.R
│ ├── nodename.R
│ ├── reg.R
│ └── template.R
├── paper/
│ ├── codemeta.json
│ ├── paper.bib
│ └── paper.md
├── src/
│ ├── Makevars
│ ├── binpack.c
│ ├── count_not_missing.c
│ ├── fill_gaps.c
│ ├── init.c
│ └── lpt.c
├── tests/
│ ├── testthat/
│ │ ├── helper.R
│ │ ├── test_Algorithm.R
│ │ ├── test_ClusterFunctionHyperQueue.R
│ │ ├── test_ClusterFunctions.R
│ │ ├── test_ClusterFunctionsMulticore.R
│ │ ├── test_ClusterFunctionsSSH.R
│ │ ├── test_ClusterFunctionsSocket.R
│ │ ├── test_ExperimentRegistry.R
│ │ ├── test_Job.R
│ │ ├── test_JobCollection.R
│ │ ├── test_JobNames.R
│ │ ├── test_Problem.R
│ │ ├── test_Registry.R
│ │ ├── test_addExperiments.R
│ │ ├── test_batchMap.R
│ │ ├── test_batchReduce.R
│ │ ├── test_btlapply.R
│ │ ├── test_chunk.R
│ │ ├── test_convertIds.R
│ │ ├── test_count.R
│ │ ├── test_doJobCollection.R
│ │ ├── test_estimateRuntimes.R
│ │ ├── test_export.R
│ │ ├── test_findConfFile.R
│ │ ├── test_findJobs.R
│ │ ├── test_foreach.R
│ │ ├── test_future.R
│ │ ├── test_getErrorMessages.R
│ │ ├── test_getJobTable.R
│ │ ├── test_getStatus.R
│ │ ├── test_grepLogs.R
│ │ ├── test_hooks.R
│ │ ├── test_joins.R
│ │ ├── test_killJobs.R
│ │ ├── test_manual.R
│ │ ├── test_memory.R
│ │ ├── test_mergeRegistries.R
│ │ ├── test_parallelMap.R
│ │ ├── test_reduceResults.R
│ │ ├── test_removeExperiments.R
│ │ ├── test_removeRegistry.R
│ │ ├── test_resetJobs.R
│ │ ├── test_runOSCommand.R
│ │ ├── test_seed.R
│ │ ├── test_showLog.R
│ │ ├── test_sleep.R
│ │ ├── test_submitJobs.R
│ │ ├── test_summarizeExperiments.R
│ │ ├── test_sweepRegistry.R
│ │ ├── test_tags.R
│ │ ├── test_testJob.R
│ │ ├── test_unwrap.R
│ │ └── test_waitForJobs.R
│ └── testthat.R
└── vignettes/
├── batchtools.Rmd
├── function_overview.tex
└── tikz_prob_algo_simple.tex
================================================
FILE CONTENTS
================================================
================================================
FILE: .Rbuildignore
================================================
^LICENSE$
^src/.+\.o$
^src/.+\.so$
\.swp$
^\.ignore$
^\.editorconfig$
^man-roxygen$
^.*\.Rproj$
^\.Rproj\.user$
^docs$
^paper$
^_pkgdown\.yml$
^README.RMD$
^.github$
registry/
^\.lintr$
================================================
FILE: .aspell/defaults.R
================================================
Rd_files <- vignettes <- R_files <- description <-
list(encoding = "UTF-8", language = "en", dictionaries = c("en_stats", "batchtools"))
================================================
FILE: .editorconfig
================================================
# See http://editorconfig.org
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
indent_style = space
trim_trailing_whitespace = true
[*.{r,R}]
indent_size = 2
[*.{c,h}]
indent_size = 4
[*.{cpp,hpp}]
indent_size = 4
[{NEWS,DESCRIPTION,LICENSE}]
max_line_length = 80
================================================
FILE: .github/dependabot.yaml
================================================
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
================================================
FILE: .github/workflows/pkgdown.yml
================================================
# pkgdown workflow of the mlr3 ecosystem v0.1.0
# https://github.com/mlr-org/actions
on:
push:
branches:
- main
pull_request:
branches:
- main
release:
types:
- published
workflow_dispatch:
name: pkgdown
jobs:
pkgdown:
runs-on: ubuntu-latest
concurrency:
group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
TORCH_INSTALL: 1
steps:
- uses: actions/checkout@v4
- uses: r-lib/actions/setup-pandoc@v2
- uses: r-lib/actions/setup-r@v2
- name: Install system dependencies
if: runner.os == 'Linux'
run: |
while read -r cmd
do
eval sudo $cmd
done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
sudo apt-get install -y libopenmpi-dev openmpi-bin
- uses: r-lib/actions/setup-r-dependencies@v2
with:
extra-packages: any::pkgdown, local::.
needs: website
- name: Install template
run: pak::pkg_install("mlr-org/mlr3pkgdowntemplate")
shell: Rscript {0}
- name: Build site
run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
shell: Rscript {0}
- name: Deploy
if: github.event_name != 'pull_request'
uses: JamesIves/github-pages-deploy-action@v4.7.3
with:
clean: false
branch: gh-pages
folder: docs
================================================
FILE: .github/workflows/r-cmd-check.yml
================================================
# r cmd check workflow of the mlr3 ecosystem v0.2.0
# https://github.com/mlr-org/actions
on:
workflow_dispatch:
inputs:
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
push:
branches:
- main
pull_request:
branches:
- main
schedule:
- cron: '0 4 * * 1'
name: r-cmd-check
jobs:
r-cmd-check:
runs-on: ${{ matrix.config.os }}
name: ${{ matrix.config.os }} (${{ matrix.config.r }})
strategy:
fail-fast: false
matrix:
config:
- {os: ubuntu-latest, r: 'devel'}
- {os: ubuntu-latest, r: 'release'}
- {os: macos-latest, r: 'release'}
- {os: windows-latest, r: 'release'}
env:
R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
_R_CHECK_FORCE_SUGGESTS_: 0
RSPM: ${{ matrix.config.rspm }}
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: r-lib/actions/setup-r@v2
with:
r-version: ${{ matrix.config.r }}
- uses: r-lib/actions/setup-pandoc@v2
- name: Install OpenMPI (macOS)
if: runner.os == 'macOS'
run: |
brew install open-mpi
- name: Install system dependencies
if: runner.os == 'Linux'
run: |
while read -r cmd
do
eval sudo $cmd
done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
sudo apt-get install -y libopenmpi-dev openmpi-bin
- uses: r-lib/actions/setup-r-dependencies@v2
with:
extra-packages: any::rcmdcheck
needs: check
- uses: r-lib/actions/check-r-package@v2
- uses: mxschmitt/action-tmate@v3
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
with:
limit-access-to-actor: true
================================================
FILE: .gitignore
================================================
.DS_Store
/inst/doc
/src/*.so
/src/*.o
/vignettes/*.html
/vignettes/*.pdf
/.Rproj.user
/batchtools.Rproj
*.tar.gz
*.Rcheck
README.html
================================================
FILE: .ignore
================================================
man/
docs/
================================================
FILE: .lintr
================================================
linters: linters_with_defaults(
# lintr defaults: https://lintr.r-lib.org/reference/default_linters.html
# the following setup changes/removes certain linters
assignment_linter = NULL, # do not force using <- for assignments
object_name_linter(c("snake_case", "CamelCase")), # only allow snake case and camel case object names
commented_code_linter = NULL, # allow code in comments
line_length_linter(200L),
object_length_linter(40L),
undesirable_function_linter(fun = c(
# base messaging
cat = "use catf()",
stop = "use stopf()",
warning = "use warningf()",
message = "use messagef()",
# perf
ifelse = "use fifelse()",
rank = "use frank()"
))
)
================================================
FILE: DESCRIPTION
================================================
Package: batchtools
Title: Tools for Computation on Batch Systems
Version: 0.9.18
Authors@R: c(
person("Michel", "Lang", , "michellang@gmail.com", role = c("cre", "aut"),
comment = c(ORCID = "0000-0001-9754-0393")),
person("Bernd", "Bischl", , "bernd_bischl@gmx.net", role = "aut"),
person("Dirk", "Surmann", , "surmann@statistik.tu-dortmund.de", role = "ctb",
comment = c(ORCID = "0000-0003-0873-137X"))
)
Description: As a successor of the packages 'BatchJobs' and
'BatchExperiments', this package provides a parallel implementation of
the Map function for high performance computing systems managed by
schedulers 'IBM Spectrum LSF'
(<https://www.ibm.com/products/hpc-workload-management>), 'Univa Grid
Engine'/'Oracle Grid Engine'
(<https://altair.com/hpc-cloud-applications/>), 'Slurm'
(<https://slurm.schedmd.com/>), 'TORQUE/PBS'
(<https://adaptivecomputing.com/cherry-services/torque-resource-manager/>),
or 'Docker Swarm' (<https://docs.docker.com/engine/swarm/>). A
multicore and socket mode allow the parallelization on a local
machines, and multiple machines can be hooked up via SSH to create a
makeshift cluster. Moreover, the package provides an abstraction
mechanism to define large-scale computer experiments in a
well-organized and reproducible way.
License: LGPL-3
URL: https://github.com/mlr-org/batchtools, https://batchtools.mlr-org.com
BugReports: https://github.com/mlr-org/batchtools/issues
Depends:
R (>= 3.0.0)
Imports:
backports (>= 1.1.2),
base64url (>= 1.1),
brew,
checkmate (>= 1.8.5),
data.table (>= 1.11.2),
digest (>= 0.6.9),
fs (>= 1.2.0),
parallel,
progress (>= 1.1.1),
R6,
rappdirs,
stats,
stringi,
utils,
withr (>= 2.0.0)
Suggests:
debugme,
doMPI,
doParallel,
e1071,
foreach,
future,
future.batchtools,
jsonlite,
knitr,
parallelMap,
ranger,
rmarkdown,
rpart,
snow,
testthat,
tibble
VignetteBuilder:
knitr
ByteCompile: yes
Encoding: UTF-8
NeedsCompilation: yes
Roxygen: list(r6 = FALSE)
RoxygenNote: 7.3.3
================================================
FILE: LICENSE
================================================
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
================================================
FILE: NAMESPACE
================================================
# Generated by roxygen2: do not edit by hand
S3method(doJobCollection,JobCollection)
S3method(doJobCollection,character)
S3method(execJob,Experiment)
S3method(execJob,Job)
S3method(execJob,JobCollection)
S3method(execJob,character)
S3method(getJob,ExperimentCollection)
S3method(getJob,JobCollection)
S3method(getJobPars,ExperimentRegistry)
S3method(getJobPars,Registry)
S3method(makeJob,ExperimentRegistry)
S3method(makeJob,Registry)
S3method(makeJobCollection,ExperimentRegistry)
S3method(makeJobCollection,Registry)
S3method(print,ClusterFunctions)
S3method(print,ExperimentRegistry)
S3method(print,JobCollection)
S3method(print,Registry)
S3method(print,RuntimeEstimate)
S3method(print,Status)
S3method(print,SubmitJobResult)
S3method(runHook,JobCollection)
S3method(runHook,Registry)
export(Worker)
export(addAlgorithm)
export(addExperiments)
export(addJobTags)
export(addProblem)
export(ajoin)
export(assertRegistry)
export(batchExport)
export(batchMap)
export(batchMapResults)
export(batchReduce)
export(binpack)
export(btlapply)
export(btmapply)
export(cfBrewTemplate)
export(cfHandleUnknownSubmitError)
export(cfKillJob)
export(cfReadBrewTemplate)
export(chunk)
export(clearRegistry)
export(doJobCollection)
export(estimateRuntimes)
export(execJob)
export(findConfFile)
export(findDone)
export(findErrors)
export(findExperiments)
export(findExpired)
export(findJobs)
export(findNotDone)
export(findNotStarted)
export(findNotSubmitted)
export(findOnSystem)
export(findQueued)
export(findRunning)
export(findStarted)
export(findSubmitted)
export(findTagged)
export(findTemplateFile)
export(flatten)
export(getDefaultRegistry)
export(getErrorMessages)
export(getJobNames)
export(getJobPars)
export(getJobResources)
export(getJobStatus)
export(getJobTable)
export(getJobTags)
export(getLog)
export(getStatus)
export(getUsedJobTags)
export(grepLogs)
export(ijoin)
export(killJobs)
export(ljoin)
export(loadRegistry)
export(loadResult)
export(lpt)
export(makeClusterFunctions)
export(makeClusterFunctionsDocker)
export(makeClusterFunctionsHyperQueue)
export(makeClusterFunctionsInteractive)
export(makeClusterFunctionsLSF)
export(makeClusterFunctionsMulticore)
export(makeClusterFunctionsOpenLava)
export(makeClusterFunctionsSGE)
export(makeClusterFunctionsSSH)
export(makeClusterFunctionsSlurm)
export(makeClusterFunctionsSocket)
export(makeClusterFunctionsTORQUE)
export(makeExperimentRegistry)
export(makeJob)
export(makeJobCollection)
export(makeRegistry)
export(makeSubmitJobResult)
export(ojoin)
export(reduceResults)
export(reduceResultsDataTable)
export(reduceResultsList)
export(removeAlgorithms)
export(removeExperiments)
export(removeJobTags)
export(removeProblems)
export(removeRegistry)
export(resetJobs)
export(rjoin)
export(runHook)
export(runOSCommand)
export(saveRegistry)
export(setDefaultRegistry)
export(setJobNames)
export(showLog)
export(sjoin)
export(submitJobs)
export(summarizeExperiments)
export(sweepRegistry)
export(syncRegistry)
export(testJob)
export(ujoin)
export(unwrap)
export(waitForJobs)
import(checkmate)
import(data.table)
import(stringi)
import(utils)
importFrom(R6,R6Class)
importFrom(base64url,base32_decode)
importFrom(base64url,base32_encode)
importFrom(brew,brew)
importFrom(digest,digest)
importFrom(progress,progress_bar)
importFrom(rappdirs,site_config_dir)
importFrom(rappdirs,user_config_dir)
importFrom(stats,pexp)
importFrom(stats,predict)
importFrom(stats,runif)
importFrom(withr,local_dir)
importFrom(withr,local_options)
importFrom(withr,with_dir)
importFrom(withr,with_seed)
useDynLib(batchtools,c_binpack)
useDynLib(batchtools,c_lpt)
useDynLib(batchtools,count_not_missing)
useDynLib(batchtools,fill_gaps)
================================================
FILE: NEWS.md
================================================
# batchtools 0.9.18
* Fixed CRAN issues with documentation
# batchtools 0.9.17
* Fixed a bug in the finalizer of `ClusterFunctionsMulticore`.
# batchtools 0.9.16
* Fixed a bug in `addExperiments()` in combination with combination method `"bind"` and repls > 1 where experiments have been duplicated.
* `addExperiments()` now also accepts a vector of replications (instead of a single scalar value) for argument `repls`.
* Improved handling of jobs in `ClusterFunctionsSlurm`.
* Fixed a bug in `waitForJobs()`
* Fixed some assertions.
# batchtools 0.9.15
* Maintenance update.
# batchtools 0.9.14
* `batchMap()` now supports unnamed `more.args`.
* Exports are now assigned with `delayedAssign()`.
* Fix an option in the LSF template.
# batchtools 0.9.13
* Maintenance release for R-4.0.0.
# batchtools 0.9.12
* Moved `data.table` from `Depends` to `Imports`.
User scripts might need to explicitly attach `data.table` via `library()` now.
* Fixes for `ClusterFunctionsMulticore`.
* Removed a workaround for `system2()` for R-devel (to be released as R-4.0.0).
* New configuration option `compress` to select the compression algorithm (passed down to `saveRDS()`).
# batchtools 0.9.11
* Removed deprecated function `chunkIds()`.
* New default for argument `fs.timeout` in the cluster function constructor is `0` (was `NA` before).
* Fixed a unit test for OSX.
* Improved stability and documentation.
* Fixed memory usage calculation.
# batchtools 0.9.10
* Exported functions `findConfFile()` and `findTemplateFile()`.
* Dropped support for providing a template file directly as string. A valid file is now always required.
* Fixed writing to `TMPDIR` instead of the R session's temporary directory.
# batchtools 0.9.9
* RDS files are explicitly stored in version 2 to ensure backward compatibility with R versions prior to 3.5.0.
* Package `fs` is now used internally for all file system operations.
* Support for per-site configuration files and templates to be set up by system administrators.
* The print of `getStatus()` now includes a time stamp.
* `chunk()` now optionally shuffles the ids before chunking.
* Support for setting per-job resources in `submitJobs()`.
* Example templates now include resources for `blas.threads` and `omp.threads`.
* Some bug fixes regarding read-only registries.
# batchtools 0.9.8
* Renamed column "memory" in the status table to "mem.used" to avoid name clashes with the resource specification.
* Exported function `assertRegistry()`.
* New function `unwrap()` as alias to `flatten()`.
The latter causes a name clash with package `purrr` and will be deprecated in a future version.
* Registries now contain a unique hash which is updated each time the registry is altered.
Can be utilized to invalidate caches, e.g. the cache of knitr.
# batchtools 0.9.7
* Added a workaround for a test to be compatible with testthat v2.0.0.
* Better and more customizable handling of expired jobs in `waitForJobs()`.
* Package `foreach` is now supported for nested parallelization as an alternative to `parallelMap`.
* Depreciated argument flatten has been removed.
* New helper function `flatten()` to manually unnest/unwrap lists in data frames.
* Removed functions `getProblemIds()` and `getAlgorithmIds()`.
Instead, you can just access `reg$problems` or `reg$algorithms`, respectively.
* The number of the maximum concurrent jobs can now also be controlled via setting resources.
* Internal data base changes to speed up some operations.
Old registries are updated on first load by `loadRegistry()`.
* Fixed a bug where the sleep mechanism between queries was not working.
* Fixed a bug where submit errors on SLURM and TORQUE were not detected as temporary.
# batchtools 0.9.6
* Fixed a bug where the wrong problem was retrieved from the cache. This was only triggered for chunked jobs in
combination with an `ExperimentRegistry`.
# batchtools 0.9.5
* Added a missing routine to upgrade registries created with batchtools prior to v0.9.3.
* Fixed a bug where the registry could not be synced if jobs failed during initialization (#135).
* The sleep duration for `waitForJobs()` and `submitJobs()` can now be set via the configuration file.
* A new heuristic will try to detect if the registry has been altered by a simultaneously running R session.
If this is detected, the registry in the current session will be set to a read-only state.
* `waitForJobs()` has been reworked to allow control over the heuristic to detect expired jobs.
Jobs are treated as expired if they have been submitted but are not detected on the system for `expire.after` iterations
(default 3 iterations, before 1 iteration).
* New argument `writeable` for `loadRegistry()` to allow loading registries explicitly as read-only.
* Removed argument `update.paths` from `loadRegistry()`.
Paths are always updated, but the registry on the file system remains unchanged unless loaded in read-write mode.
* `ClusterFunctionsSlurm` now come with an experimental nodename argument. If set, all communication with the master is
handled via SSH which effectively allows you to submit jobs from your local machine instead of the head node.
Note that mounting the file system (e.g., via SSHFS) is mandatory.
# batchtools 0.9.4
* Fixed handling of `file.dir` with special chars like whitespace.
* All backward slashes will now be converted to forward slashes on windows.
* Fixed order of arguments in `findExperiments()` (argument `ids` is now first).
* Removed code to upgrade registries created with versions prior to v0.9.0 (first CRAN release).
* `addExperiments()` now warns if a design is passed as `data.frame` with factor columns and `stringsAsFactors` is `TRUE`.
* Added functions `setJobNames()` and `getJobNames()` to control the name of jobs on batch systems.
Templates should be adapted to use `job.name` instead of `job.hash` for naming.
* Argument `flatten` of `getJobResources()`, `getJobPars()` and `getJobTable()` is deprecated and will be removed.
Future versions of the functions will behave like `flatten` is set to `FALSE` explicitly.
Single resources/parameters must be extracted manually (or with `tidyr::unnest()`).
# batchtools 0.9.3
* Running jobs now are also included while querying for status "started". This affects `findStarted()`, `findNotStarted()` and `getStatus()`.
* `findExperiments()` now performs an exact string match (instead of matching substrings) for patterns specified via `prob.name` and `algo.name`.
For substring matching, use `prob.pattern` or `algo.pattern`, respectively.
* Changed arguments for `reduceResultsDataTable()`
* Removed `fill`, now is always `TRUE`
* Introduced `flatten` to control if the result should be represented as a column of lists or flattened as separate columns.
Defaults to a backward-compatible heuristic, similar to `getJobPars`.
* Improved heuristic to lookup template files.
Templates shipped with the package can now be used by providing just the file name (w/o extension).
* Updated CITATION
# batchtools 0.9.2
* Full support for array jobs on Slurm and TORQUE.
* Array jobs have been disabled for SGE and LSF (due to missing information about the output format) but will be re-enable in a future release.
Note that the variable `n.array.jobs` has been removed from `JobCollection` in favor of the new variable `array.jobs` (logical).
* `findExperiments()` now has two additional arguments to match using regular expressions.
The possibility to prefix a string with "~" to enable regular expression matching has been removed.
* New function `batchReduce()`.
* New function `estimateRuntimes()`.
* New function `removeRegistry()`.
* Missing result files are now handled more consistently, raising an exception in its defaults if the result is not available.
The argument `missing.val` has been added to `reduceResultsList()` and `reduceResultsDataTable()` and removed from `loadResult()` and `batchMapResults()`.
* Timestamps are now stored with sub-second accuracy.
* Renamed Torque to TORQUE. This especially affects the constructor `makeClusterFunctionsTorque` which now must be called via `makeClusterFunctionsTORQUE()`
* `chunkIds()` has been deprecated. Use `chunk()`, `lpt()` or `binpack()` instead.
* Fixed listing of jobs for `ClusterFunctionsLSF` and `ClusterFunctionsOpenLava` (thanks to @phaverty).
* Job hashes are now prefixed with the literal string 'job' to ensure they start with a letter as required by some SGE systems.
* Fixed handling of `NULL` results in `reduceResultsList()`
* Fixed key lookup heuristic join functions.
* Fixed a bug where `getJobTable()` returned `difftimes` with the wrong unit (e.g., in minutes instead of seconds).
* Deactivated swap allocation for `ClusterFunctionsDocker`.
* The package is now more patient while communicating with the scheduler or file system by using a timeout-based approach.
This should make the package more reliable and robust under heavy load.
# batchtools 0.9.0
Initial CRAN release.
See the vignette for a brief comparison with [BatchJobs](https://cran.r-project.org/package=BatchJobs)/[BatchExperiments](https://cran.r-project.org/package=BatchExperiments).
================================================
FILE: R/Algorithm.R
================================================
#' @title Define Algorithms for Experiments
#'
#' @description
#' Algorithms are functions which get the \code{data} part as well as the problem instance (the return value of the
#' function defined in \code{\link{Problem}}) and return an arbitrary R object.
#'
#' This function serializes all components to the file system and registers the algorithm in the \code{\link{ExperimentRegistry}}.
#'
#' \code{removeAlgorithm} removes all jobs from the registry which depend on the specific algorithm.
#' \code{reg$algorithms} holds the IDs of already defined algorithms.
#'
#' @param name [\code{character(1)}]\cr
#' Unique identifier for the algorithm.
#' @param fun [\code{function}]\cr
#' The algorithm function. The static problem part is passed as \dQuote{data}, the generated
#' problem instance is passed as \dQuote{instance} and the \code{\link{Job}}/\code{\link{Experiment}} as \dQuote{job}.
#' Therefore, your function must have the formal arguments \dQuote{job}, \dQuote{data} and \dQuote{instance} (or dots \code{...}).
#'
#' If you do not provide a function, it defaults to a function which just returns the instance.
#' @template expreg
#' @return [\code{Algorithm}]. Object of class \dQuote{Algorithm}.
#' @aliases Algorithm
#' @seealso \code{\link{Problem}}, \code{\link{addExperiments}}
#' @export
addAlgorithm = function(name, fun = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg, class = "ExperimentRegistry", writeable = TRUE)
assertString(name, min.chars = 1L)
if (!stri_detect_regex(name, "^[[:alnum:]_.-]+$"))
stopf("Illegal characters in problem name: %s", name)
if (is.null(fun)) {
fun = function(job, data, instance, ...) instance
} else {
assert(checkFunction(fun, args = c("job", "data", "instance")), checkFunction(fun, args = "..."))
}
info("Adding algorithm '%s'", name)
algo = setClasses(list(fun = fun, name = name), "Algorithm")
writeRDS(algo, file = getAlgorithmURI(reg, name), compress = reg$compress)
reg$algorithms = union(reg$algorithms, name)
saveRegistry(reg)
invisible(algo)
}
#' @export
#' @rdname addAlgorithm
removeAlgorithms = function(name, reg = getDefaultRegistry()) {
assertRegistry(reg, class = "ExperimentRegistry", writeable = TRUE, running.ok = FALSE)
assertCharacter(name, any.missing = FALSE)
assertSubset(name, reg$algorithms)
algorithm = NULL
for (nn in name) {
def.ids = reg$defs[algorithm == nn, "def.id"]
job.ids = filter(def.ids, reg$status, "job.id")
info("Removing Algorithm '%s' and %i corresponding jobs ...", nn, nrow(job.ids))
file_remove(getAlgorithmURI(reg, nn))
reg$defs = reg$defs[!def.ids]
reg$status = reg$status[!job.ids]
reg$algorithms = chsetdiff(reg$algorithms, nn)
}
sweepRegistry(reg)
invisible(TRUE)
}
getAlgorithmURI = function(reg, name) {
fs::path(dir(reg, "algorithms"), mangle(name))
}
================================================
FILE: R/ExperimentRegistry.R
================================================
#' @title ExperimentRegistry Constructor
#'
#' @description
#' \code{makeExperimentRegistry} constructs a special \code{\link{Registry}} which
#' is suitable for the definition of large scale computer experiments.
#'
#' Each experiments consists of a \code{\link{Problem}} and an \code{\link{Algorithm}}.
#' These can be parametrized with \code{\link{addExperiments}} to actually define computational
#' jobs.
#'
#' @inheritParams makeRegistry
#' @aliases ExperimentRegistry
#' @return [\code{ExperimentRegistry}].
#' @export
#' @family Registry Experiment
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)
#'
#' # Definde one problem, two algorithms and add them with some parameters:
#' addProblem(reg = tmp, "p1",
#' fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd))
#' addAlgorithm(reg = tmp, "a1", fun = function(job, data, instance, ...) mean(instance))
#' addAlgorithm(reg = tmp, "a2", fun = function(job, data, instance, ...) median(instance))
#' ids = addExperiments(reg = tmp, list(p1 = data.table::CJ(n = c(50, 100), mean = -2:2, sd = 1:4)))
#'
#' # Overview over defined experiments:
#' tmp$problems
#' tmp$algorithms
#' summarizeExperiments(reg = tmp)
#' summarizeExperiments(reg = tmp, by = c("problem", "algorithm", "n"))
#' ids = findExperiments(prob.pars = (n == 50), reg = tmp)
#' print(unwrap(getJobPars(ids, reg = tmp)))
#'
#' # Submit jobs
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#'
#' # Reduce the results of algorithm a1
#' ids.mean = findExperiments(algo.name = "a1", reg = tmp)
#' reduceResults(ids.mean, fun = function(aggr, res, ...) c(aggr, res), reg = tmp)
#'
#' # Join info table with all results and calculate mean of results
#' # grouped by n and algorithm
#' ids = findDone(reg = tmp)
#' pars = unwrap(getJobPars(ids, reg = tmp))
#' results = unwrap(reduceResultsDataTable(ids, fun = function(res) list(res = res), reg = tmp))
#' tab = ljoin(pars, results)
#' tab[, list(mres = mean(res)), by = c("n", "algorithm")]
makeExperimentRegistry = function(file.dir = "registry", work.dir = getwd(), conf.file = findConfFile(), packages = character(0L), namespaces = character(0L),
source = character(0L), load = character(0L), seed = NULL, make.default = TRUE) {
reg = makeRegistry(file.dir = file.dir, work.dir = work.dir, conf.file = conf.file,
packages = packages, namespaces = namespaces, source = source, load = load, seed = seed, make.default = make.default)
fs::dir_create(fs::path(reg$file.dir, c("problems", "algorithms")))
reg$problems = character(0L)
reg$algorithms = character(0L)
reg$status$repl = integer(0L)
reg$defs$problem = character(0L)
reg$defs$algorithm = character(0L)
reg$defs$job.pars = NULL
reg$defs$prob.pars = list()
reg$defs$algo.pars = list()
reg$defs$pars.hash = character(0L)
class(reg) = c("ExperimentRegistry", "Registry")
saveRegistry(reg)
return(reg)
}
#' @export
print.ExperimentRegistry = function(x, ...) {
cat("Experiment Registry\n")
catf(" Backend : %s", x$cluster.functions$name)
catf(" File dir : %s", x$file.dir)
catf(" Work dir : %s", x$work.dir)
catf(" Jobs : %i", nrow(x$status))
catf(" Problems : %i", length(x$problems))
catf(" Algorithms: %i", length(x$algorithms))
catf(" Seed : %i", x$seed)
catf(" Writeable : %s", x$writeable)
}
================================================
FILE: R/Export.R
================================================
#' @title Export Objects to the Slaves
#'
#' @description
#' Objects are saved in subdirectory \dQuote{exports} of the
#' \dQuote{file.dir} of \code{reg}.
#' They are automatically loaded and placed in the global environment
#' each time the registry is loaded or a job collection is executed.
#'
#' @param export [\code{list}]\cr
#' Named list of objects to export.
#' @param unexport [\code{character}]\cr
#' Vector of object names to unexport.
#' @template reg
#' @return [\code{data.table}] with name and uri to the exported objects.
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#'
#' # list exports
#' exports = batchExport(reg = tmp)
#' print(exports)
#'
#' # add a job and required exports
#' batchMap(function(x) x^2 + y + z, x = 1:3, reg = tmp)
#' exports = batchExport(export = list(y = 99, z = 1), reg = tmp)
#' print(exports)
#'
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#' stopifnot(loadResult(1, reg = tmp) == 101)
#'
#' # Un-export z
#' exports = batchExport(unexport = "z", reg = tmp)
#' print(exports)
batchExport = function(export = list(), unexport = character(0L), reg = getDefaultRegistry()) {
assertRegistry(reg, writeable = (length(export) > 0L || length(unexport) > 0L))
assertList(export, names = "named")
assertCharacter(unexport, any.missing = FALSE, min.chars = 1L)
path = fs::path(reg$file.dir, "exports")
if (length(export) > 0L) {
nn = names(export)
fn = fs::path(path, mangle(nn))
found = fs::file_exists(fn)
if (any(!found))
info("Exporting new objects: '%s' ...", stri_flatten(nn[!found], "','"))
if (any(found))
info("Overwriting previously exported object: '%s'", stri_flatten(nn[found], "','"))
Map(writeRDS, object = export, file = fn, compress = reg$compress)
}
if (length(unexport) > 0L) {
fn = fs::path(path, mangle(unexport))
found = fs::file_exists(fn)
if (any(found))
info("Un-exporting exported objects: '%s' ...", stri_flatten(unexport[found], "','"))
file_remove(fn[found])
}
fns = list.files(path, pattern = "\\.rds")
invisible(data.table(name = unmangle(fns), uri = fs::path(path, fns)))
}
================================================
FILE: R/Hooks.R
================================================
#' @title Trigger Evaluation of Custom Function
#'
#' @description
#' Hooks allow to trigger functions calls on specific events.
#' They can be specified via the \code{\link{ClusterFunctions}} and are triggered on the following events:
#' \describe{
#' \item{\code{pre.sync}}{\code{function(reg, fns, ...)}: Run before synchronizing the registry on the master. \code{fn} is the character vector of paths to the update files.}
#' \item{\code{post.sync}}{\code{function(reg, updates, ...)}: Run after synchronizing the registry on the master. \code{updates} is the data.table of processed updates.}
#' \item{\code{pre.submit.job}}{\code{function(reg, ...)}: Run before a job is successfully submitted to the scheduler on the master.}
#' \item{\code{post.submit.job}}{\code{function(reg, ...)}: Run after a job is successfully submitted to the scheduler on the master.}
#' \item{\code{pre.submit}}{\code{function(reg, ...)}: Run before any job is submitted to the scheduler.}
#' \item{\code{post.submit}}{\code{function(reg, ...)}: Run after a jobs are submitted to the schedule.}
#' \item{\code{pre.do.collection}}{\code{function(reg, reader, ...)}: Run before starting the job collection on the slave.
#' \code{reader} is an internal cache object.}
#' \item{\code{post.do.collection}}{\code{function(reg, updates, reader, ...)}: Run after all jobs in the chunk are terminated on the slave.
#' \code{updates} is a \code{\link[data.table]{data.table}} of updates which will be merged with the \code{\link{Registry}} by the master.
#' \code{reader} is an internal cache object.}
#' \item{\code{pre.kill}}{\code{function(reg, ids, ...)}: Run before any job is killed.}
#' \item{\code{post.kill}}{\code{function(reg, ids, ...)}: Run after jobs are killed. \code{ids} is the return value of \code{\link{killJobs}}.}
#' }
#'
#' @param obj [\link{Registry} | \link{JobCollection}]\cr
#' Registry which contains the \link{ClusterFunctions} with element \dQuote{hooks}
#' or a \link{JobCollection} which holds the subset of functions which are executed
#' remotely.
#' @param hook [\code{character(1)}]\cr
#' ID of the hook as string.
#' @param ... [ANY]\cr
#' Additional arguments passed to the function referenced by \code{hook}.
#' See description.
#' @return Return value of the called function, or \code{NULL} if there is no hook
#' with the specified ID.
#' @aliases Hooks Hook
#' @export
runHook = function(obj, hook, ...) {
UseMethod("runHook")
}
#' @export
runHook.Registry = function(obj, hook, ...) {
f = obj$cluster.functions$hooks[[hook]]
if (is.null(f))
return(NULL)
"!DEBUG [runHook]: Running hook '`hook`'"
f(obj, ...)
}
#' @export
runHook.JobCollection = function(obj, hook, ...) {
f = obj$hooks[[hook]]
if (is.null(f))
return(NULL)
"!DEBUG [runHook]: Running hook '`hook`'"
f(obj, ...)
}
================================================
FILE: R/Job.R
================================================
BaseJob = R6Class("BaseJob", cloneable = FALSE,
public = list(
file.dir = NULL,
id = NULL,
seed = NULL,
resources = NULL,
reader = NULL,
initialize = function(file.dir, reader, id, seed, resources) {
self$file.dir = file.dir
self$reader = reader
self$id = id
self$seed = seed
self$resources = resources
}
),
active = list(
job.id = function() {
# alias for id. This is confusing not to have.
self$id
},
external.dir = function() {
fs::dir_create(fs::path(self$file.dir, "external", self$id))
}
)
)
Job = R6Class("Job", cloneable = FALSE, inherit = BaseJob,
public = list(
job.pars = NULL,
initialize = function(file.dir, reader, id, job.pars, seed, resources) {
self$job.pars = job.pars
super$initialize(file.dir, reader, id, seed, resources)
}
),
active = list(
fun = function() {
self$reader$get(fs::path(self$file.dir, "user.function.rds"))
},
pars = function() {
c(self$job.pars, self$reader$get(fs::path(self$file.dir, "more.args.rds")))
}
)
)
Experiment = R6Class("Experiment", cloneable = FALSE, inherit = BaseJob,
public = list(
repl = NA_integer_,
prob.name = NULL,
algo.name = NULL,
prob.pars = NULL,
algo.pars = NULL,
compress = NULL,
allow.access.to.instance = TRUE,
initialize = function(file.dir, reader, id, prob.pars, algo.pars, repl, seed, resources, prob.name, algo.name, compress = "gzip") {
super$initialize(file.dir, reader, id,seed, resources)
self$repl = repl
self$prob.name = as.character(prob.name)
self$prob.pars = prob.pars
self$algo.name = as.character(algo.name)
self$algo.pars = algo.pars
self$compress = compress
}
),
active = list(
problem = function() {
self$reader$get(getProblemURI(self, self$prob.name), slot = "..problem..")
},
algorithm = function() {
self$reader$get(getAlgorithmURI(self, self$algo.name))
},
pars = function() {
list(prob.pars = self$prob.pars, algo.pars = self$algo.pars)
},
instance = function() {
if (!self$allow.access.to.instance)
stop("You cannot access 'job$instance' in the problem generation or algorithm function")
p = self$problem
if (p$cache) {
cache.file = getProblemCacheURI(self)
if (fs::file_exists(cache.file)) {
result = try(readRDS(cache.file))
if (!inherits(result, "try-error"))
return(result)
}
}
seed = if (is.null(p$seed)) self$seed else getSeed(p$seed, self$repl - 1L)
wrapper = function(...) p$fun(job = self, data = p$data, ...)
result = with_seed(seed, do.call(wrapper, self$prob.pars, envir = .GlobalEnv))
if (p$cache)
writeRDS(result, file = cache.file, compress = self$compress)
return(result)
}
)
)
#' @title Jobs and Experiments
#'
#' @description
#' Jobs and Experiments are abstract objects which hold all information necessary to execute a single computational
#' job for a \code{\link{Registry}} or \code{\link{ExperimentRegistry}}, respectively.
#'
#' They can be created using the constructor \code{makeJob} which takes a single job id.
#' Jobs and Experiments are passed to reduce functions like \code{\link{reduceResults}}.
#' Furthermore, Experiments can be used in the functions of the \code{\link{Problem}} and \code{\link{Algorithm}}.
#' Jobs and Experiments hold these information:
#' \describe{
#' \item{\code{job.id}}{Job ID as integer.}
#' \item{\code{pars}}{
#' Job parameters as named list.
#' For \code{\link{ExperimentRegistry}}, the parameters are divided into the sublists \dQuote{prob.pars} and \dQuote{algo.pars}.
#' }
#' \item{\code{seed}}{Seed which is set via \code{\link{doJobCollection}} as scalar integer.}
#' \item{\code{resources}}{Computational resources which were set for this job as named list.}
#' \item{\code{external.dir}}{
#' Path to a directory which is created exclusively for this job. You can store external files here.
#' Directory is persistent between multiple restarts of the job and can be cleaned by calling \code{\link{resetJobs}}.
#' }
#' \item{\code{fun}}{Job only: User function passed to \code{\link{batchMap}}.}
#' \item{\code{prob.name}}{Experiments only: Problem id.}
#' \item{\code{algo.name}}{Experiments only: Algorithm id.}
#' \item{\code{problem}}{Experiments only: \code{\link{Problem}}.}
#' \item{\code{instance}}{Experiments only: Problem instance.}
#' \item{\code{algorithm}}{Experiments only: \code{\link{Algorithm}}.}
#' \item{\code{repl}}{Experiments only: Replication number.}
#' }
#'
#' Note that the slots \dQuote{pars}, \dQuote{fun}, \dQuote{algorithm} and \dQuote{problem}
#' lazy-load required files from the file system and construct the object on the first access.
#' The realizations are cached for all slots except \dQuote{instance} (which might be stochastic).
#'
#' Jobs and Experiments can be executed manually with \code{\link{execJob}}.
#'
#' @template id
#' @param reader [\code{RDSReader} | \code{NULL}]\cr
#' Reader object to retrieve files. Used internally to cache reading from the file system.
#' The default (\code{NULL}) does not make use of caching.
#' @template reg
#' @return [\code{Job} | \code{Experiment}].
#' @aliases Job Experiment
#' @rdname JobExperiment
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMap(function(x, y) x + y, x = 1:2, more.args = list(y = 99), reg = tmp)
#' submitJobs(resources = list(foo = "bar"), reg = tmp)
#' job = makeJob(1, reg = tmp)
#' print(job)
#'
#' # Get the parameters:
#' job$pars
#'
#' # Get the job resources:
#' job$resources
#'
#' # Execute the job locally:
#' execJob(job)
makeJob = function(id, reader = NULL, reg = getDefaultRegistry()) {
UseMethod("makeJob", object = reg)
}
#' @export
makeJob.Registry = function(id, reader = NULL, reg = getDefaultRegistry()) {
row = mergedJobs(reg, convertId(reg, id), c("job.id", "job.pars", "resource.id"))
resources = reg$resources[row, "resources", on = "resource.id", nomatch = NA]$resources[[1L]] %??% list()
Job$new(file.dir = reg$file.dir, reader %??% RDSReader$new(FALSE), id = row$job.id, job.pars = row$job.pars[[1L]], seed = getSeed(reg$seed, row$job.id),
resources = resources)
}
#' @export
makeJob.ExperimentRegistry = function(id, reader = NULL, reg = getDefaultRegistry()) {
row = mergedJobs(reg, convertId(reg, id), c("job.id", "problem", "prob.pars", "algorithm", "algo.pars", "repl", "resource.id"))
resources = reg$resources[row, "resources", on = "resource.id", nomatch = NA]$resources[[1L]] %??% list()
Experiment$new(file.dir = reg$file.dir, reader %??% RDSReader$new(FALSE), id = row$job.id, prob.pars = row$prob.pars[[1L]], algo.pars = row$algo.pars[[1L]], seed = getSeed(reg$seed, row$job.id),
repl = row$repl, resources = resources, prob.name = row$problem, algo.name = row$algorithm)
}
getJob = function(jc, i, reader = NULL) {
UseMethod("getJob")
}
#' @export
getJob.JobCollection = function(jc, i, reader = RDSReader$new(FALSE)) {
row = jc$jobs[i]
Job$new(file.dir = jc$file.dir, reader = reader, id = row$job.id, job.pars = row$job.pars[[1L]], seed = getSeed(jc$seed, row$job.id), resources = jc$resources)
}
#' @export
getJob.ExperimentCollection = function(jc, i, reader = RDSReader$new(FALSE)) {
row = jc$jobs[i]
Experiment$new(file.dir = jc$file.dir, reader = reader, id = row$job.id, prob.pars = row$prob.pars[[1L]],
algo.pars = row$algo.pars[[1L]], seed = getSeed(jc$seed, row$job.id), repl = row$repl,
resources = jc$resources, prob.name = row$problem, algo.name = row$algorithm, compress = jc$compress)
}
================================================
FILE: R/JobCollection.R
================================================
#' @title JobCollection Constructor
#'
#' @description
#' \code{makeJobCollection} takes multiple job ids and creates an object of class \dQuote{JobCollection} which holds all
#' necessary information for the calculation with \code{\link{doJobCollection}}. It is implemented as an environment
#' with the following variables:
#' \describe{
#' \item{file.dir}{\code{file.dir} of the \link{Registry}.}
#' \item{work.dir:}{\code{work.dir} of the \link{Registry}.}
#' \item{job.hash}{Unique identifier of the job. Used to create names on the file system.}
#' \item{jobs}{\code{\link[data.table]{data.table}} holding individual job information. See examples.}
#' \item{log.file}{Location of the designated log file for this job.}
#' \item{resources:}{Named list of of specified computational resources.}
#' \item{uri}{Location of the job description file (saved with \code{link[base]{saveRDS}} on the file system.}
#' \item{seed}{\code{integer(1)} Seed of the \link{Registry}.}
#' \item{packages}{\code{character} with required packages to load via \code{\link[base]{require}}.}
#' \item{namespaces}{\code{character} with required packages to load via \code{\link[base]{requireNamespace}}.}
#' \item{source}{\code{character} with list of files to source before execution.}
#' \item{load}{\code{character} with list of files to load before execution.}
#' \item{array.var}{\code{character(1)} of the array environment variable specified by the cluster functions.}
#' \item{array.jobs}{\code{logical(1)} signaling if jobs were submitted using \code{chunks.as.arrayjobs}.}
#' }
#' If your \link{ClusterFunctions} uses a template, \code{\link[brew]{brew}} will be executed in the environment of such
#' a collection. Thus all variables available inside the job can be used in the template.
#'
#' @templateVar ids.default all
#' @template ids
#' @param resources [\code{list}]\cr
#' Named list of resources. Default is \code{list()}.
#' @template reg
#' @return [\code{JobCollection}].
#' @family JobCollection
#' @aliases JobCollection
#' @rdname JobCollection
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE, packages = "methods")
#' batchMap(identity, 1:5, reg = tmp)
#'
#' # resources are usually set in submitJobs()
#' jc = makeJobCollection(1:3, resources = list(foo = "bar"), reg = tmp)
#' ls(jc)
#' jc$resources
makeJobCollection = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {
UseMethod("makeJobCollection", reg)
}
createCollection = function(jobs, resources = list(), reg = getDefaultRegistry()) {
jc = new.env(parent = emptyenv())
jc$jobs = setkeyv(jobs, "job.id")
jc$job.hash = rnd_hash("job")
jc$job.name = if (anyMissing(jobs$job.name)) jc$job.hash else jobs$job.name[1L]
jc$file.dir = reg$file.dir
jc$work.dir = reg$work.dir
jc$seed = reg$seed
jc$uri = getJobFiles(reg, hash = jc$job.hash)
jc$log.file = fs::path(reg$file.dir, "logs", sprintf("%s.log", jc$job.hash))
jc$packages = reg$packages
jc$namespaces = reg$namespaces
jc$source = reg$source
jc$load = reg$load
jc$resources = resources
jc$array.var = reg$cluster.functions$array.var
jc$array.jobs = isTRUE(resources$chunks.as.arrayjobs)
jc$compress = reg$compress
hooks = chintersect(names(reg$cluster.functions$hooks), batchtools$hooks$remote)
if (length(hooks) > 0L)
jc$hooks = reg$cluster.functions$hooks[hooks]
return(jc)
}
#' @export
makeJobCollection.Registry = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {
jc = createCollection(mergedJobs(reg, convertIds(reg, ids), c("job.id", "job.name", "job.pars")), resources, reg)
setClasses(jc, "JobCollection")
}
#' @export
makeJobCollection.ExperimentRegistry = function(ids = NULL, resources = list(), reg = getDefaultRegistry()) {
jc = createCollection(mergedJobs(reg, convertIds(reg, ids), c("job.id", "job.name", "problem", "algorithm", "prob.pars", "algo.pars", "repl")), resources, reg)
setClasses(jc, c("ExperimentCollection", "JobCollection"))
}
#' @export
print.JobCollection = function(x, ...) {
catf("Collection of %i jobs", nrow(x$jobs))
catf(" Hash : %s", x$job.hash)
catf(" Log file: %s", x$log.file)
}
================================================
FILE: R/JobNames.R
================================================
#' @title Set and Retrieve Job Names
#' @name JobNames
#'
#' @description
#' Set custom names for jobs. These are passed to the template as \sQuote{job.name}.
#' If no custom name is set (or any of the job names of the chunk is missing),
#' the job hash is used as job name.
#' Individual job names can be accessed via \code{jobs$job.name}.
#'
#' @templateVar ids.default all
#' @template ids
#' @param names [\code{character}]\cr
#' Character vector of the same length as provided ids.
#' @template reg
#' @return \code{setJobNames} returns \code{NULL} invisibly, \code{getJobTable}
#' returns a \code{data.table} with columns \code{job.id} and \code{job.name}.
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' ids = batchMap(identity, 1:10, reg = tmp)
#' setJobNames(ids, letters[1:nrow(ids)], reg = tmp)
#' getJobNames(reg = tmp)
setJobNames = function(ids = NULL, names, reg = getDefaultRegistry()) {
assertRegistry(reg, writeable = TRUE)
ids = convertIds(reg, ids, default = noIds())
assertCharacter(names, min.chars = 1L, len = nrow(ids))
reg$status[ids, "job.name" := names]
saveRegistry(reg)
invisible(NULL)
}
#' @export
#' @rdname JobNames
getJobNames = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
ids = convertIds(reg, ids, default = allIds(reg))
reg$status[ids, c("job.id", "job.name")]
}
================================================
FILE: R/JobTables.R
================================================
#' @title Query Job Information
#'
#' @description
#' \code{getJobStatus} returns the internal table which stores information about the computational
#' status of jobs, \code{getJobPars} a table with the job parameters, \code{getJobResources} a table
#' with the resources which were set to submit the jobs, and \code{getJobTags} the tags of the jobs
#' (see \link{Tags}).
#'
#' \code{getJobTable} returns all these tables joined.
#'
#' @templateVar ids.default all
#' @template ids
#' @template reg
#' @return [\code{\link[data.table]{data.table}}] with the following columns (not necessarily in this order):
#' \describe{
#' \item{job.id}{Unique Job ID as integer.}
#' \item{submitted}{Time the job was submitted to the batch system as \code{\link[base]{POSIXct}}.}
#' \item{started}{Time the job was started on the batch system as \code{\link[base]{POSIXct}}.}
#' \item{done}{Time the job terminated (successfully or with an error) as \code{\link[base]{POSIXct}}.}
#' \item{error}{Either \code{NA} if the job terminated successfully or the error message.}
#' \item{mem.used}{Estimate of the memory usage.}
#' \item{batch.id}{Batch ID as reported by the scheduler.}
#' \item{log.file}{Log file. If missing, defaults to \code{[job.hash].log}.}
#' \item{job.hash}{Unique string identifying the job or chunk.}
#' \item{time.queued}{Time in seconds (as \code{\link[base]{difftime}}) the job was queued.}
#' \item{time.running}{Time in seconds (as \code{\link[base]{difftime}}) the job was running.}
#' \item{pars}{List of parameters/arguments for this job.}
#' \item{resources}{List of computational resources set for this job.}
#' \item{tags}{Tags as joined string, delimited by \dQuote{,}.}
#' \item{problem}{Only for \code{\link{ExperimentRegistry}}: the problem identifier.}
#' \item{algorithm}{Only for \code{\link{ExperimentRegistry}}: the algorithm identifier.}
#' }
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' f = function(x) if (x < 0) stop("x must be > 0") else sqrt(x)
#' batchMap(f, x = c(-1, 0, 1), reg = tmp)
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#' addJobTags(1:2, "tag1", reg = tmp)
#' addJobTags(2, "tag2", reg = tmp)
#'
#' # Complete table:
#' getJobTable(reg = tmp)
#'
#' # Job parameters:
#' getJobPars(reg = tmp)
#'
#' # Set and retrieve tags:
#' getJobTags(reg = tmp)
#'
#' # Job parameters with tags right-joined:
#' rjoin(getJobPars(reg = tmp), getJobTags(reg = tmp))
getJobTable = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
ids = convertIds(reg, ids)
getJobStatus(ids, reg = reg)[getJobPars(ids, reg = reg)][getJobResources(ids = ids, reg = reg)][getJobTags(ids = ids, reg = reg)]
}
#' @export
#' @rdname getJobTable
getJobStatus = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg, sync = TRUE)
submitted = started = done = NULL
cols = chsetdiff(names(reg$status), c("def.id", "resource.id"))
tab = filter(reg$status, convertIds(reg, ids), cols)
tab[, "submitted" := as.POSIXct(submitted, origin = "1970-01-01")]
tab[, "started" := as.POSIXct(started, origin = "1970-01-01")]
tab[, "done" := as.POSIXct(done, origin = "1970-01-01")]
tab[, "time.queued" := difftime(started, submitted, units = "secs")]
tab[, "time.running" := difftime(done, started, units = "secs")]
tab[]
}
#' @export
#' @rdname getJobTable
getJobResources = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
ids = convertIds(reg, ids)
tab = merge(filter(reg$status, ids, c("job.id", "resource.id")), reg$resources, all.x = TRUE, by = "resource.id")[, c("job.id", "resources")]
setkeyv(tab, "job.id")[]
}
#' @export
#' @rdname getJobTable
getJobPars = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
UseMethod("getJobPars", object = reg)
}
#' @export
getJobPars.Registry = function(ids = NULL, reg = getDefaultRegistry()) {
ids = convertIds(reg, ids)
tab = mergedJobs(reg, ids, c("job.id", "job.pars"))
setkeyv(tab, "job.id")[]
}
#' @export
getJobPars.ExperimentRegistry = function(ids = NULL, reg = getDefaultRegistry()) {
ids = convertIds(reg, ids)
tab = mergedJobs(reg, ids, c("job.id", "problem", "prob.pars", "algorithm", "algo.pars"))
setkeyv(tab, "job.id")[]
}
#' @export
#' @rdname getJobTable
getJobTags = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
ids = convertIds(reg, ids, default = allIds(reg))
tag = NULL
reg$tags[ids, on = "job.id"][, list(tags = stri_flatten(sort(tag, na.last = TRUE), ",")), by = "job.id"]
}
================================================
FILE: R/Joins.R
================================================
#' @title Inner, Left, Right, Outer, Semi and Anti Join for Data Tables
#' @name JoinTables
#'
#' @description
#' These helper functions perform join operations on data tables.
#' Most of them are basically one-liners.
#' See \url{https://rpubs.com/ronasta/join_data_tables} for a overview of join operations in
#' data table or alternatively \pkg{dplyr}'s vignette on two table verbs.
#'
#' @param x [\code{\link{data.frame}}]\cr
#' First data.frame to join.
#' @param y [\code{\link{data.frame}}]\cr
#' Second data.frame to join.
#' @param by [\code{character}]\cr
#' Column name(s) of variables used to match rows in \code{x} and \code{y}.
#' If not provided, a heuristic similar to the one described in the \pkg{dplyr} vignette is used:
#' \enumerate{
#' \item If \code{x} is keyed, the existing key will be used if \code{y} has the same column(s).
#' \item If \code{x} is not keyed, the intersect of common columns names is used if not empty.
#' \item Raise an exception.
#' }
#' You may pass a named character vector to merge on columns with different names in \code{x} and
#' \code{y}: \code{by = c("x.id" = "y.id")} will match \code{x}'s \dQuote{x.id} column with \code{y}\'s
#' \dQuote{y.id} column.
#' @return [\code{\link[data.table]{data.table}}] with key identical to \code{by}.
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' # Create two tables for demonstration
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMap(identity, x = 1:6, reg = tmp)
#' x = getJobPars(reg = tmp)
#' y = findJobs(x >= 2 & x <= 5, reg = tmp)
#' y$extra.col = head(letters, nrow(y))
#'
#' # Inner join: similar to intersect(): keep all columns of x and y with common matches
#' ijoin(x, y)
#'
#' # Left join: use all ids from x, keep all columns of x and y
#' ljoin(x, y)
#'
#' # Right join: use all ids from y, keep all columns of x and y
#' rjoin(x, y)
#'
#' # Outer join: similar to union(): keep all columns of x and y with matches in x or y
#' ojoin(x, y)
#'
#' # Semi join: filter x with matches in y
#' sjoin(x, y)
#'
#' # Anti join: filter x with matches not in y
#' ajoin(x, y)
#'
#' # Updating join: Replace values in x with values in y
#' ujoin(x, y)
ijoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
setKey(x[y, nomatch = 0L, on = by], by)
}
#' @rdname JoinTables
#' @export
ljoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
setKey(y[x, on = by], by)
}
#' @rdname JoinTables
#' @export
rjoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
setKey(x[y, on = by], by)
}
#' @rdname JoinTables
#' @export
ojoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
res = if (is.null(names(by)))
merge(x, y, all = TRUE, by = by)
else
merge(x, y, all = TRUE, by.x = names(by), by.y = by)
setKey(res, by)
}
#' @rdname JoinTables
#' @export
sjoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
w = unique(x[y, on = by, nomatch = 0L, which = TRUE, allow.cartesian = TRUE])
setKey(x[w], by)
}
#' @rdname JoinTables
#' @export
ajoin = function(x, y, by = NULL) {
x = as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
setKey(x[!y, on = by], by)
}
#' @rdname JoinTables
#' @param all.y [logical(1)]\cr
#' Keep columns of \code{y} which are not in \code{x}?
#' @export
ujoin = function(x, y, all.y = FALSE, by = NULL) {
assertFlag(all.y)
x = if (is.data.table(x)) copy(x) else as.data.table(x)
y = as.data.table(y)
by = guessBy(x, y, by)
cn = chsetdiff(names(y), by)
if (!all.y)
cn = chintersect(names(x), cn)
if (length(cn) == 0L)
return(x)
expr = parse(text = stri_join("`:=`(", stri_flatten(sprintf("%1$s=i.%1$s", cn), ","), ")"))
setKey(x[y, eval(expr), on = by], by)
}
guessBy = function(x, y, by = NULL) {
assertDataFrame(x, min.cols = 1L)
assertDataFrame(y, min.cols = 1L)
if (is.null(by)) {
res = key(x)
if (!is.null(res) && all(res %chin% names(y)))
return(res)
res = chintersect(names(x), names(y))
if (length(res) > 0L)
return(res)
stop("Unable to guess columns to match on. Please specify them explicitly or set keys beforehand.")
} else {
if (is.null(names(by))) {
assertSubset(by, names(x))
} else {
assertSubset(names(by), names(x))
}
assertSubset(by, names(y))
return(by)
}
}
setKey = function(res, by) {
by = names(by) %??% unname(by)
if (!identical(key(res), by))
setkeyv(res, by)
res[]
}
================================================
FILE: R/Logs.R
================================================
#' @useDynLib batchtools fill_gaps
readLog = function(id, missing.as.empty = FALSE, reg = getDefaultRegistry()) {
log.file = getLogFiles(reg, id)
if (is.na(log.file) || !waitForFile(log.file, timeout = reg$cluster.functions$fs.latency, must.work = FALSE)) {
if (missing.as.empty)
return(data.table(job.id = integer(0L), lines = character(0L)))
stopf("Log file '%s' for job with id %i not available", log.file, id$job.id)
}
lines = readLines(log.file)
if (length(lines) > 0L) {
job.id = as.integer(stri_match_last_regex(lines, c("\\[batchtools job\\.id=([0-9]+)\\]$"))[, 2L])
job.id = .Call(fill_gaps, job.id)
} else {
job.id = integer(0L)
}
setkeyv(data.table(job.id = job.id, lines = lines), "job.id", physical = FALSE)
}
extractLog = function(log, id) {
job.id = NULL
log[is.na(job.id) | job.id %in% id$job.id]$lines
}
#' @title Grep Log Files for a Pattern
#'
#' @description
#' Crawls through log files and reports jobs with lines matching the \code{pattern}.
#' See \code{\link{showLog}} for an example.
#'
#' @templateVar ids.default findStarted
#' @template ids
#' @param pattern [\code{character(1L)}]\cr
#' Regular expression or string (see \code{fixed}).
#' @param ignore.case [\code{logical(1L)}]\cr
#' If \code{TRUE} the match will be performed case insensitively.
#' @param fixed [\code{logical(1L)}]\cr
#' If \code{FALSE} (default), \code{pattern} is a regular expression and a fixed string otherwise.
#' @template reg
#' @export
#' @family debug
#' @return [\code{\link[data.table]{data.table}}] with columns \dQuote{job.id} and \dQuote{message}.
grepLogs = function(ids = NULL, pattern, ignore.case = FALSE, fixed = FALSE, reg = getDefaultRegistry()) {
assertRegistry(reg, sync = TRUE)
assertString(pattern, min.chars = 1L)
assertFlag(ignore.case)
assertFlag(fixed)
job.id = job.hash = log.file = matches = NULL
ids = convertIds(reg, ids, default = .findStarted(reg = reg))
tab = filter(reg$status[!is.na(job.hash)], ids)[, list(job.id = job.id, hash = sprintf("%s-%s", job.hash, log.file))]
if (nrow(tab) == 0L)
return(data.table(job.id = integer(0L), matches = character(0L)))
setorderv(tab, "hash")
res = data.table(job.id = tab$job.id, matches = NA_character_)
hash.before = ""
matcher = if (fixed) stri_detect_fixed else stri_detect_regex
for (i in seq_row(tab)) {
if (hash.before != tab$hash[i]) {
log = readLog(tab[i], missing.as.empty = TRUE, reg = reg)
hash.before = tab$hash[i]
}
if (nrow(log) > 0L) {
lines = extractLog(log, tab[i])
m = matcher(lines, pattern, case_insensitive = ignore.case)
if (any(m))
set(res, i, "matches", stri_flatten(lines[m], "\n"))
}
}
setkeyv(res[!is.na(matches)], "job.id")[]
}
#' @title Inspect Log Files
#'
#' @description
#' \code{showLog} opens the log in the pager. For customization, see \code{\link[base]{file.show}}.
#' \code{getLog} returns the log as character vector.
#' @template id
#' @template reg
#' @export
#' @family debug
#' @return Nothing.
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#'
#' # Create some dummy jobs
#' fun = function(i) {
#' if (i == 3) stop(i)
#' if (i %% 2 == 1) warning("That's odd.")
#' }
#' ids = batchMap(fun, i = 1:5, reg = tmp)
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#' getStatus(reg = tmp)
#'
#' writeLines(getLog(ids[1], reg = tmp))
#' \dontrun{
#' showLog(ids[1], reg = tmp)
#' }
#'
#' grepLogs(pattern = "warning", ignore.case = TRUE, reg = tmp)
showLog = function(id, reg = getDefaultRegistry()) {
assertRegistry(reg, sync = TRUE)
id = convertId(reg, id)
lines = extractLog(readLog(id, reg = reg), id)
log.file = fs::path(fs::path_temp(), sprintf("%i.log", id$job.id))
writeLines(text = lines, con = log.file)
file.show(log.file, delete.file = TRUE)
}
#' @export
#' @rdname showLog
getLog = function(id, reg = getDefaultRegistry()) {
assertRegistry(reg, sync = TRUE)
id = convertId(reg, id)
extractLog(readLog(id, reg = reg), id)
}
================================================
FILE: R/Problem.R
================================================
#' @title Define Problems for Experiments
#'
#' @description
#' Problems may consist of up to two parts: A static, immutable part (\code{data} in \code{addProblem})
#' and a dynamic, stochastic part (\code{fun} in \code{addProblem}).
#' For example, for statistical learning problems a data frame would be the static problem part while
#' a resampling function would be the stochastic part which creates problem instance.
#' This instance is then typically passed to a learning algorithm like a wrapper around a statistical model
#' (\code{fun} in \code{\link{addAlgorithm}}).
#'
#' This function serialize all components to the file system and registers the problem in the \code{\link{ExperimentRegistry}}.
#'
#' \code{removeProblem} removes all jobs from the registry which depend on the specific problem.
#' \code{reg$problems} holds the IDs of already defined problems.
#'
#' @param name [\code{character(1)}]\cr
#' Unique identifier for the problem.
#' @param data [\code{ANY}]\cr
#' Static problem part. Default is \code{NULL}.
#' @param fun [\code{function}]\cr
#' The function defining the stochastic problem part.
#' The static part is passed to this function with name \dQuote{data} and the \code{\link{Job}}/\code{\link{Experiment}}
#' is passed as \dQuote{job}.
#' Therefore, your function must have the formal arguments \dQuote{job} and \dQuote{data} (or dots \code{...}).
#' If you do not provide a function, it defaults to a function which just returns the data part.
#' @param seed [\code{integer(1)}]\cr
#' Start seed for this problem. This allows the \dQuote{synchronization} of a stochastic
#' problem across algorithms, so that different algorithms are evaluated on the same stochastic instance.
#' If the problem seed is defined, the seeding mechanism works as follows:
#' (1) Before the dynamic part of a problem is instantiated,
#' the seed of the problem + [replication number] - 1 is set, i.e. the first
#' replication uses the problem seed. (2) The stochastic part of the problem is
#' instantiated. (3) From now on the usual experiment seed of the registry is used,
#' see \code{\link{ExperimentRegistry}}.
#' If \code{seed} is set to \code{NULL} (default), the job seed is used to instantiate the problem and
#' different algorithms see different stochastic instances of the same problem.
#' @param cache [\code{logical(1)}]\cr
#' If \code{TRUE} and \code{seed} is set, problem instances will be cached on the file system.
#' This assumes that each problem instance is deterministic for each combination of hyperparameter setting
#' and each replication number.
#' This feature is experimental.
#' @template expreg
#' @return [\code{Problem}]. Object of class \dQuote{Problem} (invisibly).
#' @aliases Problem
#' @seealso \code{\link{Algorithm}}, \code{\link{addExperiments}}
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)
#' addProblem("p1", fun = function(job, data) data, reg = tmp)
#' addProblem("p2", fun = function(job, data) job, reg = tmp)
#' addAlgorithm("a1", fun = function(job, data, instance) instance, reg = tmp)
#' addExperiments(repls = 2, reg = tmp)
#'
#' # List problems, algorithms and job parameters:
#' tmp$problems
#' tmp$algorithms
#' getJobPars(reg = tmp)
#'
#' # Remove one problem
#' removeProblems("p1", reg = tmp)
#'
#' # List problems and algorithms:
#' tmp$problems
#' tmp$algorithms
#' getJobPars(reg = tmp)
addProblem = function(name, data = NULL, fun = NULL, seed = NULL, cache = FALSE, reg = getDefaultRegistry()) {
assertRegistry(reg, class = "ExperimentRegistry", writeable = TRUE)
assertString(name, min.chars = 1L)
if (!stri_detect_regex(name, "^[[:alnum:]_.-]+$"))
stopf("Illegal characters in problem name: %s", name)
if (is.null(fun)) {
fun = function(job, data, ...) data
} else {
assert(checkFunction(fun, args = c("job", "data")), checkFunction(fun, args = "..."))
}
if (is.null(seed)) {
cache = FALSE
} else {
seed = asCount(seed, positive = TRUE)
cache = assertFlag(cache)
}
info("Adding problem '%s'", name)
prob = setClasses(list(name = name, seed = seed, cache = cache, data = data, fun = fun), "Problem")
writeRDS(prob, file = getProblemURI(reg, name), compress = reg$compress)
reg$problems = union(reg$problems, name)
cache.dir = getProblemCacheDir(reg, name)
if (fs::dir_exists(cache.dir))
fs::dir_delete(cache.dir)
if (cache)
fs::dir_create(cache.dir)
saveRegistry(reg)
invisible(prob)
}
#' @export
#' @rdname addProblem
removeProblems = function(name, reg = getDefaultRegistry()) {
assertRegistry(reg, class = "ExperimentRegistry", writeable = TRUE, running.ok = FALSE)
assertCharacter(name, any.missing = FALSE)
assertSubset(name, reg$problems)
problem = NULL
for (nn in name) {
def.ids = reg$defs[problem == nn, "def.id"]
job.ids = filter(def.ids, reg$status, "job.id")
info("Removing Problem '%s' and %i corresponding jobs ...", nn, nrow(job.ids))
file_remove(getProblemURI(reg, nn))
reg$defs = reg$defs[!def.ids]
reg$status = reg$status[!job.ids]
reg$problems = chsetdiff(reg$problems, nn)
cache = getProblemCacheDir(reg, nn)
if (fs::dir_exists(cache))
fs::dir_delete(cache)
}
sweepRegistry(reg)
invisible(TRUE)
}
getProblemURI = function(reg, name) {
fs::path(dir(reg, "problems"), mangle(name))
}
getProblemCacheDir = function(reg, name) {
fs::path(dir(reg, "cache"), "problems", base32_encode(name, use.padding = FALSE))
}
getProblemCacheURI = function(job) {
fs::path(getProblemCacheDir(job, job$prob.name), sprintf("%s.rds", digest(list(job$prob.name, job$prob.pars, job$repl))))
}
================================================
FILE: R/RDSReader.R
================================================
RDSReader = R6Class("RDSReader",
cloneable = FALSE,
public = list(
cache = list(),
use.cache = NA,
initialize = function(use.cache = FALSE) {
self$use.cache = use.cache
},
get = function(uri, slot = NA_character_) {
read = function(uri) if (fs::file_exists(uri)) readRDS(uri) else NULL
# no cache used, read object from disk and return
if (!self$use.cache)
return(read(uri))
# not slotted:
# look up object in cache. If not found, add to cache. Return cached object
if (is.na(slot)) {
if (! uri %chin% names(self$cache))
self$cache[[uri]] = read(uri)
return(self$cache[[uri]])
}
# slotted:
# object is stored in cache[[slot]] as list(obj = [cached obj], uri = uri)
if (is.null(self$cache[[slot]]) || self$cache[[slot]]$uri != uri)
self$cache[[slot]] = list(obj = read(uri), uri = uri)
return(self$cache[[slot]]$obj)
}
)
)
================================================
FILE: R/Registry.R
================================================
#' @title Registry Constructor
#'
#' @description
#' \code{makeRegistry} constructs the inter-communication object for all functions in \code{batchtools}.
#' All communication transactions are processed via the file system:
#' All information required to run a job is stored as \code{\link{JobCollection}} in a file in the
#' a subdirectory of the \code{file.dir} directory.
#' Each jobs stores its results as well as computational status information (start time, end time, error message, ...)
#' also on the file system which is regular merged parsed by the master using \code{\link{syncRegistry}}.
#' After integrating the new information into the Registry, the Registry is serialized to the file system via \code{\link{saveRegistry}}.
#' Both \code{\link{syncRegistry}} and \code{\link{saveRegistry}} are called whenever required internally.
#' Therefore it should be safe to quit the R session at any time.
#' Work can later be resumed by calling \code{\link{loadRegistry}} which de-serializes the registry from
#' the file system.
#'
#' The registry created last is saved in the package namespace (unless \code{make.default} is set to
#' \code{FALSE}) and can be retrieved via \code{\link{getDefaultRegistry}}.
#'
#' Canceled jobs and jobs submitted multiple times may leave stray files behind.
#' These can be swept using \code{\link{sweepRegistry}}.
#' \code{\link{clearRegistry}} completely erases all jobs from a registry, including log files and results,
#' and thus allows you to start over.
#'
#' @details
#' Currently \pkg{batchtools} understands the following options set via the configuration file:
#' \describe{
#' \item{\code{cluster.functions}:}{As returned by a constructor, e.g. \code{\link{makeClusterFunctionsSlurm}}.}
#' \item{\code{default.resources}:}{List of resources to use. Will be overruled by resources specified via \code{\link{submitJobs}}.}
#' \item{\code{temp.dir}:}{Path to directory to use for temporary registries.}
#' \item{\code{sleep}:}{Custom sleep function. See \code{\link{waitForJobs}}.}
#' \item{\code{expire.after}:}{Number of iterations before treating jobs as expired in \code{\link{waitForJobs}}.}
#' \item{\code{compress}:}{Compression algorithm to use via \code{\link{saveRDS}}.}
#' }
#'
#' @param file.dir [\code{character(1)}]\cr
#' Path where all files of the registry are saved.
#' Default is directory \dQuote{registry} in the current working directory.
#' The provided path will get normalized unless it is given relative to the home directory
#' (i.e., starting with \dQuote{~}). Note that some templates do not handle relative paths well.
#'
#' If you pass \code{NA}, a temporary directory will be used.
#' This way, you can create disposable registries for \code{\link{btlapply}} or examples.
#' By default, the temporary directory \code{\link[base]{tempdir}()} will be used.
#' If you want to use another directory, e.g. a directory which is shared between nodes,
#' you can set it in your configuration file by setting the variable \code{temp.dir}.
#' @param work.dir [\code{character(1)}]\cr
#' Working directory for R process for running jobs.
#' Defaults to the working directory currently set during Registry construction (see \code{\link[base]{getwd}}).
#' \code{loadRegistry} uses the stored \code{work.dir}, but you may also explicitly overwrite it,
#' e.g., after switching to another system.
#'
#' The provided path will get normalized unless it is given relative to the home directory
#' (i.e., starting with \dQuote{~}). Note that some templates do not handle relative paths well.
#' @param conf.file [\code{character(1)}]\cr
#' Path to a configuration file which is sourced while the registry is created.
#' In the configuration file you can define how \pkg{batchtools} interacts with the system via \code{\link{ClusterFunctions}}.
#' Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.
#'
#' The file lookup is implemented in the internal (but exported) function \code{findConfFile} which returns the first file found of the following candidates:
#' \enumerate{
#' \item{File \dQuote{batchtools.conf.R} in the path specified by the environment variable \dQuote{R_BATCHTOOLS_SEARCH_PATH}.}
#' \item{File \dQuote{batchtools.conf.R} in the current working directory.}
#' \item{File \dQuote{config.R} in the user configuration directory as reported by \code{rappdirs::user_config_dir("batchtools", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \dQuote{~/.config/batchtools/config.R}).}
#' \item{\dQuote{.batchtools.conf.R} in the home directory (\dQuote{~}).}
#' \item{\dQuote{config.R} in the site config directory as reported by \code{rappdirs::site_config_dir("batchtools")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}
#' }
#' Set to \code{NA} if you want to suppress reading any configuration file.
#' If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.
#' Therefore you can set and overwrite slots, e.g. \code{default.resources = list(walltime = 3600)} to set default resources or \dQuote{max.concurrent.jobs} to
#' limit the number of jobs allowed to run simultaneously on the system.
#' @param packages [\code{character}]\cr
#' Packages that will always be loaded on each node.
#' Uses \code{\link[base]{require}} internally.
#' Default is \code{character(0)}.
#' @param namespaces [\code{character}]\cr
#' Same as \code{packages}, but the packages will not be attached.
#' Uses \code{\link[base]{requireNamespace}} internally.
#' Default is \code{character(0)}.
#' @param source [\code{character}]\cr
#' Files which should be sourced on the slaves prior to executing a job.
#' Calls \code{\link[base]{sys.source}} using the \code{\link[base]{.GlobalEnv}}.
#' @param load [\code{character}]\cr
#' Files which should be loaded on the slaves prior to executing a job.
#' Calls \code{\link[base]{load}} using the \code{\link[base]{.GlobalEnv}}.
#' @param seed [\code{integer(1)}]\cr
#' Start seed for jobs. Each job uses the (\code{seed} + \code{job.id}) as seed.
#' Default is a random integer between 1 and 32768.
#' Note that there is an additional seeding mechanism to synchronize instantiation of
#' \code{\link{Problem}}s in a \code{\link{ExperimentRegistry}}.
#' @param make.default [\code{logical(1)}]\cr
#' If set to \code{TRUE}, the created registry is saved inside the package
#' namespace and acts as default registry. You might want to switch this
#' off if you work with multiple registries simultaneously.
#' Default is \code{TRUE}.
#' @return [\code{environment}] of class \dQuote{Registry} with the following slots:
#' \describe{
#' \item{\code{file.dir} [path]:}{File directory.}
#' \item{\code{work.dir} [path]:}{Working directory.}
#' \item{\code{temp.dir} [path]:}{Temporary directory. Used if \code{file.dir} is \code{NA} to create temporary registries.}
#' \item{\code{packages} [character()]:}{Packages to load on the slaves.}
#' \item{\code{namespaces} [character()]:}{Namespaces to load on the slaves.}
#' \item{\code{seed} [integer(1)]:}{Registry seed. Before each job is executed, the seed \code{seed + job.id} is set.}
#' \item{\code{cluster.functions} [cluster.functions]:}{Usually set in your \code{conf.file}. Set via a call to \code{\link{makeClusterFunctions}}. See example.}
#' \item{\code{default.resources} [named list()]:}{Usually set in your \code{conf.file}. Named list of default resources.}
#' \item{\code{max.concurrent.jobs} [integer(1)]:}{Usually set in your \code{conf.file}. Maximum number of concurrent jobs for a single user and current registry on the system.
#' \code{\link{submitJobs}} will try to respect this setting. The resource \dQuote{max.concurrent.jobs} has higher precedence.}
#' \item{\code{defs} [data.table]:}{Table with job definitions (i.e. parameters).}
#' \item{\code{status} [data.table]:}{Table holding information about the computational status. Also see \code{\link{getJobStatus}}.}
#' \item{\code{resources} [data.table]:}{Table holding information about the computational resources used for the job. Also see \code{\link{getJobResources}}.}
#' \item{\code{tags} [data.table]:}{Table holding information about tags. See \link{Tags}.}
#' \item{\code{hash} [character(1)]:}{Unique hash which changes each time the registry gets saved to the file system. Can be utilized to invalidate the cache of \pkg{knitr}.}
#' }
#' @aliases Registry
#' @family Registry
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' print(tmp)
#'
#' # Set cluster functions to interactive mode and start jobs in external R sessions
#' tmp$cluster.functions = makeClusterFunctionsInteractive(external = TRUE)
#'
#' # Change packages to load
#' tmp$packages = c("MASS")
#' saveRegistry(reg = tmp)
makeRegistry = function(file.dir = "registry", work.dir = getwd(), conf.file = findConfFile(), packages = character(0L), namespaces = character(0L),
source = character(0L), load = character(0L), seed = NULL, make.default = TRUE) {
assertString(file.dir, na.ok = TRUE)
if (!is.na(file.dir))
assertPathForOutput(file.dir, overwrite = FALSE)
assertString(work.dir)
assertDirectoryExists(work.dir, access = "r")
assertString(conf.file, na.ok = TRUE)
assertCharacter(packages, any.missing = FALSE, min.chars = 1L)
assertCharacter(namespaces, any.missing = FALSE, min.chars = 1L)
assertCharacter(source, any.missing = FALSE, min.chars = 1L)
assertCharacter(load, any.missing = FALSE, min.chars = 1L)
assertFlag(make.default)
seed = if (is.null(seed)) as.integer(runif(1L, 1, 32768)) else asCount(seed, positive = TRUE)
reg = new.env(parent = asNamespace("batchtools"))
reg$file.dir = file.dir
reg$work.dir = work.dir
reg$packages = packages
reg$namespaces = namespaces
reg$source = source
reg$load = load
reg$seed = seed
reg$writeable = TRUE
reg$version = packageVersion("batchtools")
reg$defs = data.table(
def.id = integer(0L),
job.pars = list(),
key = "def.id")
reg$status = data.table(
job.id = integer(0L),
def.id = integer(0L),
submitted = double(0L),
started = double(0L),
done = double(0L),
error = character(0L),
mem.used = double(0L),
resource.id = integer(0L),
batch.id = character(0L),
log.file = character(0L),
job.hash = character(0L),
job.name = character(0L),
key = "job.id")
reg$resources = data.table(
resource.id = integer(0L),
resource.hash = character(0L),
resources = list(),
key = "resource.id")
reg$tags = data.table(
job.id = integer(0L),
tag = character(0L),
key = "job.id")
setSystemConf(reg, conf.file)
if (is.na(file.dir))
reg$file.dir = fs::file_temp("registry", tmp_dir = reg$temp.dir)
"!DEBUG [makeRegistry]: Creating directories in '`reg$file.dir`'"
fs::dir_create(c(reg$file.dir, reg$work.dir))
reg$file.dir = fs::path_abs(reg$file.dir)
reg$work.dir = fs::path_abs(reg$work.dir)
fs::dir_create(fs::path(reg$file.dir, c("jobs", "results", "updates", "logs", "exports", "external")))
with_dir(reg$work.dir, loadRegistryDependencies(reg))
class(reg) = "Registry"
saveRegistry(reg)
reg$mtime = file_mtime(fs::path(reg$file.dir, "registry.rds"))
reg$hash = rnd_hash()
info("Created registry in '%s' using cluster functions '%s'", reg$file.dir, reg$cluster.functions$name)
if (make.default)
batchtools$default.registry = reg
return(reg)
}
#' @export
print.Registry = function(x, ...) {
cat("Job Registry\n")
catf(" Backend : %s", x$cluster.functions$name)
catf(" File dir : %s", x$file.dir)
catf(" Work dir : %s", x$work.dir)
catf(" Jobs : %i", nrow(x$status))
catf(" Seed : %i", x$seed)
catf(" Writeable: %s", x$writeable)
}
#' @title assertRegistry
#'
#' @description
#' Assert that a given object is a \code{batchtools} registry.
#' Additionally can sync the registry, check if it is writeable, or check if jobs are running.
#' If any check fails, throws an error indicting the reason for the failure.
#'
#' @param reg [\code{\link{Registry}}]\cr
#' The object asserted to be a \code{Registry}.
#' @param class [\code{character(1)}]\cr
#' If \code{NULL} (default), \code{reg} must only inherit from class \dQuote{Registry}.
#' Otherwise check that \code{reg} is of class \code{class}.
#' E.g., if set to \dQuote{Registry}, a \code{\link{ExperimentRegistry}} would not pass.
#' @param writeable [\code{logical(1)}]\cr
#' Check if the registry is writeable.
#' @param sync [\code{logical(1)}]\cr
#' Try to synchronize the registry by including pending results from the file system.
#' See \code{\link{syncRegistry}}.
#' @param running.ok [\code{logical(1)}]\cr
#' If \code{FALSE} throw an error if jobs associated with the registry are currently running.
#' @return \code{TRUE} invisibly.
#' @export
assertRegistry = function(reg, class = NULL, writeable = FALSE, sync = FALSE, running.ok = TRUE) {
if (batchtools$debug) {
if (!identical(key(reg$status), "job.id"))
stop("Key of reg$job.id lost")
if (!identical(key(reg$defs), "def.id"))
stop("Key of reg$defs lost")
if (!identical(key(reg$resources), "resource.id"))
stop("Key of reg$resources lost")
}
if (is.null(class)) {
assertClass(reg, "Registry")
} else {
assertString(class)
assertClass(reg, class, ordered = TRUE)
}
assertFlag(writeable)
assertFlag(sync)
assertFlag(running.ok)
if (reg$writeable && file_mtime(fs::path(reg$file.dir, "registry.rds")) > reg$mtime + 1) {
warning("Registry has been altered since last read. Switching to read-only mode in this session. See ?loadRegistry.")
reg$writeable = FALSE
}
if (writeable && !reg$writeable)
stop("Registry must be writeable. See ?loadRegistry.")
if (!running.ok && nrow(.findOnSystem(reg = reg)) > 0L)
stop("This operation is not allowed while jobs are running on the system")
if (sync) {
merged = sync(reg)
if (length(merged)) {
saveRegistry(reg)
file_remove(merged)
}
}
invisible(TRUE)
}
loadRegistryDependencies = function(x, must.work = FALSE) {
"!DEBUG [loadRegistryDependencies]: Starting ..."
pkgs = union(x$packages, "methods")
handler = if (must.work) stopf else warningf
ok = vlapply(pkgs, require, character.only = TRUE)
if (!all(ok))
handler("Failed to load packages: %s", stri_flatten(pkgs[!ok], ", "))
ok = vlapply(x$namespaces, requireNamespace)
if (!all(ok))
handler("Failed to load namespaces: %s", stri_flatten(x$namespaces[!ok], ", "))
if (length(x$source) > 0L) {
for (fn in x$source) {
ok = try(sys.source(fn, envir = .GlobalEnv), silent = TRUE)
if (is.error(ok))
handler("Failed to source file '%s': %s", fn, as.character(ok))
}
}
if (length(x$load) > 0L) {
for (fn in x$load) {
ok = try(load(fn, envir = .GlobalEnv), silent = TRUE)
if (is.error(ok))
handler("Failed to load file '%s': %s", fn, as.character(ok))
}
}
path = fs::path(x$file.dir, "exports")
fns = list.files(path, pattern = "\\.rds$")
if (length(fns) > 0L) {
ee = .GlobalEnv
Map(function(name, fn) {
delayedAssign(x = name, value = readRDS(fn), assign.env = ee)
}, name = unmangle(fns), fn = fs::path(path, fns))
}
invisible(TRUE)
}
================================================
FILE: R/Tags.R
================================================
#' @title Add or Remove Job Tags
#' @name Tags
#' @rdname Tags
#'
#' @description
#' Add and remove arbitrary tags to jobs.
#'
#' @templateVar ids.default all
#' @template ids
#' @param tags [\code{character}]\cr
#' Tags to add or remove as strings. Each tag may consist of letters, numbers, underscore and dots (pattern \dQuote{^[[:alnum:]_.]+}).
#' @return [\code{\link[data.table]{data.table}}] with job ids affected (invisible).
#' @template reg
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' ids = batchMap(sqrt, x = -3:3, reg = tmp)
#'
#' # Add new tag to all ids
#' addJobTags(ids, "needs.computation", reg = tmp)
#' getJobTags(reg = tmp)
#'
#' # Add more tags
#' addJobTags(findJobs(x < 0, reg = tmp), "x.neg", reg = tmp)
#' addJobTags(findJobs(x > 0, reg = tmp), "x.pos", reg = tmp)
#' getJobTags(reg = tmp)
#'
#' # Submit first 5 jobs and remove tag if successful
#' ids = submitJobs(1:5, reg = tmp)
#' if (waitForJobs(reg = tmp))
#' removeJobTags(ids, "needs.computation", reg = tmp)
#' getJobTags(reg = tmp)
#'
#' # Grep for warning message and add a tag
#' addJobTags(grepLogs(pattern = "NaNs produced", reg = tmp), "div.zero", reg = tmp)
#' getJobTags(reg = tmp)
#'
#' # All tags where tag x.neg is set:
#' ids = findTagged("x.neg", reg = tmp)
#' getUsedJobTags(ids, reg = tmp)
addJobTags = function(ids = NULL, tags, reg = getDefaultRegistry()) {
assertRegistry(reg, writeable = TRUE)
ids = convertIds(reg, ids, default = allIds(reg))
assertCharacter(tags, any.missing = FALSE, pattern = "^[[:alnum:]_.]+$", min.len = 1L)
for (cur in tags) {
ids[, ("tag") := cur]
reg$tags = rbind(reg$tags, ids)
}
reg$tags = setkeyv(unique(reg$tags, by = NULL), "job.id")
saveRegistry(reg)
invisible(ids[, "job.id"])
}
#' @export
#' @rdname Tags
removeJobTags = function(ids = NULL, tags, reg = getDefaultRegistry()) {
assertRegistry(reg, writeable = TRUE)
ids = convertIds(reg, ids)
assertCharacter(tags, any.missing = FALSE, pattern = "^[[:alnum:]_.]+$", min.len = 1L)
job.id = tag = NULL
if (is.null(ids)) {
i = reg$tags[tag %in% tags, which = TRUE]
} else {
i = reg$tags[job.id %in% ids$job.id & tag %in% tags, which = TRUE]
}
if (length(i) > 0L) {
ids = unique(reg$tags[i, "job.id"], by = "job.id")
reg$tags = reg$tags[-i]
saveRegistry(reg)
} else {
ids = noIds()
}
invisible(ids)
}
#' @export
#' @rdname Tags
getUsedJobTags = function(ids = NULL, reg = getDefaultRegistry()) {
assertRegistry(reg)
ids = convertIds(reg, ids)
unique(filter(reg$tags, ids), by = "tag")$tag
}
================================================
FILE: R/Worker.R
================================================
#' @title Create a Linux-Worker
#' @docType class
#' @format An \code{\link[R6]{R6Class}} generator object
#'
#' @description
#' \code{\link[R6]{R6Class}} to create local and remote linux workers.
#'
#' @field nodename Host name. Set via constructor.
#' @field ncpus Number of CPUs. Set via constructor and defaults to a heuristic which tries to detect the number of CPUs of the machine.
#' @field max.load Maximum load average (of the last 5 min). Set via constructor and defaults to the number of CPUs of the machine.
#' @field status Status of the worker; one of \dQuote{unknown}, \dQuote{available}, \dQuote{max.cpus} and \dQuote{max.load}.
#' @section Methods:
#' \describe{
#' \item{\code{new(nodename, ncpus, max.load)}}{Constructor.}
#' \item{\code{update(reg)}}{Update the worker status.}
#' \item{\code{list(reg)}}{List running jobs.}
#' \item{\code{start(reg, fn, outfile)}}{Start job collection in file \dQuote{fn} and output to \dQuote{outfile}.}
#' \item{\code{kill(reg, batch.id)}}{Kill job matching the \dQuote{batch.id}.}
#' }
#' @return [\code{\link{Worker}}].
#' @export
#' @examples
#' \dontrun{
#' # create a worker for the local machine and use 4 CPUs.
#' Worker$new("localhost", ncpus = 4)
#' }
Worker = R6Class("Worker",
cloneable = FALSE,
public = list(
nodename = NULL,
ncpus = NULL,
max.load = NULL,
script = NULL,
status = "unknown",
initialize = function(nodename, ncpus = NULL, max.load = NULL) {
if (testOS("windows"))
stop("Windows is not supported by the Worker Class")
self$nodename = assertString(nodename)
if (!is.null(ncpus))
ncpus = asCount(ncpus)
if (!is.null(max.load))
assertNumber(max.load)
if (nodename == "localhost") {
self$script = system.file("bin", "linux-helper", package = "batchtools")
} else {
args = c("-e", shQuote("message(\"[bt] --BOF--\\n\", \"[bt] \", system.file(\"bin/linux-helper\", package = \"batchtools\"), \"\\n[bt] --EOF--\\n\")"))
res = runOSCommand("Rscript", args, nodename = nodename)
script = private$filter_output(res)$output
if (!testString(script, min.chars = 1L)) {
stopf("Unable to locate helper script on SSH node '%s'. Is batchtools installed on the node?", nodename)
}
}
self$ncpus = ncpus %??% as.integer(private$run("number-of-cpus")$output)
self$max.load = max.load %??% self$ncpus
},
list = function(reg) {
stri_join(self$nodename, "#", stri_trim_both(private$run(c("list-jobs", reg$file.dir))$output))
},
start = function(reg, fn, outfile) {
private$run(c("start-job", fn, outfile))
},
kill = function(reg, batch.id) {
pid = stri_split_fixed(batch.id, "#", n = 2L)[[1L]][2L]
cfKillJob(reg, self$script, c("kill-job", pid))
},
update = function(reg) {
"!DEBUG [Worker]: Updating Worker '`self$nodename`'"
res = private$run(c("status", reg$file.dir))
res = as.numeric(stri_split_regex(res$output, "\\s+")[[1L]])
names(res) = c("load", "n.rprocs", "n.rprocs.50", "n.jobs")
self$status = if (res["load"] > self$max.load) {
"max.load"
} else if (res["n.jobs"] >= self$ncpus) {
"max.cpus"
} else {
"available"
}
return(res)
}
),
private = list(
filter_output = function(res) {
output = stri_trim_both(res$output)
marker = stri_detect_regex(output, "^\\[bt\\] --[BE]OF--$")
if (sum(marker) != 2L) {
stopf("runOSCommand failed: Expected BOF+EOF markers for '%s %s', but got:\n %s",
res$sys.cmd, stri_flatten(res$sys.args, " "), stri_flatten(res$output, "\n") %??% "")
}
info = stri_startswith_fixed(output, "[bt]") & !marker
res$output = stri_trim_left(stri_sub(output[info], 5L))
res
},
run = function(args) {
private$filter_output(runOSCommand(self$script, args, nodename = self$nodename))
}
)
)
================================================
FILE: R/addExperiments.R
================================================
#' @title Add Experiments to the Registry
#'
#' @description
#' Adds experiments (parametrized combinations of problems with algorithms) to the registry and thereby defines batch jobs.
#'
#' If multiple problem designs or algorithm designs are provided, they are combined via the Cartesian product.
#' E.g., if you have two problems \code{p1} and \code{p2} and three algorithms \code{a1}, \code{a2} and \code{a3},
#' \code{addExperiments} creates experiments for all parameters for the combinations \code{(p1, a1)}, \code{(p1, a2)},
#' \code{(p1, a3)}, \code{(p2, a1)}, \code{(p2, a2)} and \code{(p2, a3)}.
#'
#' @note
#' R's \code{data.frame} converts character vectors to factors by default in R versions prior to 4.0.0 which frequently resulted in problems using \code{addExperiments}.
#' Therefore, this function will warn about factor variables if the following conditions hold:
#' \enumerate{
#' \item R version is < 4.0.0
#' \item The design is passed as a \code{data.frame}, not a \code{\link[data.table]{data.table}} or \code{\link[tibble]{tibble}}.
#' \item The option \dQuote{stringsAsFactors} is not set or set to \code{TRUE}.
#' }
#'
#' @param prob.designs [named list of \code{\link[base]{data.frame}}]\cr
#' Named list of data frames (or \code{\link[data.table]{data.table}}).
#' The name must match the problem name while the column names correspond to parameters of the problem.
#' If \code{NULL}, experiments for all defined problems without any parameters are added.
#' @param algo.designs [named list of \code{\link[data.table]{data.table}} or \code{\link[base]{data.frame}}]\cr
#' Named list of data frames (or \code{\link[data.table]{data.table}}).
#' The name must match the algorithm name while the column names correspond to parameters of the algorithm.
#' If \code{NULL}, experiments for all defined algorithms without any parameters are added.
#' @param repls [\code{integer()}]\cr
#' Number of replications for each problem design in `prob.designs` (automatically replicated to
#' the correct length).
#' @param combine [\code{character(1)}]\cr
#' How to combine the rows of a single problem design with the rows of a single algorithm design?
#' Default is \dQuote{crossprod} which combines each row of the problem design which each row of the algorithm design
#' in a cross-product fashion. Set to \dQuote{bind} to just \code{\link[base]{cbind}} the tables of
#' problem and algorithm designs where the shorter table is repeated if necessary.
#' @template expreg
#' @return [\code{\link[data.table]{data.table}}] with ids of added jobs stored in column \dQuote{job.id}.
#' @export
#' @family Experiment
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)
#'
#' # add first problem
#' fun = function(job, data, n, mean, sd, ...) rnorm(n, mean = mean, sd = sd)
#' addProblem("rnorm", fun = fun, reg = tmp)
#'
#' # add second problem
#' fun = function(job, data, n, lambda, ...) rexp(n, rate = lambda)
#' addProblem("rexp", fun = fun, reg = tmp)
#'
#' # add first algorithm
#' fun = function(instance, method, ...) if (method == "mean") mean(instance) else median(instance)
#' addAlgorithm("average", fun = fun, reg = tmp)
#'
#' # add second algorithm
#' fun = function(instance, ...) sd(instance)
#' addAlgorithm("deviation", fun = fun, reg = tmp)
#'
#' # define problem and algorithm designs
#' library(data.table)
#' prob.designs = algo.designs = list()
#' prob.designs$rnorm = CJ(n = 100, mean = -1:1, sd = 1:5)
#' prob.designs$rexp = data.table(n = 100, lambda = 1:5)
#' algo.designs$average = data.table(method = c("mean", "median"))
#' algo.designs$deviation = data.table()
#'
#' # add experiments and submit
#' addExperiments(prob.designs, algo.designs, reg = tmp)
#'
#' # check what has been created
#' summarizeExperiments(reg = tmp)
#' unwrap(getJobPars(reg = tmp))
addExperiments = function(prob.designs = NULL, algo.designs = NULL, repls = 1L, combine = "crossprod", reg = getDefaultRegistry()) {
convertDesigns = function(type, designs, keywords) {
check.factors = getRversion() < "4.0.0" && default.stringsAsFactors()
Map(function(id, design) {
if (check.factors && identical(class(design)[1L], "data.frame")) {
i = which(vlapply(design, is.factor))
if (length(i) > 0L) {
warningf("%s design '%s' passed as 'data.frame' and 'stringsAsFactors' is TRUE. Column(s) '%s' may be encoded as factors accidentally.", type, id, stri_flatten(names(design)[i]), "','")
}
}
if (!is.data.table(design))
design = as.data.table(design)
i = wf(keywords %chin% names(design))
if (length(i) > 0L)
stopf("%s design %s contains reserved keyword '%s'", type, id, keywords[i])
design
}, id = names(designs), design = designs)
}
increment = function(ids, n = 1L) {
if (length(ids) == 0L) seq_len(n) else max(ids) + seq_len(n)
}
assertRegistry(reg, class = "ExperimentRegistry", writeable = TRUE)
if (is.null(prob.designs)) {
prob.designs = replicate(length(reg$problems), data.table(), simplify = FALSE)
names(prob.designs) = reg$problems
} else {
assertList(prob.designs, types = "data.frame", names = "named")
assertSubset(names(prob.designs), reg$problems)
prob.designs = convertDesigns("Problem", prob.designs, c("job", "data"))
}
if (is.null(algo.designs)) {
algo.designs = replicate(length(reg$algorithms), data.table(), simplify = FALSE)
names(algo.designs) = reg$algorithms
} else {
assertList(algo.designs, types = "data.frame", names = "named")
assertSubset(names(algo.designs), reg$algorithms)
algo.designs = convertDesigns("Algorithm", algo.designs, c("job", "data", "instance"))
}
repls = asInteger(repls, lower = 1L, any.missing = FALSE)
repls = rep_len(repls, length(prob.designs))
assertChoice(combine, c("crossprod", "bind"))
all.ids = integer(0L)
for (i in seq_along(prob.designs)) {
pn = names(prob.designs)[i]
pd = prob.designs[[i]]
n.pd = max(nrow(pd), 1L)
repls_cur = repls[i]
for (j in seq_along(algo.designs)) {
an = names(algo.designs)[j]
ad = algo.designs[[j]]
n.ad = max(nrow(ad), 1L)
if (combine == "crossprod") {
n.jobs = n.pd * n.ad * repls_cur
info("Adding %i experiments ('%s'[%i] x '%s'[%i] x repls[%i]) ...", n.jobs, pn, n.pd, an, n.ad, repls_cur)
idx = CJ(.i = seq_len(n.pd), .j = seq_len(n.ad))
} else {
recycle = max(n.pd, n.ad)
n.jobs = recycle * repls_cur
info("Adding %i experiments (('%s'[%i] | '%s'[%i]) x repls[%i]) ...", n.jobs, pn, n.pd, an, n.ad, repls_cur)
idx = data.table(.i = rep_len(seq_len(n.pd), recycle), .j = rep_len(seq_len(n.ad), recycle))
}
# create temp tab with prob name, algo name and pars as list
tab = data.table(
problem = pn,
algorithm = an,
prob.pars = if (nrow(pd) > 0L) .mapply(list, pd[idx$.i], list()) else list(list()),
algo.pars = if (nrow(ad) > 0L) .mapply(list, ad[idx$.j], list()) else list(list())
)
# create hash of each row of tab
tab$pars.hash = calculateHash(tab)
# merge with already defined experiments to get def.ids
if (nrow(reg$defs) == 0L) {
# this is no optimization, but fixes an strange error on r-devel/windows for merging empty data.tables
tab$def.id = NA_integer_
} else {
tab = merge(reg$defs[, !c("problem", "algorithm", "prob.pars", "algo.pars")], tab, by = "pars.hash", all.x = FALSE, all.y = TRUE, sort = FALSE)
}
# generate def ids for new experiments
w = which(is.na(tab$def.id))
if (length(w) > 0L) {
tab[w, "def.id" := increment(reg$defs$def.id, length(w))]
reg$defs = rbind(reg$defs, tab[w])
}
# create rows in status table for new defs and each repl and filter for defined
tab = CJ(def.id = tab$def.id, repl = seq_len(repls_cur))[!reg$status, on = c("def.id", "repl")]
if (nrow(tab) < n.jobs)
info("Skipping %i duplicated experiments ...", n.jobs - nrow(tab))
if (nrow(tab) > 0L) {
# rbind new status
tab$job.id = increment(reg$status$job.id, nrow(tab))
reg$status = rbind(reg$status, tab, fill = TRUE)
}
all.ids = c(all.ids, tab$job.id)
}
}
if (length(all.ids)) {
setkeyv(reg$defs, "def.id")
setkeyv(reg$status, "job.id")
saveRegistry(reg)
}
invisible(data.table(job.id = all.ids, key = "job.id"))
}
calculateHash = function(tab) {
cols = c("problem", "algorithm", "prob.pars", "algo.pars")
unlist(.mapply(function(...) digest(list(...)), tab[, cols, with = FALSE], list()))
}
================================================
FILE: R/batchMap.R
================================================
#' @title Map Operation for Batch Systems
#'
#' @description
#' A parallel and asynchronous \code{\link[base]{Map}}/\code{\link[base]{mapply}} for batch systems.
#' Note that this function only defines the computational jobs.
#' The actual computation is started with \code{\link{submitJobs}}.
#' Results and partial results can be collected with \code{\link{reduceResultsList}}, \code{\link{reduceResults}} or
#' \code{\link{loadResult}}.
#'
#' For a synchronous \code{\link[base]{Map}}-like execution, see \code{\link{btmapply}}.
#'
#' @param fun [\code{function}]\cr
#' Function to map over arguments provided via \code{...}.
#' Parameters given via \code{args} or \code{...} are passed as-is, in the respective order and possibly named.
#' If the function has the named formal argument \dQuote{.job}, the \code{\link{Job}} is passed to the function
#' on the slave.
#' @param ... [ANY]\cr
#' Arguments to vectorize over (list or vector).
#' Shorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).
#' Mutually exclusive with \code{args}.
#' Note that although it is possible to iterate over large objects (e.g., lists of data frames or matrices), this usually
#' hurts the overall performance and thus is discouraged.
#' @param args [\code{list} | \code{data.frame}]\cr
#' Arguments to vectorize over as (named) list or data frame.
#' Shorter vectors will be recycled (possibly with a warning any length is not a multiple of the longest length).
#' Mutually exclusive with \code{...}.
#' @template more.args
#' @template reg
#' @return [\code{\link[data.table]{data.table}}] with ids of added jobs stored in column \dQuote{job.id}.
#' @export
#' @seealso \code{\link{batchReduce}}
#' @examples
#' \dontshow{ batchtools:::example_push_temp(3) }
#' # example using "..." and more.args
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' f = function(x, y) x^2 + y
#' ids = batchMap(f, x = 1:10, more.args = list(y = 100), reg = tmp)
#' getJobPars(reg = tmp)
#' testJob(6, reg = tmp) # 100 + 6^2 = 136
#'
#' # vector recycling
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' f = function(...) list(...)
#' ids = batchMap(f, x = 1:3, y = 1:6, reg = tmp)
#' getJobPars(reg = tmp)
#'
#' # example for an expand.grid()-like operation on parameters
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' ids = batchMap(paste, args = data.table::CJ(x = letters[1:3], y = 1:3), reg = tmp)
#' getJobPars(reg = tmp)
#' testJob(6, reg = tmp)
batchMap = function(fun, ..., args = list(), more.args = list(), reg = getDefaultRegistry()) {
list2dt = function(x) { # converts a list to a data.table, but avoids creating column names
nn = names(x)
if (is.null(nn))
names(x) = rep.int("", length(x))
as.data.table(x)
}
assertRegistry(reg, class = "Registry", writeable = TRUE)
if (nrow(reg$defs) > 0L)
stop("Registry must be empty")
assertFunction(fun)
assert(checkList(args), checkDataFrame(args))
assertList(more.args)
if (length(args) > 0L) {
if (...length() > 0L)
stop("You may only provide arguments via '...' *or* 'args'")
ddd = list2dt(args)
} else {
ddd = list2dt(list(...))
}
if (".job" %chin% names(ddd))
stop("Name '.job' not allowed as parameter name (reserved keyword)")
if (any(dim(ddd) == 0L))
return(noIds())
info("Adding %i jobs ...", nrow(ddd))
writeRDS(fun, file = fs::path(reg$file.dir, "user.function.rds"), compress = reg$compress)
if (length(more.args) > 0L)
writeRDS(more.args, file = fs::path(reg$file.dir, "more.args.rds"), compress = reg$compress)
ids = seq_row(ddd)
reg$defs = data.table(
def.id = ids,
job.pars = .mapply(list, dots = ddd, MoreArgs = list()),
key = "def.id")
reg$status = data.table(
job.id = ids,
def.id = ids,
submitted = NA_real_,
started = NA_real_,
done = NA_real_,
error = NA_character_,
mem.used = NA_real_,
resource.id = NA_integer_,
batch.id = NA_character_,
log.file = NA_character_,
job.hash = NA_character_,
job.name = NA_character_,
key = "job.id")
saveRegistry(reg)
invisible(allIds(reg))
}
================================================
FILE: R/batchMapResults.R
================================================
#' @title Map Over Results to Create New Jobs
#'
#' @description
#' This function allows you to create new computational jobs (just like \code{\link{batchMap}} based on the results of
#' a \code{\link{Registry}}.
#'
#' @note
#' The URI to the result files in registry \code{source} is hard coded as parameter in the \code{target} registry.
#' This means that \code{target} is currently not portable between systems for computation.
#'
#' @templateVar ids.default findDone
#' @param fun [\code{function}]\cr
#' Function which takes the result as first (unnamed) argument.
#' @template ids
#' @param ... [ANY]\cr
#' Arguments to vectorize over (list or vector). Passed to \code{\link{batchMap}}.
#' @template more.args
#' @param target [\code{\link{Registry}}]\cr
#' Empty Registry where new jobs are created for.
#' @param source [\code{\link{Registry}}]\cr
#' Registry. If not explicitly passed, uses the default registry (see \code{\link{setDefaultRegistry}}).
#' @return [\code{\link[data.table]{data.table}}] with ids of jobs added to \code{target}.
#' @export
#' @family Results
#' @examples
#' \dontshow{ batchtools:::example_push_temp(2) }
#' # Source registry: calculate square of some numbers
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMap(function(x) list(square = x^2), x = 1:10, reg = tmp)
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#'
#' # Target registry: calculate the square root on results of first registry
#' target = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMapResults(fun = function(x, y) list(sqrt = sqrt(x$square)), ids = 4:8,
#' target = target, source = tmp)
#' submitJobs(reg = target)
#' waitForJobs(reg = target)
#'
#' # Map old to new ids. First, get a table with results and parameters
#' results = unwrap(rjoin(getJobPars(reg = target), reduceResultsDataTable(reg = target)))
#' print(results)
#'
#' # Parameter '.id' points to job.id in 'source'. Use a inner join to combine:
#' ijoin(results, unwrap(reduceResultsDataTable(reg = tmp)), by = c(".id" = "job.id"))
batchMapResults = function(fun, ids = NULL, ..., more.args = list(), target, source = getDefaultRegistry()) {
assertRegistry(source, sync = TRUE)
assertRegistry(target, writeable = TRUE, sync = TRUE)
assertFunction(fun)
ids = convertIds(source, ids, default = .findDone(reg = source))
assertList(more.args, names = "strict")
if (nrow(target$status) > 0L)
stop("Target registry 'target' must be empty")
fns = getResultFiles(source, ids)
names(fns) = ids$job.id
more.args = c(list(.fn = fns, .fun = fun), more.args)
args = c(list(.id = ids$job.id), list(...))
batchMap(batchMapResultsWrapper, args = args, more.args = more.args, reg = target)
}
batchMapResultsWrapper = function(.fun, .fn, .id, ...) {
.fun(readRDS(.fn[[as.character(.id)]]), ...)
}
================================================
FILE: R/batchReduce.R
================================================
#' @title Reduce Operation for Batch Systems
#'
#' @description
#' A parallel and asynchronous \code{\link[base]{Reduce}} for batch systems.
#' Note that this function only defines the computational jobs.
#' Each job reduces a certain number of elements on one slave.
#' The actual computation is started with \code{\link{submitJobs}}.
#' Results and partial results can be collected with \code{\link{reduceResultsList}}, \code{\link{reduceResults}} or
#' \code{\link{loadResult}}.
#'
#' @param fun [\code{function(aggr, x, ...)}]\cr
#' Function to reduce \code{xs} with.
#' @param xs [\code{vector}]\cr
#' Vector to reduce.
#' @param init [ANY]\cr
#' Initial object for reducing. See \code{\link[base]{Reduce}}.
#' @param chunks [\code{integer(length(xs))}]\cr
#' Group for each element of \code{xs}. Can be generated with \code{\link{chunk}}.
#' @param more.args [\code{list}]\cr
#' A list of additional arguments passed to \code{fun}.
#' @template reg
#' @return [\code{\link[data.table]{data.table}}] with ids of added jobs stored in column \dQuote{job.id}.
#' @export
#' @seealso \code{\link{batchMap}}
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' # define function to reduce on slave, we want to sum a vector
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' xs = 1:100
#' f = function(aggr, x) aggr + x
#'
#' # sum 20 numbers on each slave process, i.e. 5 jobs
#' chunks = chunk(xs, chunk.size = 5)
#' batchReduce(fun = f, 1:100, init = 0, chunks = chunks, reg = tmp)
#' submitJobs(reg = tmp)
#' waitForJobs(reg = tmp)
#'
#' # now reduce one final time on master
#' reduceResults(fun = function(aggr, job, res) f(aggr, res), reg = tmp)
batchReduce = function(fun, xs, init = NULL, chunks = seq_along(xs), more.args = list(), reg = getDefaultRegistry()) {
assertRegistry(reg, class = "Registry", writeable = TRUE)
if (nrow(reg$defs) > 0L)
stop("Registry must be empty")
assertFunction(fun, c("aggr", "x"))
assertAtomicVector(xs)
assertIntegerish(chunks, len = length(xs), any.missing = FALSE, lower = 0L)
assertList(more.args, names = "strict")
more.args = c(more.args, list(.fun = fun, .init = init))
batchMap(batchReduceWrapper, unname(split(xs, chunks)), more.args = more.args, reg = reg)
}
batchReduceWrapper = function(xs.block, .fun, .init, ...) {
fun = function(aggr, x) .fun(aggr, x, ...)
Reduce(fun, xs.block, init = .init)
}
================================================
FILE: R/btlapply.R
================================================
#' @title Synchronous Apply Functions
#'
#' @description
#' This is a set of functions acting as counterparts to the sequential popular apply functions in base R:
#' \code{btlapply} for \code{\link[base]{lapply}} and \code{btmapply} for \code{\link[base]{mapply}}.
#'
#' Internally, jobs are created using \code{\link{batchMap}} on the provided registry.
#' If no registry is provided, a temporary registry (see argument \code{file.dir} of \code{\link{makeRegistry}}) and \code{\link{batchMap}}
#' will be used.
#' After all jobs are terminated (see \code{\link{waitForJobs}}), the results are collected and returned as a list.
#'
#' Note that these functions are only suitable for short and fail-safe operations
#' on batch system. If some jobs fail, you have to retrieve partial results from the
#' registry directory yourself.
#'
#' @param X [\code{\link[base]{vector}}]\cr
#' Vector to apply over.
#' @param fun [\code{function}]\cr
#' Function to apply.
#' @param more.args [\code{list}]\cr
#' Additional arguments passed to \code{fun}.
#' @param ... [\code{ANY}]\cr
#' Additional arguments passed to \code{fun} (\code{btlapply}) or vectors to map over (\code{btmapply}).
#' @inheritParams submitJobs
#' @param n.chunks [\code{integer(1)}]\cr
#' Passed to \code{\link{chunk}} before \code{\link{submitJobs}}.
#' @param chunk.size [\code{integer(1)}]\cr
#' Passed to \code{\link{chunk}} before \code{\link{submitJobs}}.
#' @template reg
#' @return [\code{list}] List with the results of the function call.
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' btlapply(1:3, function(x) x^2)
#' btmapply(function(x, y, z) x + y + z, x = 1:3, y = 1:3, more.args = list(z = 1), simplify = TRUE)
btlapply = function(X, fun, ..., resources = list(), n.chunks = NULL, chunk.size = NULL, reg = makeRegistry(file.dir = NA)) {
assertVector(X)
assertFunction(fun)
assertRegistry(reg, class = "Registry", writeable = TRUE)
ids = batchMap(fun, X, more.args = list(...), reg = reg)
if (!is.null(n.chunks) || !is.null(chunk.size))
ids$chunk = chunk(ids$job.id, n.chunks = n.chunks, chunk.size = chunk.size)
submitJobs(ids = ids, resources = resources, reg = reg)
waitForJobs(ids = ids, reg = reg)
reduceResultsList(ids = ids, reg = reg)
}
#' @export
#' @param simplify [\code{logical(1)}]\cr
#' Simplify the results using \code{\link[base]{simplify2array}}?
#' @param use.names [\code{logical(1)}]\cr
#' Use names of the input to name the output?
#' @rdname btlapply
btmapply = function(fun, ..., more.args = list(), simplify = FALSE, use.names = TRUE, resources = list(), n.chunks = NULL, chunk.size = NULL, reg = makeRegistry(file.dir = NA)) {
assertFunction(fun)
assertFlag(simplify)
assertFlag(use.names)
assertRegistry(reg, class = "Registry", writeable = TRUE)
ids = batchMap(fun, ..., more.args = more.args, reg = reg)
if (!is.null(n.chunks) || !is.null(chunk.size))
ids$chunk = chunk(ids$job.id, n.chunks = n.chunks, chunk.size = chunk.size)
submitJobs(ids = ids, resources = resources, reg = reg)
waitForJobs(ids = ids, reg = reg)
res = reduceResultsList(ids = ids, reg = reg)
if (use.names) {
x = head(list(...), 1L)
if (length(x) > 0L) {
x = x[[1L]]
if (is.null(names(x))) {
if(is.character(x))
names(res) = x
} else {
names(res) = names(x)
}
}
}
if (simplify) simplify2array(res) else res
}
================================================
FILE: R/chunkIds.R
================================================
#' @title Chunk Jobs for Sequential Execution
#'
#' @description
#' Jobs can be partitioned into \dQuote{chunks} to be executed sequentially on the computational nodes.
#' Chunks are defined by providing a data frame with columns \dQuote{job.id} and \dQuote{chunk} (integer)
#' to \code{\link{submitJobs}}.
#' All jobs with the same chunk number will be grouped together on one node to form a single
#' computational job.
#'
#' The function \code{chunk} simply splits \code{x} into either a fixed number of groups, or
#' into a variable number of groups with a fixed number of maximum elements.
#'
#' The function \code{lpt} also groups \code{x} into a fixed number of chunks,
#' but uses the actual values of \code{x} in a greedy \dQuote{Longest Processing Time} algorithm.
#' As a result, the maximum sum of elements in minimized.
#'
#' \code{binpack} splits \code{x} into a variable number of groups whose sum of elements do
#' not exceed the upper limit provided by \code{chunk.size}.
#'
#' See examples of \code{\link{estimateRuntimes}} for an application of \code{binpack} and \code{lpt}.
#'
#' @param x [\code{numeric}]\cr
#' For \code{chunk} an atomic vector (usually the \code{job.id}).
#' For \code{binpack} and \code{lpt}, the weights to group.
#' @param chunk.size [\code{integer(1)}]\cr
#' Requested chunk size for each single chunk.
#' For \code{chunk} this is the number of elements in \code{x}, for \code{binpack} the size
#' is determined by the sum of values in \code{x}.
#' Mutually exclusive with \code{n.chunks}.
#' @param n.chunks [\code{integer(1)}]\cr
#' Requested number of chunks.
#' The function \code{chunk} distributes the number of elements in \code{x} evenly while
#' \code{lpt} tries to even out the sum of elements in each chunk.
#' If more chunks than necessary are requested, empty chunks are ignored.
#' Mutually exclusive with \code{chunks.size}.
#' @param shuffle [\code{logical(1)}]\cr
#' Shuffles the groups. Default is \code{TRUE}.
#' @return [\code{integer}] giving the chunk number for each element of \code{x}.
#' @seealso \code{\link{estimateRuntimes}}
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(2) }
#' ch = chunk(1:10, n.chunks = 2)
#' table(ch)
#'
#' ch = chunk(rep(1, 10), chunk.size = 2)
#' table(ch)
#'
#' set.seed(1)
#' x = runif(10)
#' ch = lpt(x, n.chunks = 2)
#' sapply(split(x, ch), sum)
#'
#' set.seed(1)
#' x = runif(10)
#' ch = binpack(x, 1)
#' sapply(split(x, ch), sum)
#'
#' # Job chunking
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' ids = batchMap(identity, 1:25, reg = tmp)
#'
#' ### Group into chunks with 10 jobs each
#' library(data.table)
#' ids[, chunk := chunk(job.id, chunk.size = 10)]
#' print(ids[, .N, by = chunk])
#'
#' ### Group into 4 chunks
#' ids[, chunk := chunk(job.id, n.chunks = 4)]
#' print(ids[, .N, by = chunk])
#'
#' ### Submit to batch system
#' submitJobs(ids = ids, reg = tmp)
#'
#' # Grouped chunking
#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE)
#' prob = addProblem(reg = tmp, "prob1", data = iris, fun = function(job, data) nrow(data))
#' prob = addProblem(reg = tmp, "prob2", data = Titanic, fun = function(job, data) nrow(data))
#' algo = addAlgorithm(reg = tmp, "algo", fun = function(job, data, instance, i, ...) problem)
#' prob.designs = list(prob1 = data.table(), prob2 = data.table(x = 1:2))
#' algo.designs = list(algo = data.table(i = 1:3))
#' addExperiments(prob.designs, algo.designs, repls = 3, reg = tmp)
#'
#' ### Group into chunks of 5 jobs, but do not put multiple problems into the same chunk
#' # -> only one problem has to be loaded per chunk, and only once because it is cached
#' ids = getJobTable(reg = tmp)[, .(job.id, problem, algorithm)]
#' ids[, chunk := chunk(job.id, chunk.size = 5), by = "problem"]
#' ids[, chunk := .GRP, by = c("problem", "chunk")]
#' dcast(ids, chunk ~ problem)
chunk = function(x, n.chunks = NULL, chunk.size = NULL, shuffle = TRUE) {
assertAtomicVector(x)
if (!xor(is.null(n.chunks), is.null(chunk.size)))
stop("You must provide either 'n.chunks' (x)or 'chunk.size'")
assertCount(n.chunks, positive = TRUE, null.ok = TRUE)
assertCount(chunk.size, positive = TRUE, null.ok = TRUE)
assertFlag(shuffle)
n = length(x)
if (n == 0L)
return(integer(0L))
if (is.null(n.chunks))
n.chunks = (n %/% chunk.size + (n %% chunk.size > 0L))
chunks = as.integer((seq.int(0L, n - 1L) %% min(n.chunks, n))) + 1L
if (shuffle)
chunks = sample(chunks)
else
chunks = sort(chunks)
return(chunks)
}
#' @rdname chunk
#' @useDynLib batchtools c_lpt
#' @export
lpt = function(x, n.chunks = 1L) {
assertNumeric(x, lower = 0, any.missing = FALSE, finite = TRUE)
assertCount(n.chunks, positive = TRUE)
x = as.double(x)
ord = order(x, decreasing = TRUE)
n.chunks = as.integer(n.chunks)
.Call(c_lpt, x, ord, n.chunks)
}
#' @rdname chunk
#' @useDynLib batchtools c_binpack
#' @export
binpack = function(x, chunk.size = max(x)) {
assertNumeric(x, lower = 0, any.missing = FALSE, finite = TRUE)
assertNumber(chunk.size, lower = 0)
if (length(x) == 0L)
return(integer(0L))
x = as.double(x)
ord = order(x, decreasing = TRUE)
chunk.size = as.double(chunk.size)
.Call(c_binpack, x, ord, chunk.size)
}
================================================
FILE: R/clearRegistry.R
================================================
#' @title Remove All Jobs
#' @description
#' Removes all jobs from a registry and calls \code{\link{sweepRegistry}}.
#'
#' @template reg
#' @family Registry
#' @export
clearRegistry = function(reg = getDefaultRegistry()) {
assertRegistry(reg, writeable = TRUE, sync = TRUE, running.ok = FALSE)
info("Removing %i jobs ...", nrow(reg$status))
reg$status = reg$status[FALSE]
reg$defs = reg$defs[FALSE]
reg$resources = reg$resources[FALSE]
user.fun = fs::path(reg$file.dir, "user.function.rds")
if (fs::file_exists(user.fun)) {
info("Removing user function ...")
file_remove(user.fun)
}
sweepRegistry(reg = reg)
}
================================================
FILE: R/clusterFunctions.R
================================================
#' @title ClusterFunctions Constructor
#'
#' @description
#' This is the constructor used to create \emph{custom} cluster functions.
#' Note that some standard implementations for TORQUE, Slurm, LSF, SGE, etc. ship
#' with the package.
#'
#' @param name [\code{character(1)}]\cr
#' Name of cluster functions.
#' @param submitJob [\code{function(reg, jc, ...)}]\cr
#' Function to submit new jobs. Must return a \code{\link{SubmitJobResult}} object.
#' The arguments are \code{reg} (\code{\link{Registry}}) and \code{jobs} (\code{\link{JobCollection}}).
#' @param killJob [\code{function(reg, batch.id)}]\cr
#' Function to kill a job on the batch system. Make sure that you definitely kill the job! Return
#' value is currently ignored. Must have the arguments \code{reg} (\code{\link{Registry}}) and
#' \code{batch.id} (\code{character(1)} as returned by \code{submitJob}).
#' Note that there is a helper function \code{\link{cfKillJob}} to repeatedly try to kill jobs.
#' Set \code{killJob} to \code{NULL} if killing jobs cannot be supported.
#' @param listJobsQueued [\code{function(reg)}]\cr
#' List all queued jobs on the batch system for the current user.
#' Must return an character vector of batch ids, same format as they
#' are returned by \code{submitJob}.
#' Set \code{listJobsQueued} to \code{NULL} if listing of queued jobs is not supported.
#' @param listJobsRunning [\code{function(reg)}]\cr
#' List all running jobs on the batch system for the current user.
#' Must return an character vector of batch ids, same format as they
#' are returned by \code{submitJob}. It does not matter if you return a few job ids too many (e.g.
#' all for the current user instead of all for the current registry), but you have to include all
#' relevant ones. Must have the argument are \code{reg} (\code{\link{Registry}}).
#' Set \code{listJobsRunning} to \code{NULL} if listing of running jobs is not supported.
#' @param array.var [\code{character(1)}]\cr
#' Name of the environment variable set by the scheduler to identify IDs of job arrays.
#' Default is \code{NA} for no array support.
#' @param store.job.collection [\code{logical(1)}]\cr
#' Flag to indicate that the cluster function implementation of \code{submitJob} can not directly handle \code{\link{JobCollection}} objects.
#' If set to \code{FALSE}, the \code{\link{JobCollection}} is serialized to the file system before submitting the job.
#' @param store.job.files [\code{logical(1)}]\cr
#' Flag to indicate that job files need to be stored in the file directory.
#' If set to \code{FALSE} (default), the job file is created in a temporary directory, otherwise (or if the debug mode is enabled) in
#' the subdirectory \code{jobs} of the \code{file.dir}.
#' @param scheduler.latency [\code{numeric(1)}]\cr
#' Time to sleep after important interactions with the scheduler to ensure a sane state.
#' Currently only triggered after calling \code{\link{submitJobs}}.
#' @param fs.latency [\code{numeric(1)}]\cr
#' Expected maximum latency of the file system, in seconds.
#' Set to a positive number for network file systems like NFS which enables more robust (but also more expensive) mechanisms to
#' access files and directories.
#' Usually safe to set to \code{0} to disable the heuristic, e.g. if you are working on a local file system.
#' @param hooks [\code{list}]\cr
#' Named list of functions which will we called on certain events like \dQuote{pre.submit} or \dQuote{post.sync}.
#' See \link{Hooks}.
#' @export
#' @aliases ClusterFunctions
#' @family ClusterFunctions
#' @family ClusterFunctionsHelper
makeClusterFunctions = function(name, submitJob, killJob = NULL, listJobsQueued = NULL, listJobsRunning = NULL,
array.var = NA_character_, store.job.collection = FALSE, store.job.files = FALSE, scheduler.latency = 0,
fs.latency = 0, hooks = list()) {
assertList(hooks, types = "function", names = "unique")
assertSubset(names(hooks), unlist(batchtools$hooks, use.names = FALSE))
setClasses(list(
name = assertString(name, min.chars = 1L),
submitJob = assertFunction(submitJob, c("reg", "jc"), null.ok = TRUE),
killJob = assertFunction(killJob, c("reg", "batch.id"), null.ok = TRUE),
listJobsQueued = assertFunction(listJobsQueued, "reg", null.ok = TRUE),
listJobsRunning = assertFunction(listJobsRunning, "reg", null.ok = TRUE),
array.var = assertString(array.var, na.ok = TRUE),
store.job.collection = assertFlag(store.job.collection),
store.job.files = assertFlag(store.job.files),
scheduler.latency = assertNumber(scheduler.latency, lower = 0),
fs.latency = assertNumber(fs.latency, lower = 0),
hooks = hooks),
"ClusterFunctions")
}
#' @export
print.ClusterFunctions = function(x, ...) {
catf("ClusterFunctions for mode: %s", x$name)
catf(" List queued Jobs : %s", !is.null(x$listJobsQueued))
catf(" List running Jobs: %s", !is.null(x$listJobsRunning))
catf(" Kill Jobs : %s", !is.null(x$killJob))
catf(" Hooks : %s", if (length(x$hooks)) stri_flatten(names(x$hooks), ",") else "-")
}
#' @title Create a SubmitJobResult
#'
#' @description
#' This function is only intended for use in your own cluster functions implementation.
#'
#' Use this function in your implementation of \code{\link{makeClusterFunctions}} to create a return
#' value for the \code{submitJob} function.
#'
#' @param status [\code{integer(1)}]\cr
#' Launch status of job. 0 means success, codes between 1 and 100 are temporary errors and any
#' error greater than 100 is a permanent failure.
#' @param batch.id [\code{character()}]\cr
#' Unique id of this job on batch system, as given by the batch system.
#' Must be globally unique so that the job can be terminated using just this information.
#' For array jobs, this may be a vector of length equal to the number of jobs in the array.
#' @param log.file [\code{character()}]\cr
#' Log file. If \code{NA}, defaults to \code{[job.hash].log}.
#' Some cluster functions set this for array jobs.
#' @param msg [\code{character(1)}]\cr
#' Optional error message in case \code{status} is not equal to 0. Default is \dQuote{OK},
#' \dQuote{TEMPERROR}, \dQuote{ERROR}, depending on \code{status}.
#' @return [\code{\link{SubmitJobResult}}]. A list, containing
#' \code{status}, \code{batch.id} and \code{msg}.
#' @family ClusterFunctionsHelper
#' @aliases SubmitJobResult
#' @export
makeSubmitJobResult = function(status, batch.id, log.file = NA_character_, msg = NA_character_) {
status = asInt(status)
if (is.na(msg)) {
msg = if (status == 0L)
"OK"
else if (status <= 100L)
"TEMPERROR"
else
"ERROR"
}
"!DEBUG [makeSubmitJobResult]: Result for batch.id '`paste0(batch.id, sep = ',')`': `status` (`msg`)"
setClasses(list(status = status, batch.id = batch.id, log.file = log.file, msg = msg), "SubmitJobResult")
}
#' @export
print.SubmitJobResult = function(x, ...) {
cat("Job submission result\n")
catf(" ID : %s", stri_flatten(x$batch.id, ","))
catf(" Status: %i", x$status)
catf(" Msg : %s", x$msg)
}
#' @title Cluster Functions Helper to Parse a Brew Template
#'
#' @description
#' This function is only intended for use in your own cluster functions implementation.
#'
#' This function is only intended for use in your own cluster functions implementation.
#' Simply reads your template file and returns it as a character vector.
#'
#' @param template [\code{character(1)}]\cr
#' Path to template file which is then passed to \code{\link[brew]{brew}}.
#' @param comment.string [\code{character(1)}]\cr
#' Ignore lines starting with this string.
#' @return [\code{character}].
#' @family ClusterFunctionsHelper
#' @export
cfReadBrewTemplate = function(template, comment.string = NA_character_) {
"!DEBUG [cfReadBrewTemplate]: Parsing template file '`template`'"
lines = stri_trim_both(readLines(template))
lines = lines[!stri_isempty(lines)]
if (!is.na(comment.string))
lines = lines[!stri_startswith_fixed(lines, comment.string)]
if (length(lines) == 0L)
stopf("Error reading template '%s' or empty template", template)
return(stri_flatten(lines, "\n"))
}
#' @title Cluster Functions Helper to Write Job Description Files
#'
#' @description
#' This function is only intended for use in your own cluster functions implementation.
#'
#' Calls brew silently on your template, any error will lead to an exception.
#' The file is stored at the same place as the corresponding job file in the \dQuote{jobs}-subdir
#' of your files directory.
#'
#' @template reg
#' @param text [\code{character(1)}]\cr
#' String ready to be brewed. See \code{\link{cfReadBrewTemplate}} to read a template from the file system.
#' @param jc [\code{\link{JobCollection})}]\cr
#' Will be used as environment to brew the template file in. See \code{\link{JobCollection}} for a list of all
#' available variables.
#' @return [\code{character(1)}]. File path to brewed template file.
#' @family ClusterFunctionsHelper
#' @export
cfBrewTemplate = function(reg, text, jc) {
assertString(text)
outfile = fs::path(dir(reg, "jobs"), sprintf("%s.job", jc$job.hash))
parent.env(jc) = asNamespace("batchtools")
on.exit(parent.env(jc) <- emptyenv())
"!DEBUG [cfBrewTemplate]: Brewing template to file '`outfile`'"
z = try(brew(text = text, output = outfile, envir = jc), silent = TRUE)
if (is.error(z))
stopf("Error brewing template: %s", as.character(z))
waitForFile(outfile, reg$cluster.functions$fs.latency)
return(outfile)
}
#' @title Cluster Functions Helper to Handle Unknown Errors
#'
#' @description
#' This function is only intended for use in your own cluster functions implementation.
#'
#' Simply constructs a \code{\link{SubmitJobResult}} object with status code 101, NA as batch id and
#' an informative error message containing the output of the OS command in \code{output}.
#'
#' @param cmd [\code{character(1)}]\cr
#' OS command used to submit the job, e.g. qsub.
#' @param exit.code [\code{integer(1)}]\cr
#' Exit code of the OS command, should not be 0.
#' @param output [\code{character}]\cr
#' Output of the OS command, hopefully an informative error message.
#' If these are multiple lines in a vector, they are automatically joined.
#' @return [\code{\link{SubmitJobResult}}].
#' @family ClusterFunctionsHelper
#' @export
cfHandleUnknownSubmitError = function(cmd, exit.code, output) {
assertString(cmd, min.chars = 1L)
exit.code = asInt(exit.code)
assertCharacter(output, any.missing = FALSE)
msg = sprintf("Command '%s' produced exit code %i. Output: '%s'", cmd, exit.code, stri_flatten(output, "\n"))
makeSubmitJobResult(status = 101L, batch.id = NA_character_, msg = msg)
}
#' @title Cluster Functions Helper to Kill Batch Jobs
#'
#' @description
#' This function is only intended for use in your own cluster functions implementation.
#'
#' Calls the OS command to kill a job via \code{\link[base]{system}} like this: \dQuote{cmd batch.job.id}. If the
#' command returns an exit code > 0, the command is repeated after a 1 second sleep
#' \code{max.tries-1} times. If the command failed in all tries, an error is generated.
#'
#' @template reg
#' @param cmd [\code{character(1)}]\cr
#' OS command, e.g. \dQuote{qdel}.
#' @param args [\code{character}]\cr
#' Arguments to \code{cmd}, including the batch id.
#' @param max.tries [\code{integer(1)}]\cr
#' Number of total times to try execute the OS command in cases of failures.
#' Default is \code{3}.
#' @inheritParams runOSCommand
#' @return \code{TRUE} on success. An exception is raised otherwise.
#' @family ClusterFunctionsHelper
#' @export
cfKillJob = function(reg, cmd, args = character(0L), max.tries = 3L, nodename = "localhost") {
assertString(cmd, min.chars = 1L)
assertCharacter(args, any.missing = FALSE)
assertString(nodename)
max.tries = asCount(max.tries)
for (i in seq_len(max.tries)) {
res = runOSCommand(cmd, args, nodename = nodename)
if (res$exit.code == 0L)
return(TRUE)
Sys.sleep(1)
}
stopf("Really tried to kill job, but failed %i times with '%s'.\nMessage: %s",
max.tries, stri_flatten(c(cmd, args), " "), stri_flatten(res$output, "\n"))
}
getBatchIds = function(reg, status = "all") {
cf = reg$cluster.functions
tab = data.table(batch.id = character(0L), status = character(0L))
batch.id = NULL
if (status %chin% c("all", "running") && !is.null(cf$listJobsRunning)) {
"!DEBUG [getBatchIds]: Getting running Jobs"
x = unique(cf$listJobsRunning(reg))
if (length(x) > 0L)
tab = rbind(tab, data.table(batch.id = x, status = "running"))
}
if (status %chin% c("all", "queued") && !is.null(cf$listJobsQueued)) {
"!DEBUG [getBatchIds]: Getting queued Jobs"
x = chsetdiff(cf$listJobsQueued(reg), tab$batch.id)
if (length(x) > 0L)
tab = rbind(tab, data.table(batch.id = unique(x), status = "queued"))
}
submitted = done = batch.id = NULL
batch.ids = reg$status[!is.na(submitted) & is.na(done) & !is.na(batch.id), unique(batch.id)]
tab[batch.id %in% batch.ids]
}
#' @title Find a batchtools Template File
#'
#' @description
#' This functions returns the path to a template file on the file system.
#' @template template
#' @return [\code{character}] Path to the file or \code{NA} if no template template file was found.
#' @keywords internal
#' @export
findTemplateFile = function(template) {
assertString(template, min.chars = 1L)
if (stri_endswith_fixed(template, ".tmpl")) {
assertFileExists(template, access = "r")
return(fs::path_abs(template))
}
x = Sys.getenv("R_BATCHTOOLS_SEARCH_PATH")
if (nzchar(x)) {
x = fs::path(x, sprintf("batchtools.%s.tmpl", template))
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
}
x = sprintf("batchtools.%s.tmpl", template)
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
x = fs::path(user_config_dir("batchtools", expand = FALSE), sprintf("%s.tmpl", template))
if (fs::file_access(x, "read"))
return(x)
x = fs::path("~", sprintf(".batchtools.%s.tmpl", template))
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
x = fs::path(site_config_dir("batchtools"), sprintf("%s.tmpl", template))
if (fs::file_access(x, "read"))
return(x)
x = system.file("templates", sprintf("%s.tmpl", template), package = "batchtools")
if (fs::file_access(x, "read"))
return(x)
return(NA_character_)
}
================================================
FILE: R/clusterFunctionsDocker.R
================================================
#' @title ClusterFunctions for Docker
#'
#' @description
#' Cluster functions for Docker/Docker Swarm (\url{https://docs.docker.com/engine/swarm/}).
#'
#' The \code{submitJob} function executes
#' \code{docker [docker.args] run --detach=true [image.args] [resources] [image] [cmd]}.
#' Arguments \code{docker.args}, \code{image.args} and \code{image} can be set on construction.
#' The \code{resources} part takes the named resources \code{ncpus} and \code{memory}
#' from \code{\link{submitJobs}} and maps them to the arguments \code{--cpu-shares} and \code{--memory}
#' (in Megabytes). The resource \code{threads} is mapped to the environment variables \dQuote{OMP_NUM_THREADS}
#' and \dQuote{OPENBLAS_NUM_THREADS}.
#' To reliably identify jobs in the swarm, jobs are labeled with \dQuote{batchtools=[job.hash]} and named
#' using the current login name (label \dQuote{user}) and the job hash (label \dQuote{batchtools}).
#'
#' \code{listJobsRunning} uses \code{docker [docker.args] ps --format=\{\{.ID\}\}} to filter for running jobs.
#'
#' \code{killJobs} uses \code{docker [docker.args] kill [batch.id]} to filter for running jobs.
#'
#' These cluster functions use a \link{Hook} to remove finished jobs before a new submit and every time the \link{Registry}
#' is synchronized (using \code{\link{syncRegistry}}).
#' This is currently required because docker does not remove terminated containers automatically.
#' Use \code{docker ps -a --filter 'label=batchtools' --filter 'status=exited'} to identify and remove terminated
#' containers manually (or usa a cron job).
#'
#' @param image [\code{character(1)}]\cr
#' Name of the docker image to run.
#' @param docker.args [\code{character}]\cr
#' Additional arguments passed to \dQuote{docker} *before* the command (\dQuote{run}, \dQuote{ps} or \dQuote{kill}) to execute (e.g., the docker host).
#' @param image.args [\code{character}]\cr
#' Additional arguments passed to \dQuote{docker run} (e.g., to define mounts or environment variables).
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsDocker = function(image, docker.args = character(0L), image.args = character(0L), scheduler.latency = 1, fs.latency = 65) { # nocov start
assertString(image)
assertCharacter(docker.args, any.missing = FALSE)
assertCharacter(image.args, any.missing = FALSE)
user = Sys.info()["user"]
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
assertIntegerish(jc$resources$ncpus, lower = 1L, any.missing = FALSE, .var.name = "resources$ncpus")
assertIntegerish(jc$resources$memory, lower = 1L, any.missing = FALSE, .var.name = "resources$memory")
timeout = if (is.null(jc$resources$walltime)) character(0L) else sprintf("timeout %i", asInt(jc$resources$walltime, lower = 0L))
cmd = c("docker", docker.args, "run", "--detach=true", image.args,
sprintf("-e DEBUGME='%s'", Sys.getenv("DEBUGME")),
sprintf("-e OMP_NUM_THREADS=%i", jc$resources$omp.threads %??% jc$resources$threads),
sprintf("-e OPENBLAS_NUM_THREADS=%i", jc$resources$blas.threads %??% jc$resources$threads),
sprintf("-e MKL_NUM_THREADS=%i", jc$resources$blas.threads %??% jc$resources$threads),
sprintf("-c %i", jc$resources$ncpus),
sprintf("-m %im", jc$resources$memory),
sprintf("--memory-swap %im", jc$resources$memory),
sprintf("--label batchtools=%s", jc$job.hash),
sprintf("--label user=%s", user),
sprintf("--name=%s_bt_%s", user, jc$job.hash),
image, timeout, "Rscript", stri_join("-e", shQuote(sprintf("batchtools::doJobCollection('%s', '%s')", jc$uri, jc$log.file)), sep = " "))
res = runOSCommand(cmd[1L], cmd[-1L])
if (res$exit.code > 0L) {
housekeeping(reg)
no.res.msg = "no resources available"
if (res$exit.code == 1L && any(stri_detect_fixed(res$output, no.res.msg)))
return(makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = no.res.msg))
return(cfHandleUnknownSubmitError(stri_flatten(cmd, " "), res$exit.code, res$output))
} else {
if (length(res$output != 1L)) {
matches = which(stri_detect_regex(res$output, "^[[:alnum:]]{64}$"))
if (length(matches) != 1L)
stopf("Command '%s' did not return a long UUID identitfier", stri_flatten(cmd, " "))
res$output = res$output[matches]
}
return(makeSubmitJobResult(status = 0L, batch.id = stri_sub(res$output, 1L, 12L)))
}
}
listJobs = function(reg, filter = character(0L)) {
assertRegistry(reg, writeable = FALSE)
# use a workaround for DockerSwarm: docker ps does not list all jobs correctly, only
# docker inspect reports the status correctly
args = c(docker.args, "ps", "--format={{.ID}}", "--filter 'label=batchtools'", filter)
res = runOSCommand("docker", args)
if (res$exit.code > 0L)
OSError("Listing of jobs failed", res)
if (length(res$output) == 0L || !nzchar(res$output))
return(character(0L))
res$output
}
housekeeping = function(reg, ...) {
batch.ids = chintersect(listJobs(reg, "--filter 'status=exited'"), reg$status$batch.id)
if (length(batch.ids) > 0L)
runOSCommand("docker", c(docker.args, "rm", batch.ids))
invisible(TRUE)
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "docker", c(docker.args, "kill", batch.id))
}
listJobsRunning = function(reg) {
assertRegistry(reg, writeable = FALSE)
listJobs(reg, sprintf("--filter 'user=%s'", user))
}
makeClusterFunctions(name = "Docker", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,
store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency,
hooks = list(post.submit = housekeeping, post.sync = housekeeping))
} # nocov end
================================================
FILE: R/clusterFunctionsHyperQueue.R
================================================
#' @title ClusterFunctions for HyperQueue
#'
#' @description
#' Cluster functions for HyperQueue (\url{https://it4innovations.github.io/hyperqueue/stable/}).
#'
#' Jobs are submitted via the HyperQueue CLI using \code{hq submit} and executed by calling \code{Rscript -e "batchtools::doJobCollection(...)"}.
#' The job name is set to the job hash and logs are handled internally by batchtools.
#' Listing jobs uses \code{hq job list} and cancelling jobs uses \code{hq job cancel}.
#' A running HyperQueue server and workers are required.
#'
#'
#' @inheritParams makeClusterFunctions
#' @return [ClusterFunctions].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsHyperQueue = function(scheduler.latency = 1, fs.latency = 65) {
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
ncpus = if (!is.null(jc$resources$ncpus)) sprintf("--cpus=%i", jc$resources$ncpus)
memory = if (!is.null(jc$resources$memory)) sprintf("--resource mem=%iMiB", jc$resources$memory)
# time-limit is the maximum time the job can run, time-request is the minimum remaining lifetime a worker must have
walltime = if (!is.null(jc$resources$walltime)) sprintf("--time-limit=%is --time-request=%is", jc$resources$walltime, jc$resources$walltime)
args = c(
"submit",
sprintf("--name=%s", jc$job.hash),
# hyperqueue cannot write stdout and stderr to the same file
"--stdout=none",
"--stderr=none",
ncpus,
memory,
walltime,
"--",
"Rscript", "-e",
shQuote(sprintf("batchtools::doJobCollection('%s', '%s')", jc$uri, jc$log.file))
)
res = runOSCommand("hq", args)
if (res$exit.code > 0L) {
return(cfHandleUnknownSubmitError("hq", res$exit.code, res$output))
}
batch_ids = sub(".*job ID: ([0-9]+).*", "\\1", res$output)
makeSubmitJobResult(status = 0L, batch.id = batch_ids)
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
args = c("job", "cancel", batch.id)
res = runOSCommand("hq", args)
if (res$exit.code > 0L) {
OSError("Killing of job failed", res)
}
makeSubmitJobResult(status = 0L, batch.id = batch.id)
}
listJobsQueued = function(reg) {
requireNamespace("jsonlite")
assertRegistry(reg, writeable = FALSE)
args = c("job", "list", "--filter", "waiting", "--output-mode", "json")
res = runOSCommand("hq", args)
if (res$exit.code > 0L) {
OSError("Listing of jobs failed", res)
}
jobs = jsonlite::fromJSON(res$output)
as.character(jobs$id)
}
listJobsRunning = function(reg) {
requireNamespace("jsonlite")
assertRegistry(reg, writeable = FALSE)
args = c("job", "list", "--filter", "running", "--output-mode", "json")
res = runOSCommand("hq", args)
if (res$exit.code > 0L) {
OSError("Listing of jobs failed", res)
}
jobs = jsonlite::fromJSON(res$output)
as.character(jobs$id)
}
makeClusterFunctions(
name = "HyperQueue",
submitJob = submitJob,
killJob = killJob,
listJobsRunning = listJobsRunning,
listJobsQueued = listJobsQueued,
store.job.collection = TRUE,
scheduler.latency = scheduler.latency,
fs.latency = fs.latency)
}
================================================
FILE: R/clusterFunctionsInteractive.R
================================================
#' @title ClusterFunctions for Sequential Execution in the Running R Session
#'
#' @description
#' All jobs are executed sequentially using the current R process in which \code{\link{submitJobs}} is called.
#' Thus, \code{submitJob} blocks the session until the job has finished.
#' The main use of this \code{ClusterFunctions} implementation is to test and debug programs on a local computer.
#'
#' Listing jobs returns an empty vector (as no jobs can be running when you call this)
#' and \code{killJob} is not implemented for the same reasons.
#'
#' @param external [\code{logical(1)}]\cr
#' If set to \code{TRUE}, jobs are started in a fresh R session instead of currently active but still
#' waits for its termination.
#' Default is \code{FALSE}.
#' @param write.logs [\code{logical(1)}]\cr
#' Sink the output to log files. Turning logging off can increase the speed of
#' calculations but makes it very difficult to debug.
#' Default is \code{TRUE}.
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsInteractive = function(external = FALSE, write.logs = TRUE, fs.latency = 0) {
assertFlag(external)
assertFlag(write.logs)
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
if (external) {
runOSCommand(Rscript(), sprintf("-e \"batchtools::doJobCollection('%s', output = '%s')\"", jc$uri, jc$log.file))
} else {
doJobCollection(jc, output = jc$log.file)
}
makeSubmitJobResult(status = 0L, batch.id = "cfInteractive")
}
makeClusterFunctions(name = "Interactive", submitJob = submitJob, store.job.collection = external, fs.latency = fs.latency)
}
================================================
FILE: R/clusterFunctionsLSF.R
================================================
#' @title ClusterFunctions for LSF Systems
#'
#' @description
#' Cluster functions for LSF (\url{https://www.ibm.com/products/hpc-workload-management}).
#'
#' Job files are created based on the brew template \code{template.file}. This
#' file is processed with brew and then submitted to the queue using the
#' \code{bsub} command. Jobs are killed using the \code{bkill} command and the
#' list of running jobs is retrieved using \code{bjobs -u $USER -w}. The user
#' must have the appropriate privileges to submit, delete and list jobs on the
#' cluster (this is usually the case).
#'
#' The template file can access all resources passed to \code{\link{submitJobs}}
#' as well as all variables stored in the \code{\link{JobCollection}}.
#' It is the template file's job to choose a queue for the job and handle the desired resource
#' allocations.
#'
#' @note
#' Array jobs are currently not supported.
#'
#' @template template
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsLSF = function(template = "lsf", scheduler.latency = 1, fs.latency = 65) { # nocov start
template = findTemplateFile(template)
if (testScalarNA(template))
stopf("Argument 'template' (=\"%s\") must point to a readable template file or contain the template itself as string (containing at least one newline)", template)
template = cfReadBrewTemplate(template)
# When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only
# when unfinished jobs are found, and 255 when no jobs are found,
# or a non-existent job ID is entered.
Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = "Y")
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
outfile = cfBrewTemplate(reg, template, jc)
res = runOSCommand("bsub", stdin = outfile)
if (res$exit.code > 0L) {
cfHandleUnknownSubmitError("bsub", res$exit.code, res$output)
} else {
batch.id = stri_extract_first_regex(stri_flatten(res$output, " "), "\\d+")
makeSubmitJobResult(status = 0L, batch.id = batch.id)
}
}
listJobs = function(reg, args) {
assertRegistry(reg, writeable = FALSE)
res = runOSCommand("bjobs", args)
if (res$exit.code > 0L) {
if (res$exit.code == 255L || any(stri_detect_regex(res$output, "No (unfinished|pending|running) job found")))
return(character(0L))
OSError("Listing of jobs failed", res)
}
stri_extract_first_regex(tail(res$output, -1L), "\\d+")
}
listJobsQueued = function(reg) {
listJobs(reg, c("-u $USER", "-w", "-p"))
}
listJobsRunning = function(reg) {
listJobs(reg, c("-u $USER", "-w", "-r"))
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "bkill", batch.id)
}
makeClusterFunctions(name = "LSF", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,
listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/clusterFunctionsMulticore.R
================================================
if (getRversion() < "3.3.2" && .Platform$OS.type != "windows") {
# Provided patch for upstream which is shipped with R >= 3.3.2:
# https://stat.ethz.ch/pipermail/r-devel/2016-August/073035.html
selectChildren = getFromNamespace("selectChildren", "parallel")
readChild = getFromNamespace("readChild", "parallel")
mccollect = function(pids, timeout = 0) {
if (!length(pids)) return (NULL)
if (!is.integer(pids)) stop("invalid 'jobs' argument")
s = selectChildren(pids, timeout)
if (is.logical(s) || !length(s)) return(NULL)
res = lapply(s, function(x) {
r = readChild(x)
if (is.raw(r)) unserialize(r) else NULL
})
names(res) = as.character(pids)[match(s, pids)]
res
}
} else {
mccollect = function(jobs, timeout = 0) {
parallel::mccollect(jobs, wait = FALSE, timeout = timeout)
}
}
Multicore = R6Class("Multicore",
cloneable = FALSE,
public = list(
jobs = NULL,
ncpus = NA_integer_,
initialize = function(ncpus) {
self$jobs = data.table(pid = integer(0L), count = integer(0L))
self$ncpus = ncpus
reg.finalizer(self, function(e) mccollect(self$jobs$pid, timeout = 1), onexit = FALSE)
},
spawn = function(jc) {
force(jc)
repeat {
self$collect(0)
if (nrow(self$jobs) < self$ncpus)
break
Sys.sleep(1)
}
pid = parallel::mcparallel(doJobCollection(jc, output = jc$log.file), mc.set.seed = FALSE)$pid
self$jobs = rbind(self$jobs, data.table(pid = pid, count = 0L))
invisible(as.character(pid))
},
list = function() {
self$collect(0)
as.character(self$jobs$pid)
},
collect = function(timeout) {
repeat {
res = mccollect(self$jobs$pid, timeout = timeout)
if (is.null(res))
break
pids = as.integer(names(res))
self$jobs[pid %in% pids, count := count + 1L]
self$jobs = self$jobs[count < 1L]
}
}
)
)
#' @title ClusterFunctions for Parallel Multicore Execution
#'
#' @description
#' Jobs are spawned asynchronously using the functions \code{mcparallel} and \code{mccollect} (both in \pkg{parallel}).
#' Does not work on Windows, use \code{\link{makeClusterFunctionsSocket}} instead.
#'
#' @template ncpus
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsMulticore = function(ncpus = NA_integer_, fs.latency = 0) {
if (testOS("windows"))
stop("ClusterFunctionsMulticore do not support Windows. Use makeClusterFunctionsSocket instead.")
if (is.na(ncpus)) {
ncpus = max(as.numeric(getOption("mc.cores")), parallel::detectCores(), 1L, na.rm = TRUE)
info("Auto-detected %i CPUs", ncpus)
}
ncpus = asCount(ncpus, na.ok = FALSE, positive = TRUE)
p = Multicore$new(ncpus)
submitJob = function(reg, jc) {
force(jc)
pid = p$spawn(jc)
makeSubmitJobResult(status = 0L, batch.id = pid)
}
listJobsRunning = function(reg) {
assertRegistry(reg, writeable = FALSE)
p$list()
}
makeClusterFunctions(name = "Multicore", submitJob = submitJob, listJobsRunning = listJobsRunning,
store.job.collection = FALSE, fs.latency = fs.latency, hooks = list(pre.sync = function(reg, fns) p$collect(1)))
}
================================================
FILE: R/clusterFunctionsOpenLava.R
================================================
#' @title ClusterFunctions for OpenLava
#'
#' @description
#' Cluster functions for OpenLava.
#'
#' Job files are created based on the brew template \code{template}. This
#' file is processed with brew and then submitted to the queue using the
#' \code{bsub} command. Jobs are killed using the \code{bkill} command and the
#' list of running jobs is retrieved using \code{bjobs -u $USER -w}. The user
#' must have the appropriate privileges to submit, delete and list jobs on the
#' cluster (this is usually the case).
#'
#' The template file can access all resources passed to \code{\link{submitJobs}}
#' as well as all variables stored in the \code{\link{JobCollection}}.
#' It is the template file's job to choose a queue for the job and handle the desired resource
#' allocations.
#'
#' @note
#' Array jobs are currently not supported.
#'
#' @template template
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsOpenLava = function(template = "openlava", scheduler.latency = 1, fs.latency = 65) { # nocov start
template = findTemplateFile(template)
if (testScalarNA(template))
stopf("Argument 'template' (=\"%s\") must point to a readable template file", template)
template = cfReadBrewTemplate(template)
# When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only
# when unfinished jobs are found, and 255 when no jobs are found,
# or a non-existent job ID is entered.
Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = "Y")
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
outfile = cfBrewTemplate(reg, template, jc)
res = runOSCommand("bsub", stdin = shQuote(outfile))
if (res$exit.code > 0L) {
cfHandleUnknownSubmitError("bsub", res$exit.code, res$output)
} else {
batch.id = stri_extract_first_regex(stri_flatten(res$output, " "), "\\d+")
makeSubmitJobResult(status = 0L, batch.id = batch.id)
}
}
listJobs = function(reg, args) {
assertRegistry(reg, writeable = FALSE)
res = runOSCommand("bjobs", args)
if (res$exit.code > 0L) {
if (res$exit.code == 255L || any(stri_detect_regex(res$output, "No (unfinished|pending|running) job found")))
return(character(0L))
OSError("Listing of jobs failed", res)
}
stri_extract_first_regex(tail(res$output, -1L), "\\d+")
}
listJobsQueued = function(reg) {
listJobs(reg, c("-u $USER", "-w", "-p"))
}
listJobsRunning = function(reg) {
listJobs(reg, c("-u $USER", "-w", "-r"))
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "bkill", batch.id)
}
makeClusterFunctions(name = "OpenLava", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,
listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/clusterFunctionsSGE.R
================================================
#' @title ClusterFunctions for SGE Systems
#'
#' @description
#' Cluster functions for Univa Grid Engine / Oracle Grid Engine /
#' Sun Grid Engine (\url{https://altair.com/hpc-cloud-applications/}).
#'
#' Job files are created based on the brew template \code{template}. This
#' file is processed with brew and then submitted to the queue using the
#' \code{qsub} command. Jobs are killed using the \code{qdel} command and the
#' list of running jobs is retrieved using \code{qselect}. The user must have
#' the appropriate privileges to submit, delete and list jobs on the cluster
#' (this is usually the case).
#'
#' The template file can access all resources passed to \code{\link{submitJobs}}
#' as well as all variables stored in the \code{\link{JobCollection}}.
#' It is the template file's job to choose a queue for the job and handle the desired resource
#' allocations.
#'
#' @note
#' Array jobs are currently not supported.
#'
#' @template template
#' @inheritParams makeClusterFunctions
#' @template nodename
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsSGE = function(template = "sge", nodename = "localhost", scheduler.latency = 1, fs.latency = 65) { # nocov start
assertString(nodename)
template = findTemplateFile(template)
if (testScalarNA(template))
stopf("Argument 'template' (=\"%s\") must point to a readable template file", template)
template = cfReadBrewTemplate(template)
quote = if (isLocalHost(nodename)) identity else shQuote
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
outfile = cfBrewTemplate(reg, template, jc)
res = runOSCommand("qsub", shQuote(outfile), nodename = nodename)
if (res$exit.code > 0L) {
cfHandleUnknownSubmitError("qsub", res$exit.code, res$output)
} else {
batch.id = stri_extract_first_regex(stri_flatten(res$output, " "), "\\d+")
makeSubmitJobResult(status = 0L, batch.id = batch.id)
}
}
listJobs = function(reg, args) {
assertRegistry(reg, writeable = FALSE)
res = runOSCommand("qstat", args, nodename = nodename)
if (res$exit.code > 0L)
OSError("Listing of jobs failed", res)
stri_extract_first_regex(tail(res$output, -2L), "\\d+")
}
listJobsQueued = function(reg) {
listJobs(reg, c("-u $USER", "-s p"))
}
listJobsRunning = function(reg) {
listJobs(reg, c("-u $USER", "-s rs"))
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "qdel", batch.id, nodename = nodename)
}
makeClusterFunctions(name = "SGE", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,
listJobsRunning = listJobsRunning, store.job.collection = TRUE, scheduler.latency = scheduler.latency, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/clusterFunctionsSSH.R
================================================
#' @title ClusterFunctions for Remote SSH Execution
#'
#' @description
#' Jobs are spawned by starting multiple R sessions via \code{Rscript} over SSH.
#' If the hostname of the \code{\link{Worker}} equals \dQuote{localhost},
#' \code{Rscript} is called directly so that you do not need to have an SSH client installed.
#'
#' @param workers [\code{list} of \code{\link{Worker}}]\cr
#' List of Workers as constructed with \code{\link{Worker}}.
#' @inheritParams makeClusterFunctions
#'
#' @note
#' If you use a custom \dQuote{.ssh/config} file, make sure your
#' ProxyCommand passes \sQuote{-q} to ssh, otherwise each output will
#' end with the message \dQuote{Killed by signal 1} and this will break
#' the communication with the nodes.
#'
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
#' @examples
#' \dontrun{
#' # cluster functions for multicore execution on the local machine
#' makeClusterFunctionsSSH(list(Worker$new("localhost", ncpus = 2)))
#' }
makeClusterFunctionsSSH = function(workers, fs.latency = 65) { # nocov start
assertList(workers, types = "Worker")
names(workers) = vcapply(workers, "[[", "nodename")
if (anyDuplicated(names(workers)))
stop("Duplicated hostnames found in list of workers")
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
lapply(workers, function(w) w$update(reg))
rload = vnapply(workers, function(w) w$max.load / w$ncpus)
worker = Find(function(w) w$status == "available", sample(workers, prob = 1 / (rload + 0.1)), nomatch = NULL)
if (!is.null(worker) && worker$status == "available") {
pid = try(worker$start(reg, jc$uri, jc$log.file))
if (is.error(pid)) {
makeSubmitJobResult(status = 101L, batch.id = NA_character_, msg = "Submit failed.")
} else {
makeSubmitJobResult(status = 0L, batch.id = sprintf("%s#%s", worker$nodename, pid$output))
}
} else {
makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = sprintf("Busy: %s", workers[[1L]]$status))
}
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
nodename = stri_split_fixed(batch.id, "#", n = 2L)[[1L]][1L]
workers[[nodename]]$kill(reg, batch.id)
}
listJobsRunning = function(reg) {
assertRegistry(reg, writeable = FALSE)
unlist(lapply(workers, function(w) w$list(reg)), use.names = FALSE)
}
makeClusterFunctions(name = "SSH", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,
store.job.collection = TRUE, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/clusterFunctionsSlurm.R
================================================
#' @title ClusterFunctions for Slurm Systems
#'
#' @description
#' Cluster functions for Slurm (\url{https://slurm.schedmd.com/}).
#'
#' Job files are created based on the brew template \code{template.file}. This
#' file is processed with brew and then submitted to the queue using the
#' \code{sbatch} command. Jobs are killed using the \code{scancel} command and
#' the list of running jobs is retrieved using \code{squeue}. The user must
#' have the appropriate privileges to submit, delete and list jobs on the
#' cluster (this is usually the case).
#'
#' The template file can access all resources passed to \code{\link{submitJobs}}
#' as well as all variables stored in the \code{\link{JobCollection}}.
#' It is the template file's job to choose a queue for the job and handle the desired resource
#' allocations.
#'
#' Note that you might have to specify the cluster name here if you do not want to use the default,
#' otherwise the commands for listing and killing jobs will not work.
#'
#' @template template
#' @param array.jobs [\code{logical(1)}]\cr
#' If array jobs are disabled on the computing site, set to \code{FALSE}.
#' @template nodename
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsSlurm = function(template = "slurm", array.jobs = TRUE, nodename = "localhost", scheduler.latency = 1, fs.latency = 65) { # nocov start
assertFlag(array.jobs)
assertString(nodename)
template = findTemplateFile(template)
if (testScalarNA(template))
stopf("Argument 'template' (=\"%s\") must point to a readable template file", template)
template = cfReadBrewTemplate(template, "##")
quote = if (isLocalHost(nodename)) identity else shQuote
getClusters = function(reg) {
clusters = filterNull(lapply(reg$resources$resources, "[[", "cluster"))
if (length(clusters))
return(stri_flatten(unique(as.character(clusters)), ","))
return(character(0L))
}
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
if (jc$array.jobs) {
logs = sprintf("%s_%i", fs::path_file(jc$log.file), seq_row(jc$jobs))
jc$log.file = stri_join(jc$log.file, "_%a")
}
outfile = cfBrewTemplate(reg, template, jc)
res = runOSCommand("sbatch", shQuote(outfile), nodename = nodename)
output = stri_flatten(stri_trim_both(res$output), "\n")
if (res$exit.code > 0L) {
temp.errors = c(
"Batch job submission failed: Job violates accounting policy (job submit limit, user's size and/or time limits)",
"Socket timed out on send/recv operation",
"Submission rate too high, suggest using job arrays"
)
i = wf(stri_detect_fixed(output, temp.errors))
if (length(i) == 1L)
return(makeSubmitJobResult(status = i, batch.id = NA_character_, msg = temp.errors[i]))
return(cfHandleUnknownSubmitError("sbatch", res$exit.code, res$output))
}
id = stri_split_fixed(output[1L], " ")[[1L]][4L]
if (jc$array.jobs) {
if (!array.jobs)
stop("Array jobs not supported by cluster function")
makeSubmitJobResult(status = 0L, batch.id = sprintf("%s_%i", id, seq_row(jc$jobs)), log.file = logs)
} else {
makeSubmitJobResult(status = 0L, batch.id = id)
}
}
listJobs = function(reg, args) {
assertRegistry(reg, writeable = FALSE)
args = c(args, "--noheader", "--format=%i")
if (array.jobs)
args = c(args, "-r")
clusters = getClusters(reg)
if (length(clusters))
args = c(args, sprintf("--clusters=%s", clusters))
res = runOSCommand("squeue", args, nodename = nodename)
if (res$exit.code > 0L)
OSError("Listing of jobs failed", res)
if (length(clusters)) tail(res$output, -1L) else res$output
}
# Full List of Slurm job state codes:
# https://slurm.schedmd.com/squeue.html
# BF,CA,CD,CF,CG,DL,F,NF,OOM,PD,PR,R,RD,RF,RH,RS,RV,SI,SE,SO,ST,S,TO
# Querying by RD (RESV_DEL_HOLD) status throwing error on slurm v20.11.4
listJobsQueued = function(reg) {
args = c(quote("--user=$USER"), "--states=PD,CF,RF,RH,RQ,SE")
listJobs(reg, args)
}
listJobsRunning = function(reg) {
args = c(quote("--user=$USER"), "--states=R,S,CG,RS,SI,SO,ST")
listJobs(reg, args)
}
# Slurm job state codes that will result in an expired status:
# BF,CA,CD,DL,F,NF,OOM,PR,RV,TO,RD
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "scancel", c(sprintf("--clusters=%s", getClusters(reg)), batch.id), nodename = nodename)
}
makeClusterFunctions(name = "Slurm", submitJob = submitJob, killJob = killJob, listJobsRunning = listJobsRunning,
listJobsQueued = listJobsQueued, array.var = "SLURM_ARRAY_TASK_ID", store.job.collection = TRUE,
store.job.files = !isLocalHost(nodename), scheduler.latency = scheduler.latency, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/clusterFunctionsSocket.R
================================================
Socket = R6Class("Socket",
cloneable = FALSE,
public = list(
cl = NULL,
pids = NULL,
initialize = function(ncpus) {
loadNamespace("snow")
self$cl = snow::makeSOCKcluster(rep.int("localhost", ncpus))
self$pids = character(ncpus)
reg.finalizer(self, function(e) if (!is.null(e$cl)) { snow::stopCluster(e$cl); self$cl = NULL }, onexit = TRUE)
},
spawn = function(jc, ...) {
force(jc)
if (all(nzchar(self$pids))) {
res = snow::recvOneResult(self$cl)
self$pids[self$pids == res$tag] = ""
}
i = wf(!nzchar(self$pids))
snow::sendCall(self$cl[[i]], doJobCollection, list(jc = jc, output = jc$log.file), return = FALSE, tag = jc$job.hash)
self$pids[i] = jc$job.hash
invisible(jc$job.hash)
},
list = function() {
if (is.null(self$cl))
return(character(0L))
sl = lapply(self$cl, function(x) x$con)
finished = which(socketSelect(sl, write = FALSE, timeout = 1))
for (i in seq_along(finished)) {
res = snow::recvOneResult(self$cl)
self$pids[self$pids == res$tag] = ""
}
self$pids[nzchar(self$pids)]
}
)
)
#' @title ClusterFunctions for Parallel Socket Execution
#'
#' @description
#' Jobs are spawned asynchronously using the package \pkg{snow}.
#'
#' @template ncpus
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsSocket = function(ncpus = NA_integer_, fs.latency = 65) {
assertCount(ncpus, positive = TRUE, na.ok = TRUE)
if (is.na(ncpus)) {
ncpus = max(getOption("mc.cores", parallel::detectCores()), 1L, na.rm = TRUE)
info("Auto-detected %i CPUs", ncpus)
}
p = Socket$new(ncpus)
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
p$spawn(jc)
makeSubmitJobResult(status = 0L, batch.id = jc$job.hash)
}
listJobsRunning = function(reg) {
assertRegistry(reg, writeable = FALSE)
p$list()
}
makeClusterFunctions(name = "Socket", submitJob = submitJob, listJobsRunning = listJobsRunning,
store.job.collection = FALSE, fs.latency = fs.latency, hooks = list(pre.sync = function(reg, fns) p$list()))
}
================================================
FILE: R/clusterFunctionsTORQUE.R
================================================
#' @title ClusterFunctions for OpenPBS/TORQUE Systems
#'
#' @description
#' Cluster functions for TORQUE/PBS (\url{https://adaptivecomputing.com/cherry-services/torque-resource-manager/}).
#'
#' Job files are created based on the brew template \code{template.file}. This file is processed
#' with brew and then submitted to the queue using the \code{qsub} command. Jobs are killed using
#' the \code{qdel} command and the list of running jobs is retrieved using \code{qselect}. The user
#' must have the appropriate privileges to submit, delete and list jobs on the cluster (this is
#' usually the case).
#'
#' The template file can access all resources passed to \code{\link{submitJobs}}
#' as well as all variables stored in the \code{\link{JobCollection}}.
#' It is the template file's job to choose a queue for the job and handle the desired resource
#' allocations.
#'
#' @template template
#' @inheritParams makeClusterFunctions
#' @return [\code{\link{ClusterFunctions}}].
#' @family ClusterFunctions
#' @export
makeClusterFunctionsTORQUE = function(template = "torque", scheduler.latency = 1, fs.latency = 65) { # nocov start
template = findTemplateFile(template)
if (testScalarNA(template))
stopf("Argument 'template' (=\"%s\") must point to a readable template", template)
template = cfReadBrewTemplate(template, "##")
submitJob = function(reg, jc) {
assertRegistry(reg, writeable = TRUE)
assertClass(jc, "JobCollection")
outfile = cfBrewTemplate(reg, template, jc)
res = runOSCommand("qsub", shQuote(outfile))
output = stri_flatten(stri_trim_both(res$output), "\n")
if (res$exit.code > 0L) {
max.jobs.msg = "Maximum number of jobs already in queue"
if (stri_detect_fixed(output, max.jobs.msg) || res$exit.code == 228L)
return(makeSubmitJobResult(status = 1L, batch.id = NA_character_, msg = max.jobs.msg))
return(cfHandleUnknownSubmitError("qsub", res$exit.code, res$output))
}
if (jc$array.jobs) {
logs = sprintf("%s-%i", fs::path_file(jc$log.file), seq_row(jc$jobs))
makeSubmitJobResult(status = 0L, batch.id = stri_replace_first_fixed(output, "[]", stri_paste("[", seq_row(jc$jobs), "]")), log.file = logs)
} else {
makeSubmitJobResult(status = 0L, batch.id = output)
}
}
killJob = function(reg, batch.id) {
assertRegistry(reg, writeable = TRUE)
assertString(batch.id)
cfKillJob(reg, "qdel", batch.id)
}
listJobs = function(reg, args) {
assertRegistry(reg, writeable = FALSE)
res = runOSCommand("qselect", args)
if (res$exit.code > 0L)
OSError("Listing of jobs failed", res)
res$output
}
listJobsQueued = function(reg) {
args = c("-u $USER", "-s QW")
listJobs(reg, args)
}
listJobsRunning = function(reg) {
args = c("-u $USER", "-s EHRT")
listJobs(reg, args)
}
makeClusterFunctions(name = "TORQUE", submitJob = submitJob, killJob = killJob, listJobsQueued = listJobsQueued,
listJobsRunning = listJobsRunning, array.var = "PBS_ARRAYID", store.job.collection = TRUE,
scheduler.latency = scheduler.latency, fs.latency = fs.latency)
} # nocov end
================================================
FILE: R/config.R
================================================
#' @title Find a batchtools Configuration File
#'
#' @description
#' This functions returns the path to the first configuration file found in the following locations:
#' \enumerate{
#' \item{File \dQuote{batchtools.conf.R} in the path specified by the environment variable \dQuote{R_BATCHTOOLS_SEARCH_PATH}.}
#' \item{File \dQuote{batchtools.conf.R} in the current working directory.}
#' \item{File \dQuote{config.R} in the user configuration directory as reported by \code{rappdirs::user_config_dir("batchtools", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \dQuote{~/.config/batchtools/config.R}).}
#' \item{\dQuote{.batchtools.conf.R} in the home directory (\dQuote{~}).}
#' \item{\dQuote{config.R} in the site config directory as reported by \code{rappdirs::site_config_dir("batchtools")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}
#' }
#' @return [\code{character(1)}] Path to the configuration file or \code{NA} if no configuration file was found.
#' @keywords internal
#' @export
findConfFile = function() {
x = Sys.getenv("R_BATCHTOOLS_SEARCH_PATH")
if (nzchar(x)) {
x = fs::path(x, "batchtools.conf.R")
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
}
x = "batchtools.conf.R"
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
x = fs::path(user_config_dir("batchtools", expand = FALSE), "config.R")
if (fs::file_access(x, "read"))
return(x)
x = fs::path("~", ".batchtools.conf.R")
if (fs::file_access(x, "read"))
return(fs::path_abs(x))
x = fs::path(site_config_dir("batchtools"), "config.R")
if (fs::file_access(x, "read"))
return(x)
return(NA_character_)
}
setSystemConf = function(reg, conf.file) {
reg$cluster.functions = makeClusterFunctionsInteractive()
reg$default.resources = list()
reg$temp.dir = fs::path_temp()
reg$compress = "gzip"
if (!is.na(conf.file)) {
assertString(conf.file)
info("Sourcing configuration file '%s' ...", conf.file)
sys.source(conf.file, envir = reg, keep.source = FALSE)
assertClass(reg$cluster.functions, "ClusterFunctions")
assertList(reg$default.resources, names = "unique")
fs::dir_create(reg$temp.dir)
} else {
info("No readable configuration file found")
}
}
================================================
FILE: R/doJobCollection.R
================================================
#' @title Execute Jobs of a JobCollection
#'
#' @description
#' Executes every job in a \code{\link{JobCollection}}.
#' This function is intended to be called on the slave.
#'
#' @param jc [\code{\link{JobCollection}}]\cr
#' Either an object of class \dQuote{JobCollection} as returned by
#' \code{\link{makeJobCollection}} or a string with the path to file
#' containing a \dQuote{JobCollection} as RDS file (as stored by \code{\link{submitJobs}}).
#' @param output [\code{character(1)}]\cr
#' Path to a file to write the output to. Defaults to \code{NULL} which means
#' that output is written to the active \code{\link[base]{sink}}.
#' Do not set this if your scheduler redirects output to a log file.
#' @return [\code{character(1)}]: Hash of the \code{\link{JobCollection}} executed.
#' @family JobCollection
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMap(identity, 1:2, reg = tmp)
#' jc = makeJobCollection(1:2, reg = tmp)
#' doJobCollection(jc)
doJobCollection = function(jc, output = NULL) {
UseMethod("doJobCollection")
}
#' @export
doJobCollection.character = function(jc, output = NULL) {
obj = readRDS(jc)
force(obj)
if (!batchtools$debug && !obj$array.jobs) {
fs::file_delete(jc)
job = fs::path_ext_set(jc, "job")
if (fs::file_exists(job))
fs::file_delete(job)
}
doJobCollection.JobCollection(obj, output = output)
}
#' @export
doJobCollection.JobCollection = function(jc, output = NULL) {
error = function(msg, ...) {
now = ustamp()
updates = data.table(job.id = jc$jobs$job.id, started = now, done = now,
error = stri_trunc(stri_trim_both(sprintf(msg, ...)), 500L, " [truncated]"),
mem.used = NA_real_, key = "job.id")
writeRDS(updates, file = fs::path(jc$file.dir, "updates", sprintf("%s.rds", jc$job.hash)), compress = jc$compress)
invisible(NULL)
}
# signal warnings immediately
opts = options("warn")
options(warn = 1L)
on.exit(options(opts))
# setup output connection
if (!is.null(output)) {
if (!testPathForOutput(output, overwrite = TRUE))
return(error("Cannot create output file for logging"))
fp = file(output, open = "wt")
sink(file = fp)
sink(file = fp, type = "message")
on.exit({ sink(type = "message"); sink(type = "output"); close(fp) }, add = TRUE)
}
# subset array jobs
if (jc$array.jobs) {
i = as.integer(Sys.getenv(jc$array.var))
if (!testInteger(i, any.missing = FALSE, lower = 1L, upper = nrow(jc$jobs)))
return(error("Failed to subset JobCollection using array environment variable '%s' [='%s']", jc$array.var, i))
jc$jobs = jc$jobs[i]
}
# say hi
n.jobs = nrow(jc$jobs)
s = now()
catf("### [bt%s]: This is batchtools v%s", s, packageVersion("batchtools"))
catf("### [bt%s]: Starting calculation of %i jobs", s, n.jobs)
catf("### [bt%s]: Setting working directory to '%s'", s, jc$work.dir)
# set work dir
if (!fs::dir_exists(jc$work.dir))
return(error("Work dir does not exist"))
local_dir(jc$work.dir)
# load registry dependencies: packages, source files, ...
# note that this should happen _before_ parallelMap or foreach is initialized
ok = try(loadRegistryDependencies(jc, must.work = TRUE), silent = TRUE)
if (is.error(ok))
return(error("Error loading registry dependencies: %s", as.character(ok)))
# setup inner parallelization with parallelMap
if (hasName(jc$resources, "pm.backend")) {
if (!requireNamespace("parallelMap", quietly = TRUE))
return(error("parallelMap not installed"))
pm.opts = filterNull(insert(list(mode = jc$resources$pm.backend, cpus = jc$resources$ncpus, show.info = FALSE), jc$resources$pm.opts))
do.call(parallelMap::parallelStart, pm.opts)
on.exit(parallelMap::parallelStop(), add = TRUE)
pm.opts = parallelMap::parallelGetOptions()$settings
catf("### [bt%s]: Using %i CPUs for parallelMap/%s on level '%s'", s, pm.opts$cpus, pm.opts$mode, if (is.na(pm.opts$level)) "default" else pm.opts$level)
}
# setup inner parallelization with foreach
if (hasName(jc$resources, "foreach.backend")) {
if (!requireNamespace("foreach", quietly = TRUE))
return(error("Package 'foreach' is not installed"))
backend = jc$resources$foreach.backend
ncpus = jc$resources$ncpus
if (backend == "seq") {
foreach::registerDoSEQ()
} else if (backend == "parallel") {
if (!requireNamespace("doParallel", quietly = TRUE))
return(error("Package 'doParallel' is not installed"))
doParallel::registerDoParallel(cores = ncpus)
} else if (backend == "mpi") {
if (!requireNamespace("doMPI", quietly = TRUE))
return(error("Package 'doMPI' is not installed"))
cl = doMPI::startMPIcluster(count = ncpus)
doMPI::registerDoMPI(cl)
on.exit(doMPI::closeCluster(cl), add = TRUE)
} else {
return(error("Unknwon foreach backend: '%s'", backend))
}
catf("### [bt%s]: Using %i CPUs for foreach/%s", s, ncpus, backend)
}
# setup memory measurement
measure.memory = isTRUE(jc$resources$measure.memory)
catf("### [bt%s]: Memory measurement %s", s, ifelse(measure.memory, "enabled", "disabled"))
if (measure.memory) {
memory.mult = c(if (.Machine$sizeof.pointer == 4L) 28L else 56L, 8L)
}
# try to pre-fetch some objects from the file system
reader = RDSReader$new(n.jobs > 1L)
buf = UpdateBuffer$new(jc$jobs$job.id)
runHook(jc, "pre.do.collection", reader = reader)
for (i in seq_len(n.jobs)) {
job = getJob(jc, i, reader = reader)
id = job$id
update = list(started = ustamp(), done = NA_integer_, error = NA_character_, mem.used = NA_real_)
catf("### [bt%s]: Starting job [batchtools job.id=%i]", now(), id)
if (measure.memory) {
gc(reset = TRUE)
result = try(execJob(job))
update$mem.used = sum(gc()[, 1L] * memory.mult) / 1000000L
} else {
result = try(execJob(job))
}
update$done = ustamp()
if (is.error(result)) {
catf("\n### [bt%s]: Job terminated with an exception [batchtools job.id=%i]", now(), id)
update$error = stri_trunc(stri_trim_both(as.character(result)), 500L, " [truncated]")
} else {
catf("\n### [bt%s]: Job terminated successfully [batchtools job.id=%i]", now(), id)
writeRDS(result, file = getResultFiles(jc, id), compress = jc$compress)
}
buf$add(i, update)
buf$flush(jc)
}
runHook(jc, "post.do.collection", updates = buf$updates, reader = reader)
buf$save(jc)
catf("### [bt%s]: Calculation finished!", now())
invisible(jc$job.hash)
}
UpdateBuffer = R6Class("UpdateBuffer",
cloneable = FALSE,
public = list(
updates = NULL,
next.update = NA_real_,
initialize = function(ids) {
self$updates = data.table(job.id = ids, started = NA_real_, done = NA_real_, error = NA_character_, mem.used = NA_real_, written = FALSE, key = "job.id")
self$next.update = Sys.time() + runif(1L, 60, 300)
},
add = function(i, x) {
set(self$updates, i, names(x), x)
},
save = function(jc) {
i = self$updates[!is.na(started) & (!written), which = TRUE]
if (length(i) > 0L) {
first.id = self$updates$job.id[i[1L]]
writeRDS(self$updates[i, !"written"], file = fs::path(jc$file.dir, "updates", sprintf("%s-%i.rds", jc$job.hash, first.id)), compress = jc$compress)
set(self$updates, i, "written", TRUE)
}
},
flush = function(jc) {
now = Sys.time()
if (now > self$next.update) {
self$save(jc)
self$next.update = now + runif(1L, 60, 300)
}
}
)
)
================================================
FILE: R/estimateRuntimes.R
================================================
#' @title Estimate Remaining Runtimes
#'
#' @description
#' Estimates the runtimes of jobs using the random forest implemented in \pkg{ranger}.
#' Observed runtimes are retrieved from the \code{\link{Registry}} and runtimes are
#' predicted for unfinished jobs.
#'
#' The estimated remaining time is calculated in the \code{print} method.
#' You may also pass \code{n} here to determine the number of parallel jobs which is then used
#' in a simple Longest Processing Time (LPT) algorithm to give an estimate for the parallel runtime.
#'
#' @param tab [\code{\link[data.table]{data.table}}]\cr
#' Table with column \dQuote{job.id} and additional columns to predict the runtime.
#' Observed runtimes will be looked up in the registry and serve as dependent variable.
#' All columns in \code{tab} except \dQuote{job.id} will be passed to \code{\link[ranger]{ranger}} as
#' independent variables to fit the model.
#' @param ... [ANY]\cr
#' Additional parameters passed to \code{\link[ranger]{ranger}}. Ignored for the \code{print} method.
#' @template reg
#' @return [\code{RuntimeEstimate}] which is a \code{list} with two named elements:
#' \dQuote{runtimes} is a \code{\link[data.table]{data.table}} with columns \dQuote{job.id},
#' \dQuote{runtime} (in seconds) and \dQuote{type} (\dQuote{estimated} if runtime is estimated,
#' \dQuote{observed} if runtime was observed).
#' The other element of the list named \dQuote{model}] contains the fitted random forest object.
#' @export
#' @seealso \code{\link{binpack}} and \code{\link{lpt}} to chunk jobs according to their estimated runtimes.
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' # Create a simple toy registry
#' set.seed(1)
#' tmp = makeExperimentRegistry(file.dir = NA, make.default = FALSE, seed = 1)
#' addProblem(name = "iris", data = iris, fun = function(data, ...) nrow(data), reg = tmp)
#' addAlgorithm(name = "nrow", function(instance, ...) nrow(instance), reg = tmp)
#' addAlgorithm(name = "ncol", function(instance, ...) ncol(instance), reg = tmp)
#' addExperiments(algo.designs = list(nrow = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)
#' addExperiments(algo.designs = list(ncol = data.table::CJ(x = 1:50, y = letters[1:5])), reg = tmp)
#'
#' # We use the job parameters to predict runtimes
#' tab = unwrap(getJobPars(reg = tmp))
#'
#' # First we need to submit some jobs so that the forest can train on some data.
#' # Thus, we just sample some jobs from the registry while grouping by factor variables.
#' library(data.table)
#' ids = tab[, .SD[sample(nrow(.SD), 5)], by = c("problem", "algorithm", "y")]
#' setkeyv(ids, "job.id")
#' submitJobs(ids, reg = tmp)
#' waitForJobs(reg = tmp)
#'
#' # We "simulate" some more realistic runtimes here to demonstrate the functionality:
#' # - Algorithm "ncol" is 5 times more expensive than "nrow"
#' # - x has no effect on the runtime
#' # - If y is "a" or "b", the runtimes are really high
#' runtime = function(algorithm, x, y) {
#' ifelse(algorithm == "nrow", 100L, 500L) + 1000L * (y %in% letters[1:2])
#' }
#' tmp$status[ids, done := done + tab[ids, runtime(algorithm, x, y)]]
#' rjoin(sjoin(tab, ids), getJobStatus(ids, reg = tmp)[, c("job.id", "time.running")])
#'
#' # Estimate runtimes:
#' est = estimateRuntimes(tab, reg = tmp)
#' print(est)
#' rjoin(tab, est$runtimes)
#' print(est, n = 10)
#'
#' # Submit jobs with longest runtime first:
#' ids = est$runtimes[type == "estimated"][order(runtime, decreasing = TRUE)]
#' print(ids)
#' \dontrun{
#' submitJobs(ids, reg = tmp)
#' }
#'
#' # Group jobs into chunks with runtime < 1h
#' ids = est$runtimes[type == "estimated"]
#' ids[, chunk := binpack(runtime, 3600)]
#' print(ids)
#' print(ids[, list(runtime = sum(runtime)), by = chunk])
#' \dontrun{
#' submitJobs(ids, reg = tmp)
#' }
#'
#' # Group jobs into 10 chunks with similar runtime
#' ids = est$runtimes[type == "estimated"]
#' ids[, chunk := lpt(runtime, 10)]
#' print(ids[, list(runtime = sum(runtime)), by = chunk])
estimateRuntimes = function(tab, ..., reg = getDefaultRegistry()) {
assertRegistry(reg, sync = TRUE)
data = copy(convertIds(reg, tab, keep.extra = names(tab)))
if (!requireNamespace("ranger", quietly = TRUE))
stop("Please install package 'ranger' for runtime estimation")
data[, "runtime" := as.numeric(getJobStatus(tab, reg)$time.running)]
i = is.na(data$runtime)
if (all(i))
stop("No training data available. Some jobs must be finished before estimating runtimes.")
rf = ranger::ranger(runtime ~ ., data = data[!i, !"job.id"], ...)
data[i, "runtime" := predict(rf, .SD)$predictions, .SDcols = chsetdiff(names(data), c("job.id", "runtime"))]
data$type = factor(ifelse(i, "estimated", "observed"), levels = c("observed", "estimated"))
setClasses(list(runtimes = data[, c("job.id", "type", "runtime")], model = rf), c("RuntimeEstimate", class(data)))
}
#' @rdname estimateRuntimes
#' @param x [\code{RuntimeEstimate}]\cr
#' Object to print.
#' @param n [\code{integer(1)}]\cr
#' Number of parallel jobs to assume for runtime estimation.
#' @export
print.RuntimeEstimate = function(x, n = 1L, ...) {
ps = function(x, nc = 2L) {
sprintf(paste0("%0", nc, "id %02ih %02im %.1fs"),
floor(x / 86400),
floor((x / 3600) %% 24L),
floor((x / 60) %% 60L),
x %% 60L
)
}
assertCount(n, positive = TRUE)
runtime = type = NULL
calculated = x$runtimes[type == "observed", sum(runtime)]
remaining = x$runtimes[type == "estimated", sum(runtime)]
total = calculated + remaining
nc = max(1L, nchar(total %/% 86400))
catf("Runtime Estimate for %i jobs with %i CPUs", nrow(x$runtimes), n)
catf(" Done : %s", ps(calculated, nc = nc))
if (x$runtimes[type == "estimated", .N] > 0L) {
catf(" Remaining: %s", ps(remaining, nc = nc))
if (n >= 2L) {
rt = x$runtimes[type == "estimated"]$runtime
bins = lpt(rt, n)
bins = vnapply(split(rt, bins), sum)
catf(" Parallel : %s", ps(max(bins), nc = nc))
}
}
catf(" Total : %s", ps(total, nc = nc))
}
================================================
FILE: R/execJob.R
================================================
#' @title Execute a Single Jobs
#'
#' @description
#' Executes a single job (as created by \code{\link{makeJob}}) and returns
#' its result. Also works for Experiments.
#'
#' @param job [\code{\link{Job}} | \code{\link{Experiment}}]\cr
#' Job/Experiment to execute.
#' @return Result of the job.
#' @export
#' @examples
#' \dontshow{ batchtools:::example_push_temp(1) }
#' tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#' batchMap(identity, 1:2, reg = tmp)
#' job = makeJob(1, reg = tmp)
#' execJob(job)
execJob = function(job) {
UseMethod("execJob")
}
#' @export
execJob.character = function(job) {
execJob(readRDS(job))
}
#' @export
execJob.JobCollection = function(job) {
if (nrow(job$jobs) != 1L)
stop("You must provide a JobCollection with exactly one job")
execJob(getJob(job, i = 1L))
}
#' @export
execJob.Job = function(job) {
opts = options("error")
options(error = function(e) traceback(2L))
on.exit(options(opts))
# this needs to be cat, message outputs to stderr which R cannot capture properly
catf("### [bt%s]: Setting seed to %i ...", now(), job$seed)
if (".job" %chin% names(formals(job$fun))) {
with_seed(job$seed, do.call(job$fun, c(job$pars, list(.job = job)), envir = .GlobalEnv))
} else {
with_seed(job$seed, do.call(job$fun, job$pars, envir = .GlobalEnv))
}
}
#' @export
execJob.Experiment = function(job) {
opts = options("error")
options(error = function(e) traceback(2L))
on.exit(options(opts))
# this needs to be cat, message outputs to stderr which R cannot capture properly
catf("### [bt%s]: Generating problem instance for problem '%s' ...", now(), job$prob.name)
instance = job$instance
force(instance)
job$allow.access.to.instance = FALSE
wrapper = function(...) job$algorithm$fun(job = job, data = job$problem$data, instance = instance, ...)
# this needs to be cat, message outputs to stderr which R cannot capture properly
catf("### [bt%s]: Applying algorithm '%s' on problem '%s' for job %i (seed = %i) ...", now(), job$algo.name, job$prob.name, job$id, job$seed)
with_seed(job$seed, do.call(wrapper, job$algo.pars, envir = .GlobalEnv))
}
================================================
FILE: R/files.R
================================================
dir = function(reg, what) {
fs::path(fs::path_expand(reg$file.dir), what)
}
getResultFiles = function(reg, ids) {
fs::path(dir(reg, "results"), sprintf("%i.rds", if (is.atomic(ids)) ids else ids$job.id))
}
getLogFiles = function(reg, ids) {
job.hash = log.file = NULL
tab = reg$status[list(ids), c("job.id", "job.hash", "log.file")]
tab[is.na(log.file) & !is.na(job.hash), log.file := sprintf("%s.log", job.hash)]
tab[!is.na(log.file), log.file := fs::path(dir(reg, "logs"), log.file)]$log.file
}
getJobFiles = function(reg, hash) {
fs::path(reg$file.dir, "jobs", sprintf("%s.rds", hash))
}
getExternalDirs = function(reg, ids) {
fs::path(dir(reg, "external"), if (is.atomic(ids)) ids else ids$job.id)
}
mangle = function(x) {
sprintf("%s.rds", base32_encode(x, use.padding = FALSE))
}
unmangle = function(x) {
base32_decode(stri_sub(x, to = -5L), use.padding = FALSE)
}
file_remove = function(x) {
fs::file_delete(x[fs::file_exists(x)])
while(any(i <- fs::file_exists(x))) {
Sys.sleep(0.5)
fs::file_delete(x[i])
}
}
file_mtime = function(x) {
fs::file_info(x)$modification_time
}
writeRDS = function(object, file, compress = "gzip") {
file_remove(file)
saveRDS(object, file = file, version = 2L, compress = compress)
waitForFile(file, 300)
invisible(TRUE)
}
================================================
FILE: R/findJobs.R
===========
gitextract_lqp2x_vj/
├── .Rbuildignore
├── .aspell/
│ ├── batchtools.rds
│ └── defaults.R
├── .editorconfig
├── .github/
│ ├── dependabot.yaml
│ └── workflows/
│ ├── pkgdown.yml
│ └── r-cmd-check.yml
├── .gitignore
├── .ignore
├── .lintr
├── DESCRIPTION
├── LICENSE
├── NAMESPACE
├── NEWS.md
├── R/
│ ├── Algorithm.R
│ ├── ExperimentRegistry.R
│ ├── Export.R
│ ├── Hooks.R
│ ├── Job.R
│ ├── JobCollection.R
│ ├── JobNames.R
│ ├── JobTables.R
│ ├── Joins.R
│ ├── Logs.R
│ ├── Problem.R
│ ├── RDSReader.R
│ ├── Registry.R
│ ├── Tags.R
│ ├── Worker.R
│ ├── addExperiments.R
│ ├── batchMap.R
│ ├── batchMapResults.R
│ ├── batchReduce.R
│ ├── btlapply.R
│ ├── chunkIds.R
│ ├── clearRegistry.R
│ ├── clusterFunctions.R
│ ├── clusterFunctionsDocker.R
│ ├── clusterFunctionsHyperQueue.R
│ ├── clusterFunctionsInteractive.R
│ ├── clusterFunctionsLSF.R
│ ├── clusterFunctionsMulticore.R
│ ├── clusterFunctionsOpenLava.R
│ ├── clusterFunctionsSGE.R
│ ├── clusterFunctionsSSH.R
│ ├── clusterFunctionsSlurm.R
│ ├── clusterFunctionsSocket.R
│ ├── clusterFunctionsTORQUE.R
│ ├── config.R
│ ├── doJobCollection.R
│ ├── estimateRuntimes.R
│ ├── execJob.R
│ ├── files.R
│ ├── findJobs.R
│ ├── getDefaultRegistry.R
│ ├── getErrorMessages.R
│ ├── getStatus.R
│ ├── helpers.R
│ ├── ids.R
│ ├── killJobs.R
│ ├── loadRegistry.R
│ ├── loadResult.R
│ ├── mergeRegistries.R
│ ├── reduceResults.R
│ ├── removeExperiments.R
│ ├── removeRegistry.R
│ ├── resetJobs.R
│ ├── runOSCommand.R
│ ├── saveRegistry.R
│ ├── sleep.R
│ ├── submitJobs.R
│ ├── summarizeExperiments.R
│ ├── sweepRegistry.R
│ ├── syncRegistry.R
│ ├── testJob.R
│ ├── unwrap.R
│ ├── updateRegisty.R
│ ├── waitForFiles.R
│ ├── waitForJobs.R
│ └── zzz.R
├── README.Rmd
├── README.md
├── _pkgdown.yml
├── docs/
│ ├── 404.html
│ ├── CNAME
│ ├── LICENSE-text.html
│ ├── articles/
│ │ ├── batchtools.html
│ │ ├── batchtools_files/
│ │ │ └── header-attrs-2.4/
│ │ │ └── header-attrs.js
│ │ └── index.html
│ ├── authors.html
│ ├── bootstrap-toc.css
│ ├── bootstrap-toc.js
│ ├── docsearch.css
│ ├── docsearch.js
│ ├── index.html
│ ├── news/
│ │ └── index.html
│ ├── pkgdown.css
│ ├── pkgdown.js
│ ├── pkgdown.yml
│ └── reference/
│ ├── JobCollection.html
│ ├── JobExperiment.html
│ ├── JobNames.html
│ ├── JoinTables.html
│ ├── Tags.html
│ ├── Worker.html
│ ├── addAlgorithm.html
│ ├── addExperiments.html
│ ├── addProblem.html
│ ├── assertRegistry.html
│ ├── batchExport.html
│ ├── batchMap.html
│ ├── batchMapResults.html
│ ├── batchReduce.html
│ ├── batchtools-deprecated.html
│ ├── batchtools-package.html
│ ├── btlapply.html
│ ├── cfBrewTemplate.html
│ ├── cfHandleUnknownSubmitError.html
│ ├── cfKillJob.html
│ ├── cfReadBrewTemplate.html
│ ├── chunk.html
│ ├── chunkIds.html
│ ├── clearRegistry.html
│ ├── doJobCollection.html
│ ├── estimateRuntimes.html
│ ├── execJob.html
│ ├── findConfFile.html
│ ├── findJobs.html
│ ├── findTemplateFile.html
│ ├── getDefaultRegistry.html
│ ├── getErrorMessages.html
│ ├── getJobTable.html
│ ├── getStatus.html
│ ├── grepLogs.html
│ ├── index.html
│ ├── killJobs.html
│ ├── loadRegistry.html
│ ├── loadResult.html
│ ├── makeClusterFunctions.html
│ ├── makeClusterFunctionsDocker.html
│ ├── makeClusterFunctionsInteractive.html
│ ├── makeClusterFunctionsLSF.html
│ ├── makeClusterFunctionsMulticore.html
│ ├── makeClusterFunctionsOpenLava.html
│ ├── makeClusterFunctionsSGE.html
│ ├── makeClusterFunctionsSSH.html
│ ├── makeClusterFunctionsSlurm.html
│ ├── makeClusterFunctionsSocket.html
│ ├── makeClusterFunctionsTORQUE.html
│ ├── makeExperimentRegistry.html
│ ├── makeRegistry.html
│ ├── makeSubmitJobResult.html
│ ├── reduceResults.html
│ ├── reduceResultsList.html
│ ├── removeExperiments.html
│ ├── removeRegistry.html
│ ├── resetJobs.html
│ ├── runHook.html
│ ├── runOSCommand.html
│ ├── saveRegistry.html
│ ├── showLog.html
│ ├── submitJobs.html
│ ├── summarizeExperiments.html
│ ├── sweepRegistry.html
│ ├── syncRegistry.html
│ ├── testJob.html
│ ├── unwrap.html
│ └── waitForJobs.html
├── inst/
│ ├── CITATION
│ ├── bin/
│ │ └── linux-helper
│ └── templates/
│ ├── lsf-simple.tmpl
│ ├── openlava-simple.tmpl
│ ├── sge-simple.tmpl
│ ├── slurm-dortmund.tmpl
│ ├── slurm-lido3.tmpl
│ ├── slurm-simple.tmpl
│ ├── testJob.tmpl
│ └── torque-lido.tmpl
├── man/
│ ├── JobCollection.Rd
│ ├── JobExperiment.Rd
│ ├── JobNames.Rd
│ ├── JoinTables.Rd
│ ├── Tags.Rd
│ ├── Worker.Rd
│ ├── addAlgorithm.Rd
│ ├── addExperiments.Rd
│ ├── addProblem.Rd
│ ├── assertRegistry.Rd
│ ├── batchExport.Rd
│ ├── batchMap.Rd
│ ├── batchMapResults.Rd
│ ├── batchReduce.Rd
│ ├── batchtools-package.Rd
│ ├── btlapply.Rd
│ ├── cfBrewTemplate.Rd
│ ├── cfHandleUnknownSubmitError.Rd
│ ├── cfKillJob.Rd
│ ├── cfReadBrewTemplate.Rd
│ ├── chunk.Rd
│ ├── clearRegistry.Rd
│ ├── doJobCollection.Rd
│ ├── estimateRuntimes.Rd
│ ├── execJob.Rd
│ ├── findConfFile.Rd
│ ├── findJobs.Rd
│ ├── findTemplateFile.Rd
│ ├── getDefaultRegistry.Rd
│ ├── getErrorMessages.Rd
│ ├── getJobTable.Rd
│ ├── getStatus.Rd
│ ├── grepLogs.Rd
│ ├── killJobs.Rd
│ ├── loadRegistry.Rd
│ ├── loadResult.Rd
│ ├── makeClusterFunctions.Rd
│ ├── makeClusterFunctionsDocker.Rd
│ ├── makeClusterFunctionsHyperQueue.Rd
│ ├── makeClusterFunctionsInteractive.Rd
│ ├── makeClusterFunctionsLSF.Rd
│ ├── makeClusterFunctionsMulticore.Rd
│ ├── makeClusterFunctionsOpenLava.Rd
│ ├── makeClusterFunctionsSGE.Rd
│ ├── makeClusterFunctionsSSH.Rd
│ ├── makeClusterFunctionsSlurm.Rd
│ ├── makeClusterFunctionsSocket.Rd
│ ├── makeClusterFunctionsTORQUE.Rd
│ ├── makeExperimentRegistry.Rd
│ ├── makeRegistry.Rd
│ ├── makeSubmitJobResult.Rd
│ ├── reduceResults.Rd
│ ├── reduceResultsList.Rd
│ ├── removeExperiments.Rd
│ ├── removeRegistry.Rd
│ ├── resetJobs.Rd
│ ├── runHook.Rd
│ ├── runOSCommand.Rd
│ ├── saveRegistry.Rd
│ ├── showLog.Rd
│ ├── submitJobs.Rd
│ ├── summarizeExperiments.Rd
│ ├── sweepRegistry.Rd
│ ├── syncRegistry.Rd
│ ├── testJob.Rd
│ ├── unwrap.Rd
│ └── waitForJobs.Rd
├── man-roxygen/
│ ├── expreg.R
│ ├── id.R
│ ├── ids.R
│ ├── missing.val.R
│ ├── more.args.R
│ ├── ncpus.R
│ ├── nodename.R
│ ├── reg.R
│ └── template.R
├── paper/
│ ├── codemeta.json
│ ├── paper.bib
│ └── paper.md
├── src/
│ ├── Makevars
│ ├── binpack.c
│ ├── count_not_missing.c
│ ├── fill_gaps.c
│ ├── init.c
│ └── lpt.c
├── tests/
│ ├── testthat/
│ │ ├── helper.R
│ │ ├── test_Algorithm.R
│ │ ├── test_ClusterFunctionHyperQueue.R
│ │ ├── test_ClusterFunctions.R
│ │ ├── test_ClusterFunctionsMulticore.R
│ │ ├── test_ClusterFunctionsSSH.R
│ │ ├── test_ClusterFunctionsSocket.R
│ │ ├── test_ExperimentRegistry.R
│ │ ├── test_Job.R
│ │ ├── test_JobCollection.R
│ │ ├── test_JobNames.R
│ │ ├── test_Problem.R
│ │ ├── test_Registry.R
│ │ ├── test_addExperiments.R
│ │ ├── test_batchMap.R
│ │ ├── test_batchReduce.R
│ │ ├── test_btlapply.R
│ │ ├── test_chunk.R
│ │ ├── test_convertIds.R
│ │ ├── test_count.R
│ │ ├── test_doJobCollection.R
│ │ ├── test_estimateRuntimes.R
│ │ ├── test_export.R
│ │ ├── test_findConfFile.R
│ │ ├── test_findJobs.R
│ │ ├── test_foreach.R
│ │ ├── test_future.R
│ │ ├── test_getErrorMessages.R
│ │ ├── test_getJobTable.R
│ │ ├── test_getStatus.R
│ │ ├── test_grepLogs.R
│ │ ├── test_hooks.R
│ │ ├── test_joins.R
│ │ ├── test_killJobs.R
│ │ ├── test_manual.R
│ │ ├── test_memory.R
│ │ ├── test_mergeRegistries.R
│ │ ├── test_parallelMap.R
│ │ ├── test_reduceResults.R
│ │ ├── test_removeExperiments.R
│ │ ├── test_removeRegistry.R
│ │ ├── test_resetJobs.R
│ │ ├── test_runOSCommand.R
│ │ ├── test_seed.R
│ │ ├── test_showLog.R
│ │ ├── test_sleep.R
│ │ ├── test_submitJobs.R
│ │ ├── test_summarizeExperiments.R
│ │ ├── test_sweepRegistry.R
│ │ ├── test_tags.R
│ │ ├── test_testJob.R
│ │ ├── test_unwrap.R
│ │ └── test_waitForJobs.R
│ └── testthat.R
└── vignettes/
├── batchtools.Rmd
├── function_overview.tex
└── tikz_prob_algo_simple.tex
SYMBOL INDEX (15 symbols across 7 files)
FILE: docs/docsearch.js
function matchedWords (line 54) | function matchedWords(hit) {
function updateHitURL (line 73) | function updateHitURL(hit) {
FILE: docs/pkgdown.js
function paths (line 42) | function paths(pathname) {
function prefix_length (line 53) | function prefix_length(needle, haystack) {
function changeTooltipMessage (line 72) | function changeTooltipMessage(element, msg) {
FILE: src/binpack.c
function SEXP (line 6) | SEXP attribute_hidden c_binpack(SEXP x_, SEXP order_, SEXP capacity_) {
FILE: src/count_not_missing.c
function R_len_t (line 5) | static R_len_t count_not_missing_logical(SEXP x) {
function R_len_t (line 16) | static R_len_t count_not_missing_integer(SEXP x) {
function R_len_t (line 27) | static R_len_t count_not_missing_double(SEXP x) {
function R_len_t (line 38) | static R_len_t count_not_missing_string(SEXP x) {
function R_len_t (line 48) | static R_len_t count_not_missing_list(SEXP x) {
function SEXP (line 58) | SEXP attribute_hidden count_not_missing(SEXP x) {
FILE: src/fill_gaps.c
function SEXP (line 7) | SEXP attribute_hidden fill_gaps(SEXP x) {
FILE: src/init.c
function R_init_batchtools (line 20) | void R_init_batchtools(DllInfo *dll) {
FILE: src/lpt.c
function SEXP (line 6) | SEXP attribute_hidden c_lpt(SEXP x_, SEXP order_, SEXP chunks_) {
Condensed preview — 320 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,772K chars).
[
{
"path": ".Rbuildignore",
"chars": 186,
"preview": "^LICENSE$\n^src/.+\\.o$\n^src/.+\\.so$\n\\.swp$\n^\\.ignore$\n^\\.editorconfig$\n^man-roxygen$\n^.*\\.Rproj$\n^\\.Rproj\\.user$\n^docs$\n^"
},
{
"path": ".aspell/defaults.R",
"chars": 141,
"preview": "Rd_files <- vignettes <- R_files <- description <-\n list(encoding = \"UTF-8\", language = \"en\", dictionaries = c(\"en_st"
},
{
"path": ".editorconfig",
"chars": 297,
"preview": "# See http://editorconfig.org\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\ninsert_final_newline = true\nindent_style"
},
{
"path": ".github/dependabot.yaml",
"chars": 118,
"preview": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n"
},
{
"path": ".github/workflows/pkgdown.yml",
"chars": 1521,
"preview": "# pkgdown workflow of the mlr3 ecosystem v0.1.0\n# https://github.com/mlr-org/actions\non:\n push:\n branches:\n - m"
},
{
"path": ".github/workflows/r-cmd-check.yml",
"chars": 2005,
"preview": "# r cmd check workflow of the mlr3 ecosystem v0.2.0\n# https://github.com/mlr-org/actions\non:\n workflow_dispatch:\n in"
},
{
"path": ".gitignore",
"chars": 136,
"preview": ".DS_Store\n/inst/doc\n/src/*.so\n/src/*.o\n/vignettes/*.html\n/vignettes/*.pdf\n/.Rproj.user\n/batchtools.Rproj\n*.tar.gz\n*.Rche"
},
{
"path": ".ignore",
"chars": 11,
"preview": "man/\ndocs/\n"
},
{
"path": ".lintr",
"chars": 732,
"preview": "linters: linters_with_defaults(\n # lintr defaults: https://lintr.r-lib.org/reference/default_linters.html\n # the f"
},
{
"path": "DESCRIPTION",
"chars": 2166,
"preview": "Package: batchtools\nTitle: Tools for Computation on Batch Systems\nVersion: 0.9.18\nAuthors@R: c(\n person(\"Michel\", \"La"
},
{
"path": "LICENSE",
"chars": 7651,
"preview": " GNU LESSER GENERAL PUBLIC LICENSE\n Version 3, 29 June 2007\n\n Copyright (C) 2007"
},
{
"path": "NAMESPACE",
"chars": 3664,
"preview": "# Generated by roxygen2: do not edit by hand\n\nS3method(doJobCollection,JobCollection)\nS3method(doJobCollection,character"
},
{
"path": "NEWS.md",
"chars": 9194,
"preview": "# batchtools 0.9.18\n\n* Fixed CRAN issues with documentation\n\n# batchtools 0.9.17\n\n* Fixed a bug in the finalizer of `Clu"
},
{
"path": "R/Algorithm.R",
"chars": 2875,
"preview": "#' @title Define Algorithms for Experiments\n#'\n#' @description\n#' Algorithms are functions which get the \\code{data} par"
},
{
"path": "R/ExperimentRegistry.R",
"chars": 3426,
"preview": "#' @title ExperimentRegistry Constructor\n#'\n#' @description\n#' \\code{makeExperimentRegistry} constructs a special \\code{"
},
{
"path": "R/Export.R",
"chars": 2220,
"preview": "#' @title Export Objects to the Slaves\n#'\n#' @description\n#' Objects are saved in subdirectory \\dQuote{exports} of the\n#"
},
{
"path": "R/Hooks.R",
"chars": 2862,
"preview": "#' @title Trigger Evaluation of Custom Function\n#'\n#' @description\n#' Hooks allow to trigger functions calls on specific"
},
{
"path": "R/Job.R",
"chars": 7849,
"preview": "BaseJob = R6Class(\"BaseJob\", cloneable = FALSE,\n public = list(\n file.dir = NULL,\n id = NULL,\n seed "
},
{
"path": "R/JobCollection.R",
"chars": 4337,
"preview": "#' @title JobCollection Constructor\n#'\n#' @description\n#' \\code{makeJobCollection} takes multiple job ids and creates an"
},
{
"path": "R/JobNames.R",
"chars": 1436,
"preview": "#' @title Set and Retrieve Job Names\n#' @name JobNames\n#'\n#' @description\n#' Set custom names for jobs. These are passed"
},
{
"path": "R/JobTables.R",
"chars": 4677,
"preview": "#' @title Query Job Information\n#'\n#' @description\n#' \\code{getJobStatus} returns the internal table which stores inform"
},
{
"path": "R/Joins.R",
"chars": 4715,
"preview": "#' @title Inner, Left, Right, Outer, Semi and Anti Join for Data Tables\n#' @name JoinTables\n#'\n#' @description\n#' These "
},
{
"path": "R/Logs.R",
"chars": 4096,
"preview": "#' @useDynLib batchtools fill_gaps\nreadLog = function(id, missing.as.empty = FALSE, reg = getDefaultRegistry()) {\n log."
},
{
"path": "R/Problem.R",
"chars": 5751,
"preview": "#' @title Define Problems for Experiments\n#'\n#' @description\n#' Problems may consist of up to two parts: A static, immut"
},
{
"path": "R/RDSReader.R",
"chars": 971,
"preview": "RDSReader = R6Class(\"RDSReader\",\n cloneable = FALSE,\n public = list(\n cache = list(),\n use.cache = NA,\n\n init"
},
{
"path": "R/Registry.R",
"chars": 15742,
"preview": "#' @title Registry Constructor\n#'\n#' @description\n#' \\code{makeRegistry} constructs the inter-communication object for a"
},
{
"path": "R/Tags.R",
"chars": 2652,
"preview": "#' @title Add or Remove Job Tags\n#' @name Tags\n#' @rdname Tags\n#'\n#' @description\n#' Add and remove arbitrary tags to jo"
},
{
"path": "R/Worker.R",
"chars": 3979,
"preview": "#' @title Create a Linux-Worker\n#' @docType class\n#' @format An \\code{\\link[R6]{R6Class}} generator object\n#'\n#' @descri"
},
{
"path": "R/addExperiments.R",
"chars": 8758,
"preview": "#' @title Add Experiments to the Registry\n#'\n#' @description\n#' Adds experiments (parametrized combinations of problems "
},
{
"path": "R/batchMap.R",
"chars": 4267,
"preview": "#' @title Map Operation for Batch Systems\n#'\n#' @description\n#' A parallel and asynchronous \\code{\\link[base]{Map}}/\\cod"
},
{
"path": "R/batchMapResults.R",
"chars": 2832,
"preview": "#' @title Map Over Results to Create New Jobs\n#'\n#' @description\n#' This function allows you to create new computational"
},
{
"path": "R/batchReduce.R",
"chars": 2410,
"preview": "#' @title Reduce Operation for Batch Systems\n#'\n#' @description\n#' A parallel and asynchronous \\code{\\link[base]{Reduce}"
},
{
"path": "R/btlapply.R",
"chars": 3447,
"preview": "#' @title Synchronous Apply Functions\n#'\n#' @description\n#' This is a set of functions acting as counterparts to the seq"
},
{
"path": "R/chunkIds.R",
"chars": 5279,
"preview": "#' @title Chunk Jobs for Sequential Execution\n#'\n#' @description\n#' Jobs can be partitioned into \\dQuote{chunks} to be e"
},
{
"path": "R/clearRegistry.R",
"chars": 636,
"preview": "#' @title Remove All Jobs\n#' @description\n#' Removes all jobs from a registry and calls \\code{\\link{sweepRegistry}}.\n#'\n"
},
{
"path": "R/clusterFunctions.R",
"chars": 14517,
"preview": "#' @title ClusterFunctions Constructor\n#'\n#' @description\n#' This is the constructor used to create \\emph{custom} cluste"
},
{
"path": "R/clusterFunctionsDocker.R",
"chars": 5965,
"preview": "#' @title ClusterFunctions for Docker\n#'\n#' @description\n#' Cluster functions for Docker/Docker Swarm (\\url{https://docs"
},
{
"path": "R/clusterFunctionsHyperQueue.R",
"chars": 3295,
"preview": "#' @title ClusterFunctions for HyperQueue\n#'\n#' @description\n#' Cluster functions for HyperQueue (\\url{https://it4innova"
},
{
"path": "R/clusterFunctionsInteractive.R",
"chars": 1755,
"preview": "#' @title ClusterFunctions for Sequential Execution in the Running R Session\n#'\n#' @description\n#' All jobs are executed"
},
{
"path": "R/clusterFunctionsLSF.R",
"chars": 3143,
"preview": "#' @title ClusterFunctions for LSF Systems\n#'\n#' @description\n#' Cluster functions for LSF (\\url{https://www.ibm.com/pro"
},
{
"path": "R/clusterFunctionsMulticore.R",
"chars": 3291,
"preview": "if (getRversion() < \"3.3.2\" && .Platform$OS.type != \"windows\") {\n # Provided patch for upstream which is shipped with R"
},
{
"path": "R/clusterFunctionsOpenLava.R",
"chars": 3028,
"preview": "#' @title ClusterFunctions for OpenLava\n#'\n#' @description\n#' Cluster functions for OpenLava.\n#'\n#' Job files are create"
},
{
"path": "R/clusterFunctionsSGE.R",
"chars": 2889,
"preview": "#' @title ClusterFunctions for SGE Systems\n#'\n#' @description\n#' Cluster functions for Univa Grid Engine / Oracle Grid E"
},
{
"path": "R/clusterFunctionsSSH.R",
"chars": 2660,
"preview": "#' @title ClusterFunctions for Remote SSH Execution\n#'\n#' @description\n#' Jobs are spawned by starting multiple R sessio"
},
{
"path": "R/clusterFunctionsSlurm.R",
"chars": 4993,
"preview": "#' @title ClusterFunctions for Slurm Systems\n#'\n#' @description\n#' Cluster functions for Slurm (\\url{https://slurm.sched"
},
{
"path": "R/clusterFunctionsSocket.R",
"chars": 2274,
"preview": "Socket = R6Class(\"Socket\",\n cloneable = FALSE,\n public = list(\n cl = NULL,\n pids = NULL,\n\n initialize = funct"
},
{
"path": "R/clusterFunctionsTORQUE.R",
"chars": 3140,
"preview": "#' @title ClusterFunctions for OpenPBS/TORQUE Systems\n#'\n#' @description\n#' Cluster functions for TORQUE/PBS (\\url{https"
},
{
"path": "R/config.R",
"chars": 2330,
"preview": "#' @title Find a batchtools Configuration File\n#'\n#' @description\n#' This functions returns the path to the first config"
},
{
"path": "R/doJobCollection.R",
"chars": 7675,
"preview": "#' @title Execute Jobs of a JobCollection\n#'\n#' @description\n#' Executes every job in a \\code{\\link{JobCollection}}.\n#' "
},
{
"path": "R/estimateRuntimes.R",
"chars": 6056,
"preview": "#' @title Estimate Remaining Runtimes\n#'\n#' @description\n#' Estimates the runtimes of jobs using the random forest imple"
},
{
"path": "R/execJob.R",
"chars": 2150,
"preview": "#' @title Execute a Single Jobs\n#'\n#' @description\n#' Executes a single job (as created by \\code{\\link{makeJob}}) and re"
},
{
"path": "R/files.R",
"chars": 1313,
"preview": "dir = function(reg, what) {\n fs::path(fs::path_expand(reg$file.dir), what)\n}\n\ngetResultFiles = function(reg, ids) {\n f"
},
{
"path": "R/findJobs.R",
"chars": 9458,
"preview": "#' @title Find and Filter Jobs\n#'\n#' @description\n#' These functions are used to find and filter jobs, depending on eith"
},
{
"path": "R/getDefaultRegistry.R",
"chars": 602,
"preview": "#' @title Get and Set the Default Registry\n#' @description\n#' \\code{getDefaultRegistry} returns the registry currently s"
},
{
"path": "R/getErrorMessages.R",
"chars": 1487,
"preview": "#' @title Retrieve Error Messages\n#'\n#' @description\n#' Extracts error messages from the internal data base and returns "
},
{
"path": "R/getStatus.R",
"chars": 3457,
"preview": "#' @title Summarize the Computational Status\n#'\n#' @description\n#' This function gives an encompassing overview over the"
},
{
"path": "R/helpers.R",
"chars": 3859,
"preview": "mergedJobs = function(reg, ids, cols) {\n if (is.null(ids))\n reg$defs[reg$status, cols, on = \"def.id\", nomatch = 0L, "
},
{
"path": "R/ids.R",
"chars": 2143,
"preview": "allIds = function(reg) {\n reg$status[, \"job.id\"]\n}\n\nnoIds = function() {\n data.table(job.id = integer(0L), key = \"job."
},
{
"path": "R/killJobs.R",
"chars": 2333,
"preview": "#' @title Kill Jobs\n#'\n#' @description\n#' Kill jobs which are currently running on the batch system.\n#'\n#' In case of an"
},
{
"path": "R/loadRegistry.R",
"chars": 3859,
"preview": "#' @title Load a Registry from the File System\n#'\n#' @description\n#' Loads a registry from its \\code{file.dir}.\n#'\n#' Mu"
},
{
"path": "R/loadResult.R",
"chars": 472,
"preview": "#' @title Load the Result of a Single Job\n#'\n#' @description\n#' Loads the result of a single job.\n#'\n#' @template id\n#' "
},
{
"path": "R/mergeRegistries.R",
"chars": 3391,
"preview": "# @title Merge the computational status of two registries\n#\n# @description\n# Merges the computational status of jobs fou"
},
{
"path": "R/reduceResults.R",
"chars": 9035,
"preview": "#' @title Reduce Results\n#'\n#' @description\n#' A version of \\code{\\link[base]{Reduce}} for \\code{\\link{Registry}} object"
},
{
"path": "R/removeExperiments.R",
"chars": 1087,
"preview": "#' @title Remove Experiments\n#'\n#' @description\n#' Remove Experiments from an \\code{\\link{ExperimentRegistry}}.\n#' This "
},
{
"path": "R/removeRegistry.R",
"chars": 1272,
"preview": "#' @title Remove a Registry from the File System\n#'\n#' @description\n#' All files will be erased from the file system, in"
},
{
"path": "R/resetJobs.R",
"chars": 1237,
"preview": "#' @title Reset the Computational State of Jobs\n#'\n#' @description\n#' Resets the computational state of jobs in the \\cod"
},
{
"path": "R/runOSCommand.R",
"chars": 2558,
"preview": "#' @title Run OS Commands on Local or Remote Machines\n#'\n#' @description\n#' This is a helper function to run arbitrary O"
},
{
"path": "R/saveRegistry.R",
"chars": 1143,
"preview": "#' @title Store the Registy to the File System\n#'\n#' @description\n#' Stores the registry on the file system in its \\dQuo"
},
{
"path": "R/sleep.R",
"chars": 457,
"preview": "getSleepFunction = function(reg, sleep) {\n if (is.null(sleep)) {\n if (is.null(reg$sleep))\n return(function(i) {"
},
{
"path": "R/submitJobs.R",
"chars": 17651,
"preview": "#' @title Submit Jobs to the Batch Systems\n#'\n#' @description\n#' Submits defined jobs to the batch system.\n#'\n#' After s"
},
{
"path": "R/summarizeExperiments.R",
"chars": 841,
"preview": "#' @title Quick Summary over Experiments\n#'\n#' @description\n#' Returns a frequency table of defined experiments.\n#' See "
},
{
"path": "R/sweepRegistry.R",
"chars": 2212,
"preview": "#' @title Check Consistency and Remove Obsolete Information\n#'\n#' @description\n#' Canceled jobs and jobs submitted multi"
},
{
"path": "R/syncRegistry.R",
"chars": 1546,
"preview": "#' @title Synchronize the Registry\n#'\n#' @description\n#' Parses update files written by the slaves to the file system an"
},
{
"path": "R/testJob.R",
"chars": 2214,
"preview": "#' @title Run Jobs Interactively\n#'\n#' @description\n#' Starts a single job on the local machine.\n#'\n#' @template id\n#' @"
},
{
"path": "R/unwrap.R",
"chars": 3502,
"preview": "#' @title Unwrap Nested Data Frames\n#'\n#' @description\n#' Some functions (e.g., \\code{\\link{getJobPars}}, \\code{\\link{ge"
},
{
"path": "R/updateRegisty.R",
"chars": 3727,
"preview": "# returns TRUE if the state possibly changed\nupdateRegistry = function(reg = getDefaultRegistry()) { # nocov start\n \"!D"
},
{
"path": "R/waitForFiles.R",
"chars": 1247,
"preview": "# use list.files() here as this seems to trick the nfs cache\n# see https://github.com/mlr-org/batchtools/issues/85\nwaitF"
},
{
"path": "R/waitForJobs.R",
"chars": 5357,
"preview": "#' @title Wait for Termination of Jobs\n#'\n#' @description\n#' This function simply waits until all jobs are terminated.\n#"
},
{
"path": "R/zzz.R",
"chars": 2087,
"preview": "#' @description\n#' For bug reports and feature requests please use the tracker:\n#' \\url{https://github.com/mlr-org/batch"
},
{
"path": "README.Rmd",
"chars": 6538,
"preview": "---\noutput: github_document\n---\n\n# batchtools\n\nPackage website: [release](https://batchtools.mlr-org.com/) | [dev](https"
},
{
"path": "README.md",
"chars": 6597,
"preview": "\n# batchtools\n\nPackage website: [release](https://batchtools.mlr-org.com/) \\|\n[dev](https://batchtools.mlr-org.com/dev/)"
},
{
"path": "_pkgdown.yml",
"chars": 2509,
"preview": "url: https://batchtools.mlr-org.com\n\ntemplate:\n bootstrap: 5\n light-switch: true\n math-rendering: mathjax\n package: "
},
{
"path": "docs/404.html",
"chars": 4972,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/CNAME",
"chars": 22,
"preview": "batchtools.mlr-org.com"
},
{
"path": "docs/LICENSE-text.html",
"chars": 12551,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/articles/batchtools.html",
"chars": 82140,
"preview": "<!DOCTYPE html>\n<!-- Generated by pkgdown: do not edit by hand --><html lang=\"en\">\n<head>\n<meta http-equiv=\"Content-Type"
},
{
"path": "docs/articles/batchtools_files/header-attrs-2.4/header-attrs.js",
"chars": 507,
"preview": "// Pandoc 2.9 adds attributes on both header and div. We remove the former (to\n// be compatible with the behavior of Pan"
},
{
"path": "docs/articles/index.html",
"chars": 4933,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/authors.html",
"chars": 6773,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/bootstrap-toc.css",
"chars": 1843,
"preview": "/*!\n * Bootstrap Table of Contents v0.4.1 (http://afeld.github.io/bootstrap-toc/)\n * Copyright 2015 Aidan Feldman\n * Lic"
},
{
"path": "docs/bootstrap-toc.js",
"chars": 4764,
"preview": "/*!\n * Bootstrap Table of Contents v0.4.1 (http://afeld.github.io/bootstrap-toc/)\n * Copyright 2015 Aidan Feldman\n * Lic"
},
{
"path": "docs/docsearch.css",
"chars": 11758,
"preview": "/* Docsearch -------------------------------------------------------------- */\n/*\n Source: https://github.com/algolia/d"
},
{
"path": "docs/docsearch.js",
"chars": 2018,
"preview": "$(function() {\n\n // register a handler to move the focus to the search bar\n // upon pressing shift + \"/\" (i.e. \"?\")\n "
},
{
"path": "docs/index.html",
"chars": 15277,
"preview": "<!DOCTYPE html>\n<!-- Generated by pkgdown: do not edit by hand --><html lang=\"en\">\n<head>\n<meta http-equiv=\"Content-Type"
},
{
"path": "docs/news/index.html",
"chars": 20302,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/pkgdown.css",
"chars": 6932,
"preview": "/* Sticky footer */\n\n/**\n * Basic idea: https://philipwalton.github.io/solved-by-flexbox/demos/sticky-footer/\n * Details"
},
{
"path": "docs/pkgdown.js",
"chars": 3232,
"preview": "/* http://gregfranko.com/blog/jquery-best-practices/ */\n(function($) {\n $(function() {\n\n $('.navbar-fixed-top').head"
},
{
"path": "docs/pkgdown.yml",
"chars": 118,
"preview": "pandoc: 2.11.0.2\npkgdown: 1.6.1\npkgdown_sha: ~\narticles:\n batchtools: batchtools.html\nlast_built: 2020-10-21T07:39Z\n\n"
},
{
"path": "docs/reference/JobCollection.html",
"chars": 13714,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/JobExperiment.html",
"chars": 15113,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/JobNames.html",
"chars": 10181,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/JoinTables.html",
"chars": 14892,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/Tags.html",
"chars": 14817,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/Worker.html",
"chars": 7732,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/addAlgorithm.html",
"chars": 8628,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/addExperiments.html",
"chars": 23229,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/addProblem.html",
"chars": 15867,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/assertRegistry.html",
"chars": 7824,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/batchExport.html",
"chars": 12137,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/batchMap.html",
"chars": 16518,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/batchMapResults.html",
"chars": 15529,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/batchReduce.html",
"chars": 12181,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/batchtools-deprecated.html",
"chars": 4757,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html>\n <head>\n <meta charset=\"utf-8\">\n<meta http-e"
},
{
"path": "docs/reference/batchtools-package.html",
"chars": 7395,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/btlapply.html",
"chars": 14380,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/cfBrewTemplate.html",
"chars": 8070,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/cfHandleUnknownSubmitError.html",
"chars": 7796,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/cfKillJob.html",
"chars": 8751,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/cfReadBrewTemplate.html",
"chars": 7518,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/chunk.html",
"chars": 27581,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/chunkIds.html",
"chars": 7937,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html>\n <head>\n <meta charset=\"utf-8\">\n<meta http-e"
},
{
"path": "docs/reference/clearRegistry.html",
"chars": 6795,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/doJobCollection.html",
"chars": 9666,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/estimateRuntimes.html",
"chars": 36066,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/execJob.html",
"chars": 7930,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/findConfFile.html",
"chars": 7675,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/findJobs.html",
"chars": 18691,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/findTemplateFile.html",
"chars": 7095,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/getDefaultRegistry.html",
"chars": 6967,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/getErrorMessages.html",
"chars": 12082,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/getJobTable.html",
"chars": 16845,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/getStatus.html",
"chars": 14283,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/grepLogs.html",
"chars": 8545,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/index.html",
"chars": 22715,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/killJobs.html",
"chars": 8112,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/loadRegistry.html",
"chars": 13572,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/loadResult.html",
"chars": 6933,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctions.html",
"chars": 12891,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsDocker.html",
"chars": 12076,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsInteractive.html",
"chars": 9242,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsLSF.html",
"chars": 10721,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsMulticore.html",
"chars": 8458,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsOpenLava.html",
"chars": 10560,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsSGE.html",
"chars": 11393,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsSSH.html",
"chars": 9470,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsSlurm.html",
"chars": 11834,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsSocket.html",
"chars": 8139,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeClusterFunctionsTORQUE.html",
"chars": 10682,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeExperimentRegistry.html",
"chars": 27160,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeRegistry.html",
"chars": 22215,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/makeSubmitJobResult.html",
"chars": 8441,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/reduceResults.html",
"chars": 18774,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/reduceResultsList.html",
"chars": 30466,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/removeExperiments.html",
"chars": 7731,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/removeRegistry.html",
"chars": 8791,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/resetJobs.html",
"chars": 8088,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/runHook.html",
"chars": 10251,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/runOSCommand.html",
"chars": 8836,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/saveRegistry.html",
"chars": 7457,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/showLog.html",
"chars": 12791,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/submitJobs.html",
"chars": 35716,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/summarizeExperiments.html",
"chars": 7810,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/sweepRegistry.html",
"chars": 7111,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/syncRegistry.html",
"chars": 7097,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/testJob.html",
"chars": 10240,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/unwrap.html",
"chars": 10235,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "docs/reference/waitForJobs.html",
"chars": 10073,
"preview": "<!-- Generated by pkgdown: do not edit by hand -->\n<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\">\n<m"
},
{
"path": "inst/CITATION",
"chars": 1754,
"preview": "## -*- mode: r -*-\n\ncitHeader(\"To cite BatchJobs, BatchExperiments or batchtools in publications use:\")\n\nbibentry(\"Artic"
},
{
"path": "inst/bin/linux-helper",
"chars": 2830,
"preview": "#!/bin/bash\n\n## linux-helper: Helper for the multicore and SSH cluster functions of the BatchJobs R\n## package.\n##\n## "
},
{
"path": "inst/templates/lsf-simple.tmpl",
"chars": 959,
"preview": "## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named li"
},
{
"path": "inst/templates/openlava-simple.tmpl",
"chars": 961,
"preview": "## Default resources can be set in your .batchtools.conf.R by defining the variable\n## 'default.resources' as a named li"
},
{
"path": "inst/templates/sge-simple.tmpl",
"chars": 910,
"preview": "#!/bin/bash\n\n## The name of the job, can be anything, simply used when displaying the list of running jobs\n#$ -N <%= job"
},
{
"path": "inst/templates/slurm-dortmund.tmpl",
"chars": 1290,
"preview": "#!/bin/bash\n<%\nbackend = resources$pm.backend %??% \"local\"\nncpus = resources$ncpus %??% 1L\nwalltime = asInt(resources$wa"
},
{
"path": "inst/templates/slurm-lido3.tmpl",
"chars": 2080,
"preview": "#!/bin/bash\n\n## Job Resource Interface Definition\n##\n## ncpus [integer(1)]: Number of required cpus per task,\n## "
},
{
"path": "inst/templates/slurm-simple.tmpl",
"chars": 2034,
"preview": "#!/bin/bash\n\n## Job Resource Interface Definition\n##\n## ntasks [integer(1)]: Number of required tasks,\n## "
},
{
"path": "inst/templates/testJob.tmpl",
"chars": 356,
"preview": "options(warn = 1L)\nSys.setenv(DEBUGME = \"<%= Sys.getenv('DEBUGME') %>\")\nrequireNamespace(\"batchtools\", quietly = TRUE)\nj"
},
{
"path": "inst/templates/torque-lido.tmpl",
"chars": 3279,
"preview": "#!/bin/bash\n<%\n## Check some resources and set sane defaults\nresources$walltime = asInt(resources$walltime, lower = 60L,"
},
{
"path": "man/JobCollection.Rd",
"chars": 2959,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/JobCollection.R\n\\name{makeJobCollection}\n\\"
},
{
"path": "man/JobExperiment.Rd",
"chars": 3214,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Job.R\n\\name{makeJob}\n\\alias{makeJob}\n\\alia"
},
{
"path": "man/JobNames.Rd",
"chars": 1526,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/JobNames.R\n\\name{JobNames}\n\\alias{JobNames"
},
{
"path": "man/JoinTables.Rd",
"chars": 2547,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Joins.R\n\\name{JoinTables}\n\\alias{JoinTable"
},
{
"path": "man/Tags.Rd",
"chars": 2004,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Tags.R\n\\name{Tags}\n\\alias{Tags}\n\\alias{add"
},
{
"path": "man/Worker.Rd",
"chars": 1340,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Worker.R\n\\docType{class}\n\\name{Worker}\n\\al"
},
{
"path": "man/addAlgorithm.Rd",
"chars": 1646,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Algorithm.R\n\\name{addAlgorithm}\n\\alias{add"
},
{
"path": "man/addExperiments.Rd",
"chars": 4192,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/addExperiments.R\n\\name{addExperiments}\n\\al"
},
{
"path": "man/addProblem.Rd",
"chars": 3718,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Problem.R\n\\name{addProblem}\n\\alias{addProb"
},
{
"path": "man/assertRegistry.Rd",
"chars": 1272,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Registry.R\n\\name{assertRegistry}\n\\alias{as"
},
{
"path": "man/batchExport.Rd",
"chars": 1389,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/Export.R\n\\name{batchExport}\n\\alias{batchEx"
},
{
"path": "man/batchMap.Rd",
"chars": 2838,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchMap.R\n\\name{batchMap}\n\\alias{batchMap"
},
{
"path": "man/batchMapResults.Rd",
"chars": 2706,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchMapResults.R\n\\name{batchMapResults}\n\\"
},
{
"path": "man/batchReduce.Rd",
"chars": 1978,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/batchReduce.R\n\\name{batchReduce}\n\\alias{ba"
},
{
"path": "man/batchtools-package.Rd",
"chars": 1486,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/zzz.R\n\\docType{package}\n\\name{batchtools-p"
},
{
"path": "man/btlapply.Rd",
"chars": 3084,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/btlapply.R\n\\name{btlapply}\n\\alias{btlapply"
},
{
"path": "man/cfBrewTemplate.Rd",
"chars": 1373,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfBrewTemplate}\n\\"
},
{
"path": "man/cfHandleUnknownSubmitError.Rd",
"chars": 1251,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfHandleUnknownSu"
},
{
"path": "man/cfKillJob.Rd",
"chars": 1640,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfKillJob}\n\\alias"
},
{
"path": "man/cfReadBrewTemplate.Rd",
"chars": 1066,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clusterFunctions.R\n\\name{cfReadBrewTemplat"
},
{
"path": "man/chunk.Rd",
"chars": 3914,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/chunkIds.R\n\\name{chunk}\n\\alias{chunk}\n\\ali"
},
{
"path": "man/clearRegistry.Rd",
"chars": 723,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/clearRegistry.R\n\\name{clearRegistry}\n\\alia"
},
{
"path": "man/doJobCollection.Rd",
"chars": 1257,
"preview": "% Generated by roxygen2: do not edit by hand\n% Please edit documentation in R/doJobCollection.R\n\\name{doJobCollection}\n\\"
}
]
// ... and 120 more files (download for full content)
About this extraction
This page contains the full source code of the mllg/batchtools GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 320 files (1.6 MB), approximately 508.8k tokens, and a symbol index with 15 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.