From 9f9a78274a250ff2a0d7b21b2a8b7bb0c04ecaf9 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Wed, 5 Apr 2017 16:10:24 -0400 Subject: [PATCH 01/25] use .RDS extension; deduplicate overview/readme --- DESCRIPTION | 3 +- R/get_slurm_out.R | 10 +- R/rslurm.R | 2 +- R/slurm_apply.R | 8 +- R/slurm_call.R | 8 +- R/slurm_utils.R | 3 +- README.md | 235 +----------------- inst/templates/slurm_run_R.txt | 4 +- inst/templates/slurm_run_single_R.txt | 4 +- vignettes/overview.Rmd | 197 +++++++++++++++ ...urm-vignette.Rmd => x-rslurm-vignette.Rmd} | 0 11 files changed, 226 insertions(+), 248 deletions(-) create mode 100644 vignettes/overview.Rmd rename vignettes/{rslurm-vignette.Rmd => x-rslurm-vignette.Rmd} (100%) diff --git a/DESCRIPTION b/DESCRIPTION index 7def5d9..511cf96 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,8 @@ Imports: parallel, whisker (>= 0.3) RoxygenNote: 6.0.1 -Suggests: testthat, +Suggests: + testthat, knitr, rmarkdown VignetteBuilder: knitr diff --git a/R/get_slurm_out.R b/R/get_slurm_out.R index fcba98c..95912b7 100755 --- a/R/get_slurm_out.R +++ b/R/get_slurm_out.R @@ -27,7 +27,9 @@ get_slurm_out <- function(slr_job, outtype = "raw", wait = TRUE) { # Check arguments - if (!(class(slr_job) == "slurm_job")) stop("slr_job must be a slurm_job") + if (!(class(slr_job) == "slurm_job")) { + stop("slr_job must be a slurm_job") + } outtypes <- c("table", "raw") if (!(outtype %in% outtypes)) { stop(paste("outtype should be one of:", paste(outtypes, collapse = ', '))) @@ -37,9 +39,11 @@ get_slurm_out <- function(slr_job, outtype = "raw", wait = TRUE) { } # Wait for slr_job using SLURM dependency - if (wait) wait_for_job(slr_job) + if (wait) { + wait_for_job(slr_job) + } - res_files <- paste0("results_", 0:(slr_job$nodes - 1), ".RData") + res_files <- paste0("results_", 0:(slr_job$nodes - 1), ".RDS") tmpdir <- paste0("_rslurm_", slr_job$jobname) missing_files <- setdiff(res_files, dir(path = tmpdir)) if (length(missing_files) > 0) { diff --git a/R/rslurm.R b/R/rslurm.R index a477ffc..214d21b 100755 --- a/R/rslurm.R +++ b/R/rslurm.R @@ -19,7 +19,7 @@ #' \code{\link{get_slurm_out}} and \code{\link{cleanup_files}}. #' #' For bug reports or questions about this package, contact -#' Philippe Marchand (pmarchand@@sesync.org). +#' Ian Carroll(icarroll@sesync.org). #' #' @section Function Specification: #' To be compatible with \code{\link{slurm_apply}}, a function may accept diff --git a/R/slurm_apply.R b/R/slurm_apply.R index 7c4aeb4..b980572 100755 --- a/R/slurm_apply.R +++ b/R/slurm_apply.R @@ -4,8 +4,8 @@ #' parameters in parallel, spread across multiple nodes of a SLURM cluster. #' #' This function creates a temporary folder ("_rslurm_[jobname]") in the current -#' directory, holding the .RData files, the R script and the Bash submission -#' script generated for the SLURM job. +#' directory, holding .RData and .RDS data files, the R script to run and the Bash +#' submission script generated for the SLURM job. #' #' The set of input parameters is divided in equal chunks sent to each node, and #' \code{f} is evaluated in parallel within each node using functions from the @@ -23,7 +23,7 @@ #' #' When processing the computation job, the SLURM cluster will output two types #' of files in the temporary folder: those containing the return values of the -#' function for each subset of parameters ("results_[node_id].RData") and those +#' function for each subset of parameters ("results_[node_id].RDS") and those #' containing any console or error output produced by R on each node #' ("slurm_[node_id].out"). #' @@ -102,7 +102,7 @@ slurm_apply <- function(f, params, jobname = NA, nodes = 2, cpus_per_node = 2, tmpdir <- paste0("_rslurm_", jobname) dir.create(tmpdir) - saveRDS(params, file = file.path(tmpdir, "params.RData")) + saveRDS(params, file = file.path(tmpdir, "params.RDS")) if (!is.null(add_objects)) { save(list = add_objects, file = file.path(tmpdir, "add_objects.RData")) } diff --git a/R/slurm_call.R b/R/slurm_call.R index d486554..edca447 100755 --- a/R/slurm_call.R +++ b/R/slurm_call.R @@ -4,8 +4,8 @@ #' cluster. #' #' This function creates a temporary folder ("_rslurm_[jobname]") in the current -#' directory, holding the .RData files, the R script and the Bash submission -#' script generated for the SLURM job. +#' directory, holding .RData and .RDS data files, the R script to run and the Bash +#' submission script generated for the SLURM job. #' #' The names of any other R objects (besides \code{params}) that \code{f} needs #' to access should be listed in the \code{add_objects} argument. @@ -20,7 +20,7 @@ #' #' When processing the computation job, the SLURM cluster will output two files #' in the temporary folder: one with the return value of the function -#' ("results_0.RData") and one containing any console or error output produced +#' ("results_0.RDS") and one containing any console or error output produced #' by R ("slurm_[node_id].out"). #' #' If \code{submit = TRUE}, the job is sent to the cluster and a confirmation @@ -76,7 +76,7 @@ slurm_call <- function(f, params, jobname = NA, add_objects = NULL, tmpdir <- paste0("_rslurm_", jobname) dir.create(tmpdir) - saveRDS(params, file = file.path(tmpdir, "params.RData")) + saveRDS(params, file = file.path(tmpdir, "params.RDS")) if (!is.null(add_objects)) { save(list = add_objects, file = file.path(tmpdir, "add_objects.RData")) } diff --git a/R/slurm_utils.R b/R/slurm_utils.R index e541a45..5c629ca 100755 --- a/R/slurm_utils.R +++ b/R/slurm_utils.R @@ -7,7 +7,8 @@ func_to_str <- function(f) { } -# Make jobname by cleaning user-provided name or (if NA) generate one from clock +# Make jobname by cleaning user-provided name or (if NA) generate one +# from base::tempfile make_jobname <- function(name) { if (is.na(name)) { tmpfile <- tempfile("_rslurm_", tmpdir=".") diff --git a/README.md b/README.md index 97044d5..44229ef 100755 --- a/README.md +++ b/README.md @@ -1,236 +1,11 @@ -rslurm -====== +# rslurm [![Travis-CI Build Status](https://travis-ci.org/SESYNC-ci/rslurm.svg?branch=master)](https://travis-ci.org/SESYNC-ci/rslurm) -Many computing-intensive processes in R involve the repeated evaluation of -a function over many items or parameter sets. These so-called -[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) -calculations can be run serially with the `lapply` or `Map` function, or in parallel -on a single machine with `mclapply` or `mcMap` (from the **parallel** package). - -The rslurm package simplifies the process of distributing this type of calculation -across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) -workload manager. Its main function, `slurm_apply`, automatically divides the -computation over multiple nodes and writes the necessary submission scripts. -It also includes functions to retrieve and combine the output from different nodes, -as well as wrappers for common SLURM commands. - -*Development of this R package was supported by the National Socio-Environmental Synthesis Center (SESYNC) under funding received from the National Science Foundation DBI-1052875.* - - -### Table of contents - -- [Basic example](#basic-example) -- [Single function evaluation](#single-function-evaluation) -- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) -- [Configuring SLURM options](#configuring-slurm-options) -- [Generating scripts for later submission](#generating-scripts-for-later-submission) -- [How it works / advanced customization](#how-it-works-advanced-customization) - - -## Basic example - -To illustrate a typical rslurm workflow, we use a simple function that takes -a mean and standard deviation as parameters, generates a million normal deviates -and returns the sample mean and standard deviation. - -```r -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} -``` - -We then create a parameter data frame where each row is a parameter set and each -column matches an argument of the function. - -```r -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) -``` - -``` - par_mu par_sd -1 1 0.1 -2 2 0.2 -3 3 0.3 -``` - -We can now pass that function and the parameters data frame to `slurm_apply`, -specifiying the number of cluster nodes to use and the number of CPUs per node. -The latter (`cpus_per_node`) determines how many processes will be forked on -each node, as the `mc.cores` argument of `parallel::mcMap`. -```r -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) -``` -The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of -information (job name and number of nodes) needed to retrieve the job's output. - -Assuming the function is run on a machine with access to the cluster, it also -prints a message confirming the job has been submitted to SLURM. -``` -Submitted batch job 352375 -``` - -Particular clusters may require the specification of additional SLURM options, -such as time and memory limits for the job. Also, when running R on a local -machine without direct cluster access, you may want to generate scripts to be -copied to the cluster and run at a later time. These topics are covered in -additional sections below this basic example. - -After the job has been submitted, you can call `print_job_status` to display its -status (in queue, running or completed) or call `cancel_slurm` to cancel its -execution. These functions are R wrappers for the SLURM command line functions -`squeue` and `scancel`, respectively. - -Once the job completes, `get_slurm_out` reads and combines the output from all -nodes. -```r -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) -``` - -``` - s_mu s_sd -1 1.000005 0.09987899 -2 2.000185 0.20001108 -3 3.000238 0.29988789 -``` - -When `outtype = "table"`, the outputs from each function evaluation are -row-bound into a single data frame; this is an appropriate format when the -function returns a simple vector. The default `outtype = "raw"` combines the -outputs into a list and can thus handle arbitrarily complex return objects. - -```r -res_raw <- get_slurm_out(sjob, outtype = "raw") -res_raw[1:3] -``` - -``` -[[1]] - s_mu s_sd -1.00000506 0.09987899 - -[[2]] - s_mu s_sd -2.0001852 0.2000111 - -[[3]] - s_mu s_sd -3.0002377 0.2998879 -``` - -The files generated by `slurm_apply` are saved in a folder named -*\_rslurm_[jobname]* under the current working directory. - -```r -dir("_rslurm_test_job") -``` - -``` -[1] "params.RData" "results_0.RData" "results_1.RData" "slurm_0.out" -[5] "slurm_1.out" "slurm_run.R" "submit.sh" -``` - -The utility function `cleanup_files` deletes the temporary folder for the -specified *slurm_job*. - - -## Single function evaluation - -In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which -sends a single function call to the cluster. It is analogous in syntax to the -base R function `do.call`, accepting a function and a named list of parameters -as arguments. - -```r -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) -``` - -Because `slurm_call` involves a single process on a single node, it does not -recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the -same additional arguments (detailed in the sections below) as `slurm_apply`. - - -## Adding auxiliary data and functions - -The function passed to `slurm_apply` can only receive atomic parameters stored -within a data frame. Suppose we want instead to apply a function `func` to a list -of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap -`func` in an inline function that takes an integer parameter. - -```r -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) -``` - -The `add_objects` argument specifies the names of any R objects (besides the -parameters data frame) that must be accessed by the function passed to -`slurm_apply`. These objects are saved to a `.RData` file that is loaded -on each cluster node prior to evaluating the function in parallel. - -By default, all R packages attached to the current R session will also be -attached (with `library`) on each cluster node, though this can be modified with -the optional `pkgs` argument. - - -## Configuring SLURM options - -The `slurm_options` argument allows you to set any of the command line -options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the -SLURM `sbatch` command. It should be formatted as a named list, using the long -names of each option (e.g. "time" rather than "t"). Flags, i.e. command line -options that are toggled rather than set to a particular value, should be set to -`TRUE` in `slurm_options`. For example, the following code: -```r -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) -``` -sets the command line options `--time=1:00:00 --share`. - - -## Generating scripts for later submission - -When working from a R session without direct access to the cluster, you can set -`submit = FALSE` within `slurm_apply`. The function will create the -*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without -submitting the job. You may then copy those files to the cluster and submit the -job manually by calling `sbatch submit.sh` from the command line. - - -## How it works / advanced customization - -As mentioned above, the `slurm_apply` function creates a job-specific folder. -This folder contains the parameters data frame and (if applicable) the objects -specified as `add_objects`, both saved in *.RData* files. The function also -generates a R script (`slurm_run.R`) to be run on each cluster node, as well -as a Bash script (`submit.sh`) to submit the job to SLURM. - -More specifically, the Bash script creates a SLURM job array, with each cluster -node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment -variable. This variable is read by `slurm_run.R`, which allows each instance of -the script to operate on a different parameter subset and write its output to -a different results file. The R script calls `parallel::mcMap` to parallelize -calculations on each node. - -Both `slurm_run.R` and `submit.sh` are generated from templates, using the -**whisker** package; these templates can be found in the `rslurm/templates` -subfolder in your R package library. There are two templates for each script, -one for `slurm_apply` and the other (with the word *single* in its title) for -`slurm_call`. - -While you should avoid changing any existing lines in the template scripts, you -may want to add `#SBATCH` lines to the `submit.sh` templates in order to -permanently set certain SLURM command line options and thus customize the package -to your particular cluster setup. - +## Overview +See [inst/overview.html] +## Acknowledgement +Development of this R package was supported by the National Socio-Environmental Synthesis Center (SESYNC) under funding received from the National Science Foundation DBI-1052875. diff --git a/inst/templates/slurm_run_R.txt b/inst/templates/slurm_run_R.txt index 232ed02..41a55a6 100755 --- a/inst/templates/slurm_run_R.txt +++ b/inst/templates/slurm_run_R.txt @@ -6,7 +6,7 @@ load('add_objects.RData') .rslurm_func <- {{{func}}} -.rslurm_params <- readRDS('params.RData') +.rslurm_params <- readRDS('params.RDS') .rslurm_id <- as.numeric(Sys.getenv('SLURM_ARRAY_TASK_ID')) .rslurm_istart <- .rslurm_id * {{{nchunk}}} + 1 .rslurm_iend <- min((.rslurm_id + 1) * {{{nchunk}}}, nrow(.rslurm_params)) @@ -14,4 +14,4 @@ load('add_objects.RData') .rslurm_params[.rslurm_istart:.rslurm_iend, , drop = FALSE], mc.cores = {{{cpus_per_node}}})) -saveRDS(.rslurm_result, file = paste0('results_', .rslurm_id, '.RData')) +saveRDS(.rslurm_result, file = paste0('results_', .rslurm_id, '.RDS')) diff --git a/inst/templates/slurm_run_single_R.txt b/inst/templates/slurm_run_single_R.txt index 284c0dc..51c3b53 100755 --- a/inst/templates/slurm_run_single_R.txt +++ b/inst/templates/slurm_run_single_R.txt @@ -6,7 +6,7 @@ load('add_objects.RData') .rslurm_func <- {{{func}}} -.rslurm_params <- readRDS('params.RData') +.rslurm_params <- readRDS('params.RDS') .rslurm_result <- do.call(.rslurm_func, .rslurm_params) -saveRDS(.rslurm_result, file = 'results_0.RData') +saveRDS(.rslurm_result, file = 'results_0.RDS') diff --git a/vignettes/overview.Rmd b/vignettes/overview.Rmd new file mode 100644 index 0000000..389de1e --- /dev/null +++ b/vignettes/overview.Rmd @@ -0,0 +1,197 @@ +--- +title: "Parallelize R code on a SLURM cluster" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{rslurm-vignette} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +Many computing-intensive processes in R involve the repeated evaluation of +a function over many items or parameter sets. These so-called +[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) +calculations can be run serially with the `lapply` or `Map` function, or in parallel +on a single machine with `mclapply` or `mcMap` (from the **parallel** package). + +The rslurm package simplifies the process of distributing this type of calculation +across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) +workload manager. Its main function, `slurm_apply`, automatically divides the +computation over multiple nodes and writes the necessary submission scripts. +It also includes functions to retrieve and combine the output from different nodes, +as well as wrappers for common SLURM commands. + +### Table of contents + +- [Basic example](#basic-example) +- [Single function evaluation](#single-function-evaluation) +- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) +- [Configuring SLURM options](#configuring-slurm-options) +- [Generating scripts for later submission](#generating-scripts-for-later-submission) +- [How it works / advanced customization](#how-it-works-advanced-customization) + + +## Basic example + +To illustrate a typical rslurm workflow, we use a simple function that takes +a mean and standard deviation as parameters, generates a million normal deviates +and returns the sample mean and standard deviation. + +```{r} +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} +``` + +We then create a parameter data frame where each row is a parameter set and each +column matches an argument of the function. + +```{r} +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) +``` + +We can now pass that function and the parameters data frame to `slurm_apply`, +specifiying the number of cluster nodes to use and the number of CPUs per node. +The latter (`cpus_per_node`) determines how many processes will be forked on +each node, as the `mc.cores` argument of `parallel::mcMap`. +```{r} +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) +``` +The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of +information (job name and number of nodes) needed to retrieve the job's output. + +Assuming the function is run on a machine with access to the cluster, it also +prints a message confirming the job has been submitted to SLURM. + +Particular clusters may require the specification of additional SLURM options, +such as time and memory limits for the job. Also, when running R on a local +machine without direct cluster access, you may want to generate scripts to be +copied to the cluster and run at a later time. These topics are covered in +additional sections below this basic example. + +After the job has been submitted, you can call `print_job_status` to display its +status (in queue, running or completed) or call `cancel_slurm` to cancel its +execution. These functions are R wrappers for the SLURM command line functions +`squeue` and `scancel`, respectively. + +Once the job completes, `get_slurm_out` reads and combines the output from all +nodes. +```{r} +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) +``` + +When `outtype = "table"`, the outputs from each function evaluation are +row-bound into a single data frame; this is an appropriate format when the +function returns a simple vector. The default `outtype = "raw"` combines the +outputs into a list and can thus handle arbitrarily complex return objects. + +```{r} +res_raw <- get_slurm_out(sjob, outtype = "raw") +res_raw[1:3] +``` + +The files generated by `slurm_apply` are saved in a folder named +*\_rslurm_[jobname]* under the current working directory. + +```{r} +dir("_rslurm_test_job") +``` + +The utility function `cleanup_files` deletes the temporary folder for the +specified *slurm_job*. + + +## Single function evaluation + +In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which +sends a single function call to the cluster. It is analogous in syntax to the +base R function `do.call`, accepting a function and a named list of parameters +as arguments. + +```{r} +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) +``` + +Because `slurm_call` involves a single process on a single node, it does not +recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the +same additional arguments (detailed in the sections below) as `slurm_apply`. + + +## Adding auxiliary data and functions + +The function passed to `slurm_apply` can only receive atomic parameters stored +within a data frame. Suppose we want instead to apply a function `func` to a list +of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap +`func` in an inline function that takes an integer parameter. + +```{r} +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) +``` + +The `add_objects` argument specifies the names of any R objects (besides the +parameters data frame) that must be accessed by the function passed to +`slurm_apply`. These objects are saved to a `.RData` file that is loaded +on each cluster node prior to evaluating the function in parallel. + +By default, all R packages attached to the current R session will also be +attached (with `library`) on each cluster node, though this can be modified with +the optional `pkgs` argument. + + +## Configuring SLURM options + +The `slurm_options` argument allows you to set any of the command line +options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the +SLURM `sbatch` command. It should be formatted as a named list, using the long +names of each option (e.g. "time" rather than "t"). Flags, i.e. command line +options that are toggled rather than set to a particular value, should be set to +`TRUE` in `slurm_options`. For example, the following code: +```{r} +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) +``` +sets the command line options `--time=1:00:00 --share`. + + +## Generating scripts for later submission + +When working from a R session without direct access to the cluster, you can set +`submit = FALSE` within `slurm_apply`. The function will create the +*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without +submitting the job. You may then copy those files to the cluster and submit the +job manually by calling `sbatch submit.sh` from the command line. + + +## How it works / advanced customization + +As mentioned above, the `slurm_apply` function creates a job-specific folder. +This folder contains the parameters as a *.RDS* file and (if applicable) the objects +specified as `add_objects` saved together in a *.RData* file. The function also +generates a R script (`slurm_run.R`) to be run on each cluster node, as well +as a Bash script (`submit.sh`) to submit the job to SLURM. + +More specifically, the Bash script creates a SLURM job array, with each cluster +node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment +variable. This variable is read by `slurm_run.R`, which allows each instance of +the script to operate on a different parameter subset and write its output to +a different results file. The R script calls `parallel::mcMap` to parallelize +calculations on each node. + +Both `slurm_run.R` and `submit.sh` are generated from templates, using the +**whisker** package; these templates can be found in the `rslurm/templates` +subfolder in your R package library. There are two templates for each script, +one for `slurm_apply` and the other (with the word *single* in its title) for +`slurm_call`. + +While you should avoid changing any existing lines in the template scripts, you +may want to add `#SBATCH` lines to the `submit.sh` templates in order to +permanently set certain SLURM command line options and thus customize the package +to your particular cluster setup. \ No newline at end of file diff --git a/vignettes/rslurm-vignette.Rmd b/vignettes/x-rslurm-vignette.Rmd similarity index 100% rename from vignettes/rslurm-vignette.Rmd rename to vignettes/x-rslurm-vignette.Rmd From f46ef282f4aa2614143928c1f62cbc179a643db8 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 10:33:14 -0400 Subject: [PATCH 02/25] update vignette; fix test for SLURM --- NEWS.md | 6 +- R/slurm_apply.R | 4 + R/slurm_call.R | 4 + cran-comments.md | 10 +- man/rslurm-package.Rd | 2 +- man/slurm_apply.Rd | 6 +- man/slurm_call.Rd | 6 +- vignettes/overview.Rmd | 10 +- vignettes/x-rslurm-vignette.Rmd | 233 -------------------------------- 9 files changed, 28 insertions(+), 253 deletions(-) delete mode 100755 vignettes/x-rslurm-vignette.Rmd diff --git a/NEWS.md b/NEWS.md index fb90110..ede3710 100755 --- a/NEWS.md +++ b/NEWS.md @@ -1,14 +1,16 @@ rslurm 0.3.2 ============ -*2017-03-07* +*2017-04-06* -Minor update to include new feature and bug fixes: +Minor update to include new feature and bug fix: * 'Wait' feature: adds option to slurm_apply and slurm_call to block the calling script until the submitted job completes. This option can be used to allow immediate processing of a submitted job's output. +* Use '.RDS' file extension, rather than '.RData', for serialized objects. + rslurm 0.3.1 ============ diff --git a/R/slurm_apply.R b/R/slurm_apply.R index b980572..9600c74 100755 --- a/R/slurm_apply.R +++ b/R/slurm_apply.R @@ -142,6 +142,10 @@ slurm_apply <- function(f, params, jobname = NA, nodes = 2, cpus_per_node = 2, writeLines(script_sh, file.path(tmpdir, "submit.sh")) # Submit job to SLURM if applicable + if (system('squeue', ignore.stdout = TRUE)) { + submit <- FALSE + cat("Cannot submit; no SLURM workload manager on path\n") + } if (submit) { jobid <- submit_slurm_job(tmpdir) } else { diff --git a/R/slurm_call.R b/R/slurm_call.R index edca447..eb2965a 100755 --- a/R/slurm_call.R +++ b/R/slurm_call.R @@ -103,6 +103,10 @@ slurm_call <- function(f, params, jobname = NA, add_objects = NULL, writeLines(script_sh, file.path(tmpdir, "submit.sh")) # Submit job to SLURM if applicable + if (system('squeue', ignore.stdout = TRUE)) { + submit <- FALSE + cat("Cannot submit; no SLURM workload manager on path\n") + } if (submit) { jobid <- submit_slurm_job(tmpdir) } else { diff --git a/cran-comments.md b/cran-comments.md index aa41dab..4eea8cc 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,5 +1,4 @@ -This is a minor update to the package to fix a previous error found by CRAN -when checking package on r-devel on Mac OS X. +This is a minor update to the package to add a new feature, update vignette, and fix bugs ## Tested on @@ -9,9 +8,4 @@ OS X with R 3.3 (local machine) ## R CMD check results -1 NOTE (for some versions) - Namespace in Imports field not imported from: ‘parallel’ - All declared Imports should be used. - -This note occurs because the `parallel::mcMap` function is called in the batch -scripts generated by this package's functions (from the template located at inst/templates/slurm\_run\_R.txt), rather than in the package's R code itself. +0 errors | 0 warnings | 0 notes diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 0012186..1db92f3 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -26,7 +26,7 @@ in the package: \code{\link{print_job_status}}, \code{\link{cancel_slurm}}, \code{\link{get_slurm_out}} and \code{\link{cleanup_files}}. For bug reports or questions about this package, contact -Philippe Marchand (pmarchand@sesync.org). +Ian Carroll(icarroll@sesync.org). } \section{Function Specification}{ diff --git a/man/slurm_apply.Rd b/man/slurm_apply.Rd index b730848..c942697 100755 --- a/man/slurm_apply.Rd +++ b/man/slurm_apply.Rd @@ -52,8 +52,8 @@ parameters in parallel, spread across multiple nodes of a SLURM cluster. } \details{ This function creates a temporary folder ("_rslurm_[jobname]") in the current -directory, holding the .RData files, the R script and the Bash submission -script generated for the SLURM job. +directory, holding .RData and .RDS data files, the R script to run and the Bash +submission script generated for the SLURM job. The set of input parameters is divided in equal chunks sent to each node, and \code{f} is evaluated in parallel within each node using functions from the @@ -71,7 +71,7 @@ not be manually set. When processing the computation job, the SLURM cluster will output two types of files in the temporary folder: those containing the return values of the -function for each subset of parameters ("results_[node_id].RData") and those +function for each subset of parameters ("results_[node_id].RDS") and those containing any console or error output produced by R on each node ("slurm_[node_id].out"). diff --git a/man/slurm_call.Rd b/man/slurm_call.Rd index 0cd265d..9859ab4 100755 --- a/man/slurm_call.Rd +++ b/man/slurm_call.Rd @@ -40,8 +40,8 @@ cluster. } \details{ This function creates a temporary folder ("_rslurm_[jobname]") in the current -directory, holding the .RData files, the R script and the Bash submission -script generated for the SLURM job. +directory, holding .RData and .RDS data files, the R script to run and the Bash +submission script generated for the SLURM job. The names of any other R objects (besides \code{params}) that \code{f} needs to access should be listed in the \code{add_objects} argument. @@ -56,7 +56,7 @@ not be manually set. When processing the computation job, the SLURM cluster will output two files in the temporary folder: one with the return value of the function -("results_0.RData") and one containing any console or error output produced +("results_0.RDS") and one containing any console or error output produced by R ("slurm_[node_id].out"). If \code{submit = TRUE}, the job is sent to the cluster and a confirmation diff --git a/vignettes/overview.Rmd b/vignettes/overview.Rmd index 389de1e..5d8b1f1 100644 --- a/vignettes/overview.Rmd +++ b/vignettes/overview.Rmd @@ -2,7 +2,7 @@ title: "Parallelize R code on a SLURM cluster" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{rslurm-vignette} + %\VignetteIndexEntry{overview} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- @@ -81,7 +81,7 @@ execution. These functions are R wrappers for the SLURM command line functions Once the job completes, `get_slurm_out` reads and combines the output from all nodes. ```{r} -res <- get_slurm_out(sjob, outtype = "table") +res <- get_slurm_out(sjob, outtype = "table", wait = TRUE) head(res, 3) ``` @@ -91,7 +91,7 @@ function returns a simple vector. The default `outtype = "raw"` combines the outputs into a list and can thus handle arbitrarily complex return objects. ```{r} -res_raw <- get_slurm_out(sjob, outtype = "raw") +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) res_raw[1:3] ``` @@ -129,6 +129,10 @@ within a data frame. Suppose we want instead to apply a function `func` to a lis of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap `func` in an inline function that takes an integer parameter. +```{r echo=FALSE} +obj_list <- list(NULL) +func <- function(obj) {} +``` ```{r} sjob <- slurm_apply(function(i) func(obj_list[[i]]), data.frame(i = seq_along(obj_list)), diff --git a/vignettes/x-rslurm-vignette.Rmd b/vignettes/x-rslurm-vignette.Rmd deleted file mode 100755 index d38bc77..0000000 --- a/vignettes/x-rslurm-vignette.Rmd +++ /dev/null @@ -1,233 +0,0 @@ ---- -title: "Parallelize R code on a SLURM cluster" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{rslurm-vignette} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -Many computing-intensive processes in R involve the repeated evaluation of -a function over many items or parameter sets. These so-called -[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) -calculations can be run serially with the `lapply` or `Map` function, or in parallel -on a single machine with `mclapply` or `mcMap` (from the **parallel** package). - -The rslurm package simplifies the process of distributing this type of calculation -across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) -workload manager. Its main function, `slurm_apply`, automatically divides the -computation over multiple nodes and writes the necessary submission scripts. -It also includes functions to retrieve and combine the output from different nodes, -as well as wrappers for common SLURM commands. - -### Table of contents - -- [Basic example](#basic-example) -- [Single function evaluation](#single-function-evaluation) -- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) -- [Configuring SLURM options](#configuring-slurm-options) -- [Generating scripts for later submission](#generating-scripts-for-later-submission) -- [How it works / advanced customization](#how-it-works-advanced-customization) - - -## Basic example - -To illustrate a typical rslurm workflow, we use a simple function that takes -a mean and standard deviation as parameters, generates a million normal deviates -and returns the sample mean and standard deviation. - -```r -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} -``` - -We then create a parameter data frame where each row is a parameter set and each -column matches an argument of the function. - -```r -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) -``` - -``` - par_mu par_sd -1 1 0.1 -2 2 0.2 -3 3 0.3 -``` - -We can now pass that function and the parameters data frame to `slurm_apply`, -specifiying the number of cluster nodes to use and the number of CPUs per node. -The latter (`cpus_per_node`) determines how many processes will be forked on -each node, as the `mc.cores` argument of `parallel::mcMap`. -```r -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) -``` -The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of -information (job name and number of nodes) needed to retrieve the job's output. - -Assuming the function is run on a machine with access to the cluster, it also -prints a message confirming the job has been submitted to SLURM. -``` -Submitted batch job 352375 -``` - -Particular clusters may require the specification of additional SLURM options, -such as time and memory limits for the job. Also, when running R on a local -machine without direct cluster access, you may want to generate scripts to be -copied to the cluster and run at a later time. These topics are covered in -additional sections below this basic example. - -After the job has been submitted, you can call `print_job_status` to display its -status (in queue, running or completed) or call `cancel_slurm` to cancel its -execution. These functions are R wrappers for the SLURM command line functions -`squeue` and `scancel`, respectively. - -Once the job completes, `get_slurm_out` reads and combines the output from all -nodes. -```r -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) -``` - -``` - s_mu s_sd -1 1.000005 0.09987899 -2 2.000185 0.20001108 -3 3.000238 0.29988789 -``` - -When `outtype = "table"`, the outputs from each function evaluation are -row-bound into a single data frame; this is an appropriate format when the -function returns a simple vector. The default `outtype = "raw"` combines the -outputs into a list and can thus handle arbitrarily complex return objects. - -```r -res_raw <- get_slurm_out(sjob, outtype = "raw") -res_raw[1:3] -``` - -``` -[[1]] - s_mu s_sd -1.00000506 0.09987899 - -[[2]] - s_mu s_sd -2.0001852 0.2000111 - -[[3]] - s_mu s_sd -3.0002377 0.2998879 -``` - -The files generated by `slurm_apply` are saved in a folder named -*\_rslurm_[jobname]* under the current working directory. - -```r -dir("_rslurm_test_job") -``` - -``` -[1] "params.RData" "results_0.RData" "results_1.RData" "slurm_0.out" -[5] "slurm_1.out" "slurm_run.R" "submit.sh" -``` - -The utility function `cleanup_files` deletes the temporary folder for the -specified *slurm_job*. - - -## Single function evaluation - -In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which -sends a single function call to the cluster. It is analogous in syntax to the -base R function `do.call`, accepting a function and a named list of parameters -as arguments. - -```r -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) -``` - -Because `slurm_call` involves a single process on a single node, it does not -recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the -same additional arguments (detailed in the sections below) as `slurm_apply`. - - -## Adding auxiliary data and functions - -The function passed to `slurm_apply` can only receive atomic parameters stored -within a data frame. Suppose we want instead to apply a function `func` to a list -of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap -`func` in an inline function that takes an integer parameter. - -```r -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) -``` - -The `add_objects` argument specifies the names of any R objects (besides the -parameters data frame) that must be accessed by the function passed to -`slurm_apply`. These objects are saved to a `.RData` file that is loaded -on each cluster node prior to evaluating the function in parallel. - -By default, all R packages attached to the current R session will also be -attached (with `library`) on each cluster node, though this can be modified with -the optional `pkgs` argument. - - -## Configuring SLURM options - -The `slurm_options` argument allows you to set any of the command line -options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the -SLURM `sbatch` command. It should be formatted as a named list, using the long -names of each option (e.g. "time" rather than "t"). Flags, i.e. command line -options that are toggled rather than set to a particular value, should be set to -`TRUE` in `slurm_options`. For example, the following code: -```r -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) -``` -sets the command line options `--time=1:00:00 --share`. - - -## Generating scripts for later submission - -When working from a R session without direct access to the cluster, you can set -`submit = FALSE` within `slurm_apply`. The function will create the -*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without -submitting the job. You may then copy those files to the cluster and submit the -job manually by calling `sbatch submit.sh` from the command line. - - -## How it works / advanced customization - -As mentioned above, the `slurm_apply` function creates a job-specific folder. -This folder contains the parameters data frame and (if applicable) the objects -specified as `add_objects`, both saved in *.RData* files. The function also -generates a R script (`slurm_run.R`) to be run on each cluster node, as well -as a Bash script (`submit.sh`) to submit the job to SLURM. - -More specifically, the Bash script creates a SLURM job array, with each cluster -node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment -variable. This variable is read by `slurm_run.R`, which allows each instance of -the script to operate on a different parameter subset and write its output to -a different results file. The R script calls `parallel::mcMap` to parallelize -calculations on each node. - -Both `slurm_run.R` and `submit.sh` are generated from templates, using the -**whisker** package; these templates can be found in the `rslurm/templates` -subfolder in your R package library. There are two templates for each script, -one for `slurm_apply` and the other (with the word *single* in its title) for -`slurm_call`. - -While you should avoid changing any existing lines in the template scripts, you -may want to add `#SBATCH` lines to the `submit.sh` templates in order to -permanently set certain SLURM command line options and thus customize the package -to your particular cluster setup. \ No newline at end of file From 31bda7f2625e26cb3eeafab491cebcd294691680 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 11:04:51 -0400 Subject: [PATCH 03/25] keep slurm_job jobid as NULL --- DESCRIPTION | 8 ++++---- R/slurm_apply.R | 4 ++-- R/slurm_call.R | 2 +- R/slurm_job.R | 5 +---- man/cancel_slurm.Rd | 1 + man/cleanup_files.Rd | 1 + man/get_slurm_out.Rd | 1 + man/print_job_status.Rd | 1 + man/rslurm-package.Rd | 4 ++-- man/slurm_apply.Rd | 1 + man/slurm_call.Rd | 1 + man/slurm_job.Rd | 1 + vignettes/.build.timestamp | 0 vignettes/overview.Rmd | 2 +- 14 files changed, 18 insertions(+), 14 deletions(-) mode change 100755 => 100644 DESCRIPTION create mode 100644 vignettes/.build.timestamp diff --git a/DESCRIPTION b/DESCRIPTION old mode 100755 new mode 100644 index 511cf96..4add8e9 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,9 @@ Package: rslurm Type: Package Title: Submit R Calculations to a 'SLURM' Cluster -Description: Functions that simplify the R interface the 'SLURM' cluster workload - manager, and automate the process of dividing a parallel calculation across - cluster nodes. +Description: Functions that simplify the R interface the 'SLURM' cluster + workload manager, and automate the process of dividing a parallel calculation + across cluster nodes. Version: 0.3.2 License: GPL-3 URL: https://github.com/SESYNC-ci/rslurm @@ -17,7 +17,7 @@ Depends: Imports: parallel, whisker (>= 0.3) -RoxygenNote: 6.0.1 +RoxygenNote: 5.0.1 Suggests: testthat, knitr, diff --git a/R/slurm_apply.R b/R/slurm_apply.R index 9600c74..1833f25 100755 --- a/R/slurm_apply.R +++ b/R/slurm_apply.R @@ -100,7 +100,7 @@ slurm_apply <- function(f, params, jobname = NA, nodes = 2, cpus_per_node = 2, # Create temp folder tmpdir <- paste0("_rslurm_", jobname) - dir.create(tmpdir) + dir.create(tmpdir, showWarnings = FALSE) saveRDS(params, file = file.path(tmpdir, "params.RDS")) if (!is.null(add_objects)) { @@ -142,7 +142,7 @@ slurm_apply <- function(f, params, jobname = NA, nodes = 2, cpus_per_node = 2, writeLines(script_sh, file.path(tmpdir, "submit.sh")) # Submit job to SLURM if applicable - if (system('squeue', ignore.stdout = TRUE)) { + if (submit && system('squeue', ignore.stdout = TRUE)) { submit <- FALSE cat("Cannot submit; no SLURM workload manager on path\n") } diff --git a/R/slurm_call.R b/R/slurm_call.R index eb2965a..81c1467 100755 --- a/R/slurm_call.R +++ b/R/slurm_call.R @@ -74,7 +74,7 @@ slurm_call <- function(f, params, jobname = NA, add_objects = NULL, # Create temp folder tmpdir <- paste0("_rslurm_", jobname) - dir.create(tmpdir) + dir.create(tmpdir, showWarnings = FALSE) saveRDS(params, file = file.path(tmpdir, "params.RDS")) if (!is.null(add_objects)) { diff --git a/R/slurm_job.R b/R/slurm_job.R index 83ab812..d5a481f 100755 --- a/R/slurm_job.R +++ b/R/slurm_job.R @@ -16,10 +16,7 @@ #' @return A \code{slurm_job} object. #' @export slurm_job <- function(jobname, nodes, jobid) { - slr_job <- list(jobname = jobname, nodes = nodes) - if (!missing(jobid)) { - slr_job$jobid <- as.integer(jobid) - } + slr_job <- list(jobname = jobname, nodes = nodes, jobid = jobid) class(slr_job) <- "slurm_job" slr_job } diff --git a/man/cancel_slurm.Rd b/man/cancel_slurm.Rd index 9be0c23..6d6172b 100755 --- a/man/cancel_slurm.Rd +++ b/man/cancel_slurm.Rd @@ -19,3 +19,4 @@ files. \seealso{ \code{\link{cleanup_files}} } + diff --git a/man/cleanup_files.Rd b/man/cleanup_files.Rd index c170029..534efb7 100755 --- a/man/cleanup_files.Rd +++ b/man/cleanup_files.Rd @@ -29,3 +29,4 @@ cleanup_files(sjob) \seealso{ \code{\link{slurm_apply}}, \code{\link{slurm_call}} } + diff --git a/man/get_slurm_out.Rd b/man/get_slurm_out.Rd index 7376662..3f3eb22 100755 --- a/man/get_slurm_out.Rd +++ b/man/get_slurm_out.Rd @@ -39,3 +39,4 @@ object, and setting \code{outtype = "table"} creates an error in that case. \seealso{ \code{\link{slurm_apply}}, \code{\link{slurm_call}} } + diff --git a/man/print_job_status.Rd b/man/print_job_status.Rd index 797123d..cc9e927 100755 --- a/man/print_job_status.Rd +++ b/man/print_job_status.Rd @@ -22,3 +22,4 @@ the portions of the job still in queue ("PD" in the "ST" column), if any. If all portions of the job have completed or stopped, the function prints the console and error output, if any, generated by each node. } + diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 1db92f3..39298d4 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -2,8 +2,8 @@ % Please edit documentation in R/rslurm.R \docType{package} \name{rslurm-package} -\alias{rslurm-package} \alias{rslurm} +\alias{rslurm-package} \title{rslurm: Submit R calculations to a SLURM cluster} \description{ This package automates the process of sending simple function calls or @@ -57,7 +57,6 @@ function passed to \code{slurm_apply} produces a vector output, you may use \code{outtype = "table"} to collect the output in a single data frame, with one row by function call. } - \examples{ \dontrun{ # Create a data frame of mean/sd values for normal distributions @@ -78,3 +77,4 @@ cleanup_files(sjob1) } } + diff --git a/man/slurm_apply.Rd b/man/slurm_apply.Rd index c942697..e4bceb3 100755 --- a/man/slurm_apply.Rd +++ b/man/slurm_apply.Rd @@ -101,3 +101,4 @@ cleanup_files(sjob) \code{\link{get_slurm_out}} and \code{\link{print_job_status}} which use the output of this function. } + diff --git a/man/slurm_call.Rd b/man/slurm_call.Rd index 9859ab4..7da4296 100755 --- a/man/slurm_call.Rd +++ b/man/slurm_call.Rd @@ -77,3 +77,4 @@ the description of the related functions for more details. \code{\link{get_slurm_out}} and \code{\link{print_job_status}} which use the output of this function. } + diff --git a/man/slurm_job.Rd b/man/slurm_job.Rd index 847c833..737f566 100755 --- a/man/slurm_job.Rd +++ b/man/slurm_job.Rd @@ -28,3 +28,4 @@ In general, \code{slurm_job} objects are created automatically as the output of \code{\link{slurm_apply}} or \code{\link{slurm_call}}, but it may be necessary to manually recreate one if the job was submitted in a different R session. } + diff --git a/vignettes/.build.timestamp b/vignettes/.build.timestamp new file mode 100644 index 0000000..e69de29 diff --git a/vignettes/overview.Rmd b/vignettes/overview.Rmd index 5d8b1f1..67096e3 100644 --- a/vignettes/overview.Rmd +++ b/vignettes/overview.Rmd @@ -81,7 +81,7 @@ execution. These functions are R wrappers for the SLURM command line functions Once the job completes, `get_slurm_out` reads and combines the output from all nodes. ```{r} -res <- get_slurm_out(sjob, outtype = "table", wait = TRUE) +res <- get_slurm_out(sjob, outtype = "table") head(res, 3) ``` From a9c032b2409cbf6ee81bb60b924ed7047c2362f9 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 11:45:15 -0400 Subject: [PATCH 04/25] cleanup docs --- DESCRIPTION | 2 +- NEWS.md | 4 +++- R/slurm_utils.R | 2 ++ cran-comments.md | 12 ++++++++---- man/cancel_slurm.Rd | 1 - man/cleanup_files.Rd | 1 - man/get_slurm_out.Rd | 1 - man/print_job_status.Rd | 1 - man/rslurm-package.Rd | 4 ++-- man/slurm_apply.Rd | 1 - man/slurm_call.Rd | 1 - man/slurm_job.Rd | 1 - tests/testthat/test_slurm_apply.R | 8 ++++---- tests/testthat/test_slurm_call.R | 11 ++++++----- vignettes/.build.timestamp | 0 vignettes/overview.Rmd | 13 +++++++++++++ 16 files changed, 39 insertions(+), 24 deletions(-) delete mode 100644 vignettes/.build.timestamp diff --git a/DESCRIPTION b/DESCRIPTION index 4add8e9..c04336a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -17,7 +17,7 @@ Depends: Imports: parallel, whisker (>= 0.3) -RoxygenNote: 5.0.1 +RoxygenNote: 6.0.1 Suggests: testthat, knitr, diff --git a/NEWS.md b/NEWS.md index ede3710..b6388f8 100755 --- a/NEWS.md +++ b/NEWS.md @@ -3,7 +3,7 @@ rslurm 0.3.2 *2017-04-06* -Minor update to include new feature and bug fix: +Minor update to include new feature and bug fixes: * 'Wait' feature: adds option to slurm_apply and slurm_call to block the calling script until the submitted job completes. This option can be used to @@ -11,6 +11,8 @@ allow immediate processing of a submitted job's output. * Use '.RDS' file extension, rather than '.RData', for serialized objects. +* Minor bug fixes. + rslurm 0.3.1 ============ diff --git a/R/slurm_utils.R b/R/slurm_utils.R index 5c629ca..06c3b22 100755 --- a/R/slurm_utils.R +++ b/R/slurm_utils.R @@ -48,7 +48,9 @@ local_slurm_array <- function(slr_job) { "Sys.setenv(SLURM_ARRAY_TASK_ID = i)", "source('slurm_run.R')", "}"), "local_run.R") system(paste(rscript_path, "--vanilla local_run.R")) + slr_job$jobid = 0L }, finally = setwd(olddir)) + return(slr_job) } # Submit job capturing jobid diff --git a/cran-comments.md b/cran-comments.md index 4eea8cc..936baee 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,11 +1,15 @@ -This is a minor update to the package to add a new feature, update vignette, and fix bugs +This is a minor update to the package to add a new feature, update vignette, and fix bugs. + +The 'overview.Rmd' vignette only works on a SLURM head node. The built vignette in inst/doc should be used instead of any vignette built on CRAN. ## Tested on -win-builder (devel and release) +?win-builder (devel and release) Ubuntu 12.04 with R 3.3 (on travis-ci) -OS X with R 3.3 (local machine) +macOS 10.12 with R 3.3 (local machine) ## R CMD check results -0 errors | 0 warnings | 0 notes +1 error | 0 warnings | 0 notes + +The error results from building the 'overview.Rmd' vignette without a SLURM workload manager. \ No newline at end of file diff --git a/man/cancel_slurm.Rd b/man/cancel_slurm.Rd index 6d6172b..9be0c23 100755 --- a/man/cancel_slurm.Rd +++ b/man/cancel_slurm.Rd @@ -19,4 +19,3 @@ files. \seealso{ \code{\link{cleanup_files}} } - diff --git a/man/cleanup_files.Rd b/man/cleanup_files.Rd index 534efb7..c170029 100755 --- a/man/cleanup_files.Rd +++ b/man/cleanup_files.Rd @@ -29,4 +29,3 @@ cleanup_files(sjob) \seealso{ \code{\link{slurm_apply}}, \code{\link{slurm_call}} } - diff --git a/man/get_slurm_out.Rd b/man/get_slurm_out.Rd index 3f3eb22..7376662 100755 --- a/man/get_slurm_out.Rd +++ b/man/get_slurm_out.Rd @@ -39,4 +39,3 @@ object, and setting \code{outtype = "table"} creates an error in that case. \seealso{ \code{\link{slurm_apply}}, \code{\link{slurm_call}} } - diff --git a/man/print_job_status.Rd b/man/print_job_status.Rd index cc9e927..797123d 100755 --- a/man/print_job_status.Rd +++ b/man/print_job_status.Rd @@ -22,4 +22,3 @@ the portions of the job still in queue ("PD" in the "ST" column), if any. If all portions of the job have completed or stopped, the function prints the console and error output, if any, generated by each node. } - diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 39298d4..1db92f3 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -2,8 +2,8 @@ % Please edit documentation in R/rslurm.R \docType{package} \name{rslurm-package} -\alias{rslurm} \alias{rslurm-package} +\alias{rslurm} \title{rslurm: Submit R calculations to a SLURM cluster} \description{ This package automates the process of sending simple function calls or @@ -57,6 +57,7 @@ function passed to \code{slurm_apply} produces a vector output, you may use \code{outtype = "table"} to collect the output in a single data frame, with one row by function call. } + \examples{ \dontrun{ # Create a data frame of mean/sd values for normal distributions @@ -77,4 +78,3 @@ cleanup_files(sjob1) } } - diff --git a/man/slurm_apply.Rd b/man/slurm_apply.Rd index e4bceb3..c942697 100755 --- a/man/slurm_apply.Rd +++ b/man/slurm_apply.Rd @@ -101,4 +101,3 @@ cleanup_files(sjob) \code{\link{get_slurm_out}} and \code{\link{print_job_status}} which use the output of this function. } - diff --git a/man/slurm_call.Rd b/man/slurm_call.Rd index 7da4296..9859ab4 100755 --- a/man/slurm_call.Rd +++ b/man/slurm_call.Rd @@ -77,4 +77,3 @@ the description of the related functions for more details. \code{\link{get_slurm_out}} and \code{\link{print_job_status}} which use the output of this function. } - diff --git a/man/slurm_job.Rd b/man/slurm_job.Rd index 737f566..847c833 100755 --- a/man/slurm_job.Rd +++ b/man/slurm_job.Rd @@ -28,4 +28,3 @@ In general, \code{slurm_job} objects are created automatically as the output of \code{\link{slurm_apply}} or \code{\link{slurm_call}}, but it may be necessary to manually recreate one if the job was submitted in a different R session. } - diff --git a/tests/testthat/test_slurm_apply.R b/tests/testthat/test_slurm_apply.R index da5dd77..7e917a1 100755 --- a/tests/testthat/test_slurm_apply.R +++ b/tests/testthat/test_slurm_apply.R @@ -22,7 +22,7 @@ msg <- capture.output( sjob1 <- slurm_apply(ftest, pars, jobname = "test1", nodes = 2, cpus_per_node = 1, submit = FALSE) ) -local_slurm_array(sjob1) +sjob1 <- local_slurm_array(sjob1) res <- get_slurm_out(sjob1, "table", wait = FALSE) res_raw <- get_slurm_out(sjob1, "raw", wait = FALSE) test_that("slurm_apply gives correct output", { @@ -38,7 +38,7 @@ msg <- capture.output( sjob2 <- slurm_apply(ftest, pars[, 1, drop = FALSE], jobname = "test2", nodes = 2, cpus_per_node = 1, submit = FALSE) ) -local_slurm_array(sjob2) +sjob2 <- local_slurm_array(sjob2) res <- get_slurm_out(sjob2, "table", wait = FALSE) test_that("slurm_apply works with single parameter", { expect_equal(pars$par_m, res$s_m, tolerance = 0.01) @@ -48,7 +48,7 @@ msg <- capture.output( sjob3 <- slurm_apply(ftest, pars[1, ], nodes = 2, jobname = "test3", cpus_per_node = 1, submit = FALSE) ) -local_slurm_array(sjob3) +sjob3 <- local_slurm_array(sjob3) res <- get_slurm_out(sjob3, "table", wait = FALSE) test_that("slurm_apply works with single row", { expect_equal(sjob3$nodes, 1) @@ -59,7 +59,7 @@ msg <- capture.output( sjob4 <- slurm_apply(ftest, pars[1, 1, drop = FALSE], jobname = "test4", nodes = 2, cpus_per_node = 1, submit = FALSE) ) -local_slurm_array(sjob4) +sjob4 <- local_slurm_array(sjob4) res <- get_slurm_out(sjob4, "table", wait = FALSE) test_that("slurm_apply works with single parameter and single row", { expect_equal(pars$par_m[1], res$s_m, tolerance = 0.01) diff --git a/tests/testthat/test_slurm_call.R b/tests/testthat/test_slurm_call.R index 5c0cd53..0362b6d 100755 --- a/tests/testthat/test_slurm_call.R +++ b/tests/testthat/test_slurm_call.R @@ -14,11 +14,12 @@ test_that("slurm_job name is correctly edited", { expect_equal(sjob$jobname, "test_call") }) -olddir <- getwd() -rscript_path <- file.path(R.home("bin"), "Rscript") -setwd(paste0("_rslurm_", sjob$jobname)) -tryCatch(system(paste(rscript_path, "--vanilla slurm_run.R")), - finally = setwd(olddir)) +sjob <- local_slurm_array(sjob) +# olddir <- getwd() +# rscript_path <- file.path(R.home("bin"), "Rscript") +# setwd(paste0("_rslurm_", sjob$jobname)) +# tryCatch(system(paste(rscript_path, "--vanilla slurm_run.R")), +# finally = setwd(olddir)) res <- get_slurm_out(sjob, wait = FALSE) test_that("slurm_call returns correct output", { diff --git a/vignettes/.build.timestamp b/vignettes/.build.timestamp deleted file mode 100644 index e69de29..0000000 diff --git a/vignettes/overview.Rmd b/vignettes/overview.Rmd index 67096e3..bdc2659 100644 --- a/vignettes/overview.Rmd +++ b/vignettes/overview.Rmd @@ -105,6 +105,10 @@ dir("_rslurm_test_job") The utility function `cleanup_files` deletes the temporary folder for the specified *slurm_job*. +```{r echo=FALSE} +cleanup_files(sjob) +``` + ## Single function evaluation @@ -116,6 +120,9 @@ as arguments. ```{r} sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) ``` +```{r echo=FALSE} +cleanup_files(sjob) +``` Because `slurm_call` involves a single process on a single node, it does not recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the @@ -139,6 +146,9 @@ sjob <- slurm_apply(function(i) func(obj_list[[i]]), add_objects = c("func", "obj_list"), nodes = 2, cpus_per_node = 2) ``` +```{r echo=FALSE} +cleanup_files(sjob) +``` The `add_objects` argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to @@ -162,6 +172,9 @@ options that are toggled rather than set to a particular value, should be set to sjob <- slurm_apply(test_func, pars, slurm_options = list(time = "1:00:00", share = TRUE)) ``` +```{r echo=FALSE} +cleanup_files(sjob) +``` sets the command line options `--time=1:00:00 --share`. From 879f088d52194540f21d48218a18bbf1f952816f Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 12:00:28 -0400 Subject: [PATCH 05/25] no-build-vignettes --- .travis.yml | 2 ++ rslurm.Rproj | 1 + 2 files changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b79e15a..d7fd203 100755 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: r warnings_are_errors: true sudo: required +r_build_args: "--no-build-vignettes" +r_check_args: "--no-vignettes" env: global: diff --git a/rslurm.Rproj b/rslurm.Rproj index 828602d..fdd5a94 100755 --- a/rslurm.Rproj +++ b/rslurm.Rproj @@ -15,4 +15,5 @@ LaTeX: pdfLaTeX BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source +PackageBuildArgs: --no-build-vignettes PackageRoxygenize: rd,collate,namespace From 252611103f0b12eb72625c960bfb6cdfbf28d011 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 12:14:06 -0400 Subject: [PATCH 06/25] unignore inst/doc --- .gitignore | 1 - inst/doc/overview.R | 56 +++++++++++ inst/doc/overview.Rmd | 214 +++++++++++++++++++++++++++++++++++++++++ inst/doc/overview.html | 188 ++++++++++++++++++++++++++++++++++++ 4 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 inst/doc/overview.R create mode 100644 inst/doc/overview.Rmd create mode 100644 inst/doc/overview.html diff --git a/.gitignore b/.gitignore index 09a72cb..807ea25 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ .Rproj.user .Rhistory .RData -inst/doc diff --git a/inst/doc/overview.R b/inst/doc/overview.R new file mode 100644 index 0000000..f2f1994 --- /dev/null +++ b/inst/doc/overview.R @@ -0,0 +1,56 @@ +## ------------------------------------------------------------------------ +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} + +## ------------------------------------------------------------------------ +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) + +## ------------------------------------------------------------------------ +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) + +## ------------------------------------------------------------------------ +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) + +## ------------------------------------------------------------------------ +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] + +## ------------------------------------------------------------------------ +dir("_rslurm_test_job") + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ----echo=FALSE---------------------------------------------------------- +obj_list <- list(NULL) +func <- function(obj) {} + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + diff --git a/inst/doc/overview.Rmd b/inst/doc/overview.Rmd new file mode 100644 index 0000000..bdc2659 --- /dev/null +++ b/inst/doc/overview.Rmd @@ -0,0 +1,214 @@ +--- +title: "Parallelize R code on a SLURM cluster" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{overview} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +Many computing-intensive processes in R involve the repeated evaluation of +a function over many items or parameter sets. These so-called +[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) +calculations can be run serially with the `lapply` or `Map` function, or in parallel +on a single machine with `mclapply` or `mcMap` (from the **parallel** package). + +The rslurm package simplifies the process of distributing this type of calculation +across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) +workload manager. Its main function, `slurm_apply`, automatically divides the +computation over multiple nodes and writes the necessary submission scripts. +It also includes functions to retrieve and combine the output from different nodes, +as well as wrappers for common SLURM commands. + +### Table of contents + +- [Basic example](#basic-example) +- [Single function evaluation](#single-function-evaluation) +- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) +- [Configuring SLURM options](#configuring-slurm-options) +- [Generating scripts for later submission](#generating-scripts-for-later-submission) +- [How it works / advanced customization](#how-it-works-advanced-customization) + + +## Basic example + +To illustrate a typical rslurm workflow, we use a simple function that takes +a mean and standard deviation as parameters, generates a million normal deviates +and returns the sample mean and standard deviation. + +```{r} +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} +``` + +We then create a parameter data frame where each row is a parameter set and each +column matches an argument of the function. + +```{r} +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) +``` + +We can now pass that function and the parameters data frame to `slurm_apply`, +specifiying the number of cluster nodes to use and the number of CPUs per node. +The latter (`cpus_per_node`) determines how many processes will be forked on +each node, as the `mc.cores` argument of `parallel::mcMap`. +```{r} +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) +``` +The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of +information (job name and number of nodes) needed to retrieve the job's output. + +Assuming the function is run on a machine with access to the cluster, it also +prints a message confirming the job has been submitted to SLURM. + +Particular clusters may require the specification of additional SLURM options, +such as time and memory limits for the job. Also, when running R on a local +machine without direct cluster access, you may want to generate scripts to be +copied to the cluster and run at a later time. These topics are covered in +additional sections below this basic example. + +After the job has been submitted, you can call `print_job_status` to display its +status (in queue, running or completed) or call `cancel_slurm` to cancel its +execution. These functions are R wrappers for the SLURM command line functions +`squeue` and `scancel`, respectively. + +Once the job completes, `get_slurm_out` reads and combines the output from all +nodes. +```{r} +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) +``` + +When `outtype = "table"`, the outputs from each function evaluation are +row-bound into a single data frame; this is an appropriate format when the +function returns a simple vector. The default `outtype = "raw"` combines the +outputs into a list and can thus handle arbitrarily complex return objects. + +```{r} +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] +``` + +The files generated by `slurm_apply` are saved in a folder named +*\_rslurm_[jobname]* under the current working directory. + +```{r} +dir("_rslurm_test_job") +``` + +The utility function `cleanup_files` deletes the temporary folder for the +specified *slurm_job*. + +```{r echo=FALSE} +cleanup_files(sjob) +``` + + +## Single function evaluation + +In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which +sends a single function call to the cluster. It is analogous in syntax to the +base R function `do.call`, accepting a function and a named list of parameters +as arguments. + +```{r} +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +Because `slurm_call` involves a single process on a single node, it does not +recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the +same additional arguments (detailed in the sections below) as `slurm_apply`. + + +## Adding auxiliary data and functions + +The function passed to `slurm_apply` can only receive atomic parameters stored +within a data frame. Suppose we want instead to apply a function `func` to a list +of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap +`func` in an inline function that takes an integer parameter. + +```{r echo=FALSE} +obj_list <- list(NULL) +func <- function(obj) {} +``` +```{r} +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +The `add_objects` argument specifies the names of any R objects (besides the +parameters data frame) that must be accessed by the function passed to +`slurm_apply`. These objects are saved to a `.RData` file that is loaded +on each cluster node prior to evaluating the function in parallel. + +By default, all R packages attached to the current R session will also be +attached (with `library`) on each cluster node, though this can be modified with +the optional `pkgs` argument. + + +## Configuring SLURM options + +The `slurm_options` argument allows you to set any of the command line +options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the +SLURM `sbatch` command. It should be formatted as a named list, using the long +names of each option (e.g. "time" rather than "t"). Flags, i.e. command line +options that are toggled rather than set to a particular value, should be set to +`TRUE` in `slurm_options`. For example, the following code: +```{r} +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` +sets the command line options `--time=1:00:00 --share`. + + +## Generating scripts for later submission + +When working from a R session without direct access to the cluster, you can set +`submit = FALSE` within `slurm_apply`. The function will create the +*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without +submitting the job. You may then copy those files to the cluster and submit the +job manually by calling `sbatch submit.sh` from the command line. + + +## How it works / advanced customization + +As mentioned above, the `slurm_apply` function creates a job-specific folder. +This folder contains the parameters as a *.RDS* file and (if applicable) the objects +specified as `add_objects` saved together in a *.RData* file. The function also +generates a R script (`slurm_run.R`) to be run on each cluster node, as well +as a Bash script (`submit.sh`) to submit the job to SLURM. + +More specifically, the Bash script creates a SLURM job array, with each cluster +node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment +variable. This variable is read by `slurm_run.R`, which allows each instance of +the script to operate on a different parameter subset and write its output to +a different results file. The R script calls `parallel::mcMap` to parallelize +calculations on each node. + +Both `slurm_run.R` and `submit.sh` are generated from templates, using the +**whisker** package; these templates can be found in the `rslurm/templates` +subfolder in your R package library. There are two templates for each script, +one for `slurm_apply` and the other (with the word *single* in its title) for +`slurm_call`. + +While you should avoid changing any existing lines in the template scripts, you +may want to add `#SBATCH` lines to the `submit.sh` templates in order to +permanently set certain SLURM command line options and thus customize the package +to your particular cluster setup. \ No newline at end of file diff --git a/inst/doc/overview.html b/inst/doc/overview.html new file mode 100644 index 0000000..4b93513 --- /dev/null +++ b/inst/doc/overview.html @@ -0,0 +1,188 @@ +<!DOCTYPE html> + +<html xmlns="http://www.w3.org/1999/xhtml"> + +<head> + +<meta charset="utf-8"> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="pandoc" /> + +<meta name="viewport" content="width=device-width, initial-scale=1"> + + + +<title>Parallelize R code on a SLURM cluster</title> + + + +<style type="text/css">code{white-space: pre;}</style> +<style type="text/css"> +div.sourceCode { overflow-x: auto; } +table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode { + margin: 0; padding: 0; vertical-align: baseline; border: none; } +table.sourceCode { width: 100%; line-height: 100%; } +td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; } +td.sourceCode { padding-left: 5px; } +code > span.kw { color: #007020; font-weight: bold; } /* Keyword */ +code > span.dt { color: #902000; } /* DataType */ +code > span.dv { color: #40a070; } /* DecVal */ +code > span.bn { color: #40a070; } /* BaseN */ +code > span.fl { color: #40a070; } /* Float */ +code > span.ch { color: #4070a0; } /* Char */ +code > span.st { color: #4070a0; } /* String */ +code > span.co { color: #60a0b0; font-style: italic; } /* Comment */ +code > span.ot { color: #007020; } /* Other */ +code > span.al { color: #ff0000; font-weight: bold; } /* Alert */ +code > span.fu { color: #06287e; } /* Function */ +code > span.er { color: #ff0000; font-weight: bold; } /* Error */ +code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */ +code > span.cn { color: #880000; } /* Constant */ +code > span.sc { color: #4070a0; } /* SpecialChar */ +code > span.vs { color: #4070a0; } /* VerbatimString */ +code > span.ss { color: #bb6688; } /* SpecialString */ +code > span.im { } /* Import */ +code > span.va { color: #19177c; } /* Variable */ +code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */ +code > span.op { color: #666666; } /* Operator */ +code > span.bu { } /* BuiltIn */ +code > span.ex { } /* Extension */ +code > span.pp { color: #bc7a00; } /* Preprocessor */ +code > span.at { color: #7d9029; } /* Attribute */ +code > span.do { color: #ba2121; font-style: italic; } /* Documentation */ +code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */ +code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */ +code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */ +</style> + + + +<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" /> + +</head> + +<body> + + + + +<h1 class="title toc-ignore">Parallelize R code on a SLURM cluster</h1> + + + +<p>Many computing-intensive processes in R involve the repeated evaluation of a function over many items or parameter sets. These so-called <a href="https://en.wikipedia.org/wiki/Embarrassingly_parallel">embarrassingly parallel</a> calculations can be run serially with the <code>lapply</code> or <code>Map</code> function, or in parallel on a single machine with <code>mclapply</code> or <code>mcMap</code> (from the <strong>parallel</strong> package).</p> +<p>The rslurm package simplifies the process of distributing this type of calculation across a computing cluster that uses the <a href="http://slurm.schedmd.com/">SLURM</a> workload manager. Its main function, <code>slurm_apply</code>, automatically divides the computation over multiple nodes and writes the necessary submission scripts. It also includes functions to retrieve and combine the output from different nodes, as well as wrappers for common SLURM commands.</p> +<div id="table-of-contents" class="section level3"> +<h3>Table of contents</h3> +<ul> +<li><a href="#basic-example">Basic example</a></li> +<li><a href="#single-function-evaluation">Single function evaluation</a></li> +<li><a href="#adding-auxiliary-data-and-functions">Adding auxiliary data and functions</a></li> +<li><a href="#configuring-slurm-options">Configuring SLURM options</a></li> +<li><a href="#generating-scripts-for-later-submission">Generating scripts for later submission</a></li> +<li><a href="#how-it-works-advanced-customization">How it works / advanced customization</a></li> +</ul> +</div> +<div id="basic-example" class="section level2"> +<h2>Basic example</h2> +<p>To illustrate a typical rslurm workflow, we use a simple function that takes a mean and standard deviation as parameters, generates a million normal deviates and returns the sample mean and standard deviation.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">test_func <-<span class="st"> </span>function(par_mu, par_sd) { + samp <-<span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">10</span>^<span class="dv">6</span>, par_mu, par_sd) + <span class="kw">c</span>(<span class="dt">s_mu =</span> <span class="kw">mean</span>(samp), <span class="dt">s_sd =</span> <span class="kw">sd</span>(samp)) +}</code></pre></div> +<p>We then create a parameter data frame where each row is a parameter set and each column matches an argument of the function.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">pars <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">par_mu =</span> <span class="dv">1</span>:<span class="dv">10</span>, + <span class="dt">par_sd =</span> <span class="kw">seq</span>(<span class="fl">0.1</span>, <span class="dv">1</span>, <span class="dt">length.out =</span> <span class="dv">10</span>)) +<span class="kw">head</span>(pars, <span class="dv">3</span>)</code></pre></div> +<pre><code>## par_mu par_sd +## 1 1 0.1 +## 2 2 0.2 +## 3 3 0.3</code></pre> +<p>We can now pass that function and the parameters data frame to <code>slurm_apply</code>, specifiying the number of cluster nodes to use and the number of CPUs per node. The latter (<code>cpus_per_node</code>) determines how many processes will be forked on each node, as the <code>mc.cores</code> argument of <code>parallel::mcMap</code>.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) +sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 550970</code></pre> +<p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> +<p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> +<p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> +<p>After the job has been submitted, you can call <code>print_job_status</code> to display its status (in queue, running or completed) or call <code>cancel_slurm</code> to cancel its execution. These functions are R wrappers for the SLURM command line functions <code>squeue</code> and <code>scancel</code>, respectively.</p> +<p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) +<span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> +<pre><code>## s_mu s_sd +## 1 0.9999736 0.09993745 +## 2 2.0000803 0.19982385 +## 3 2.9994306 0.30002325</code></pre> +<p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) +res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> +<pre><code>## [[1]] +## s_mu s_sd +## 0.99997359 0.09993745 +## +## [[2]] +## s_mu s_sd +## 2.0000803 0.1998238 +## +## [[3]] +## s_mu s_sd +## 2.9994306 0.3000233</code></pre> +<p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> +<pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" +## [5] "slurm_1.out" "slurm_run.R" "submit.sh"</code></pre> +<p>The utility function <code>cleanup_files</code> deletes the temporary folder for the specified <em>slurm_job</em>.</p> +</div> +<div id="single-function-evaluation" class="section level2"> +<h2>Single function evaluation</h2> +<p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> +<pre><code>## Submitted batch job 550974</code></pre> +<p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> +</div> +<div id="adding-auxiliary-data-and-functions" class="section level2"> +<h2>Adding auxiliary data and functions</h2> +<p>The function passed to <code>slurm_apply</code> can only receive atomic parameters stored within a data frame. Suppose we want instead to apply a function <code>func</code> to a list of complex R objects, <code>obj_list</code>. To use <code>slurm_apply</code> in this case, we can wrap <code>func</code> in an inline function that takes an integer parameter.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(function(i) <span class="kw">func</span>(obj_list[[i]]), + <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), + <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 550976</code></pre> +<p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> +<p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> +</div> +<div id="configuring-slurm-options" class="section level2"> +<h2>Configuring SLURM options</h2> +<p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, + <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> +<pre><code>## Submitted batch job 550978</code></pre> +<p>sets the command line options <code>--time=1:00:00 --share</code>.</p> +</div> +<div id="generating-scripts-for-later-submission" class="section level2"> +<h2>Generating scripts for later submission</h2> +<p>When working from a R session without direct access to the cluster, you can set <code>submit = FALSE</code> within <code>slurm_apply</code>. The function will create the <em>_rslurm_[jobname]</em> folder and generate the scripts and .RData files, without submitting the job. You may then copy those files to the cluster and submit the job manually by calling <code>sbatch submit.sh</code> from the command line.</p> +</div> +<div id="how-it-works-advanced-customization" class="section level2"> +<h2>How it works / advanced customization</h2> +<p>As mentioned above, the <code>slurm_apply</code> function creates a job-specific folder. This folder contains the parameters as a <em>.RDS</em> file and (if applicable) the objects specified as <code>add_objects</code> saved together in a <em>.RData</em> file. The function also generates a R script (<code>slurm_run.R</code>) to be run on each cluster node, as well as a Bash script (<code>submit.sh</code>) to submit the job to SLURM.</p> +<p>More specifically, the Bash script creates a SLURM job array, with each cluster node receiving a different value of the <em>SLURM_ARRAY_TASK_ID</em> environment variable. This variable is read by <code>slurm_run.R</code>, which allows each instance of the script to operate on a different parameter subset and write its output to a different results file. The R script calls <code>parallel::mcMap</code> to parallelize calculations on each node.</p> +<p>Both <code>slurm_run.R</code> and <code>submit.sh</code> are generated from templates, using the <strong>whisker</strong> package; these templates can be found in the <code>rslurm/templates</code> subfolder in your R package library. There are two templates for each script, one for <code>slurm_apply</code> and the other (with the word <em>single</em> in its title) for <code>slurm_call</code>.</p> +<p>While you should avoid changing any existing lines in the template scripts, you may want to add <code>#SBATCH</code> lines to the <code>submit.sh</code> templates in order to permanently set certain SLURM command line options and thus customize the package to your particular cluster setup.</p> +</div> + + + +<!-- dynamically load mathjax for compatibility with self-contained --> +<script> + (function () { + var script = document.createElement("script"); + script.type = "text/javascript"; + script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; + document.getElementsByTagName("head")[0].appendChild(script); + })(); +</script> + +</body> +</html> From 2911c6d25c54edd0d602c554d7919eb3abc61321 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 12:27:45 -0400 Subject: [PATCH 07/25] test allow check vignette --- .travis.yml | 1 - inst/doc/overview.html | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7fd203..9699bf9 100755 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: r warnings_are_errors: true sudo: required r_build_args: "--no-build-vignettes" -r_check_args: "--no-vignettes" env: global: diff --git a/inst/doc/overview.html b/inst/doc/overview.html index 4b93513..3afdbb5 100644 --- a/inst/doc/overview.html +++ b/inst/doc/overview.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 550970</code></pre> +<pre><code>## Submitted batch job 550992</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9999736 0.09993745 -## 2 2.0000803 0.19982385 -## 3 2.9994306 0.30002325</code></pre> +<pre><code>## s_mu s_sd +## 1 1.000042 0.1000799 +## 2 2.000120 0.2002692 +## 3 2.999651 0.2999178</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] -## s_mu s_sd -## 0.99997359 0.09993745 +## s_mu s_sd +## 1.0000423 0.1000799 ## ## [[2]] ## s_mu s_sd -## 2.0000803 0.1998238 +## 2.0001201 0.2002692 ## ## [[3]] ## s_mu s_sd -## 2.9994306 0.3000233</code></pre> +## 2.9996509 0.2999178</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 550974</code></pre> +<pre><code>## Submitted batch job 550996</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 550976</code></pre> +<pre><code>## Submitted batch job 550998</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 550978</code></pre> +<pre><code>## Submitted batch job 551000</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> From b6549f363b3771e185ac8d676e11fe4421e4f757 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 14:09:09 -0400 Subject: [PATCH 08/25] attempting a "clean" check workflow --- .travis.yml | 1 + cran-comments.md | 17 ++-- inst/doc/overview.R | 56 ----------- inst/doc/overview.Rmd | 214 ----------------------------------------- inst/doc/overview.html | 188 ------------------------------------ rslurm.Rproj | 1 - 6 files changed, 9 insertions(+), 468 deletions(-) delete mode 100644 inst/doc/overview.R delete mode 100644 inst/doc/overview.Rmd delete mode 100644 inst/doc/overview.html diff --git a/.travis.yml b/.travis.yml index 9699bf9..d7fd203 100755 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: r warnings_are_errors: true sudo: required r_build_args: "--no-build-vignettes" +r_check_args: "--no-vignettes" env: global: diff --git a/cran-comments.md b/cran-comments.md index 936baee..8a6fd84 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,15 +1,14 @@ -This is a minor update to the package to add a new feature, update vignette, and fix bugs. +This is a minor update to the package to add a new feature, update the vignette, and fix bugs. -The 'overview.Rmd' vignette only works on a SLURM head node. The built vignette in inst/doc should be used instead of any vignette built on CRAN. +The 'overview.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. ## Tested on -?win-builder (devel and release) -Ubuntu 12.04 with R 3.3 (on travis-ci) -macOS 10.12 with R 3.3 (local machine) +Ubuntu 14.04 with R 3.3 (rstudio.sesync.org, SLURM head node) +Ubuntu 12.04 with R 3.3 (travis-ci) +macOS 10.12 with R 3.3 (local machine, R CMD build with '--no-build-vignettes', R CMD check with '--no-vignettes') +?win-builder (release and devel, after manual removal of 'overview.Rmd' vignette) -## R CMD check results +## R CMD check -1 error | 0 warnings | 0 notes - -The error results from building the 'overview.Rmd' vignette without a SLURM workload manager. \ No newline at end of file +0 errors | 0 warnings | 0 notes \ No newline at end of file diff --git a/inst/doc/overview.R b/inst/doc/overview.R deleted file mode 100644 index f2f1994..0000000 --- a/inst/doc/overview.R +++ /dev/null @@ -1,56 +0,0 @@ -## ------------------------------------------------------------------------ -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} - -## ------------------------------------------------------------------------ -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) - -## ------------------------------------------------------------------------ -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) - -## ------------------------------------------------------------------------ -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) - -## ------------------------------------------------------------------------ -res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) -res_raw[1:3] - -## ------------------------------------------------------------------------ -dir("_rslurm_test_job") - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ------------------------------------------------------------------------ -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ----echo=FALSE---------------------------------------------------------- -obj_list <- list(NULL) -func <- function(obj) {} - -## ------------------------------------------------------------------------ -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ------------------------------------------------------------------------ -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - diff --git a/inst/doc/overview.Rmd b/inst/doc/overview.Rmd deleted file mode 100644 index bdc2659..0000000 --- a/inst/doc/overview.Rmd +++ /dev/null @@ -1,214 +0,0 @@ ---- -title: "Parallelize R code on a SLURM cluster" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{overview} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -Many computing-intensive processes in R involve the repeated evaluation of -a function over many items or parameter sets. These so-called -[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) -calculations can be run serially with the `lapply` or `Map` function, or in parallel -on a single machine with `mclapply` or `mcMap` (from the **parallel** package). - -The rslurm package simplifies the process of distributing this type of calculation -across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) -workload manager. Its main function, `slurm_apply`, automatically divides the -computation over multiple nodes and writes the necessary submission scripts. -It also includes functions to retrieve and combine the output from different nodes, -as well as wrappers for common SLURM commands. - -### Table of contents - -- [Basic example](#basic-example) -- [Single function evaluation](#single-function-evaluation) -- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) -- [Configuring SLURM options](#configuring-slurm-options) -- [Generating scripts for later submission](#generating-scripts-for-later-submission) -- [How it works / advanced customization](#how-it-works-advanced-customization) - - -## Basic example - -To illustrate a typical rslurm workflow, we use a simple function that takes -a mean and standard deviation as parameters, generates a million normal deviates -and returns the sample mean and standard deviation. - -```{r} -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} -``` - -We then create a parameter data frame where each row is a parameter set and each -column matches an argument of the function. - -```{r} -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) -``` - -We can now pass that function and the parameters data frame to `slurm_apply`, -specifiying the number of cluster nodes to use and the number of CPUs per node. -The latter (`cpus_per_node`) determines how many processes will be forked on -each node, as the `mc.cores` argument of `parallel::mcMap`. -```{r} -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) -``` -The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of -information (job name and number of nodes) needed to retrieve the job's output. - -Assuming the function is run on a machine with access to the cluster, it also -prints a message confirming the job has been submitted to SLURM. - -Particular clusters may require the specification of additional SLURM options, -such as time and memory limits for the job. Also, when running R on a local -machine without direct cluster access, you may want to generate scripts to be -copied to the cluster and run at a later time. These topics are covered in -additional sections below this basic example. - -After the job has been submitted, you can call `print_job_status` to display its -status (in queue, running or completed) or call `cancel_slurm` to cancel its -execution. These functions are R wrappers for the SLURM command line functions -`squeue` and `scancel`, respectively. - -Once the job completes, `get_slurm_out` reads and combines the output from all -nodes. -```{r} -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) -``` - -When `outtype = "table"`, the outputs from each function evaluation are -row-bound into a single data frame; this is an appropriate format when the -function returns a simple vector. The default `outtype = "raw"` combines the -outputs into a list and can thus handle arbitrarily complex return objects. - -```{r} -res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) -res_raw[1:3] -``` - -The files generated by `slurm_apply` are saved in a folder named -*\_rslurm_[jobname]* under the current working directory. - -```{r} -dir("_rslurm_test_job") -``` - -The utility function `cleanup_files` deletes the temporary folder for the -specified *slurm_job*. - -```{r echo=FALSE} -cleanup_files(sjob) -``` - - -## Single function evaluation - -In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which -sends a single function call to the cluster. It is analogous in syntax to the -base R function `do.call`, accepting a function and a named list of parameters -as arguments. - -```{r} -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` - -Because `slurm_call` involves a single process on a single node, it does not -recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the -same additional arguments (detailed in the sections below) as `slurm_apply`. - - -## Adding auxiliary data and functions - -The function passed to `slurm_apply` can only receive atomic parameters stored -within a data frame. Suppose we want instead to apply a function `func` to a list -of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap -`func` in an inline function that takes an integer parameter. - -```{r echo=FALSE} -obj_list <- list(NULL) -func <- function(obj) {} -``` -```{r} -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` - -The `add_objects` argument specifies the names of any R objects (besides the -parameters data frame) that must be accessed by the function passed to -`slurm_apply`. These objects are saved to a `.RData` file that is loaded -on each cluster node prior to evaluating the function in parallel. - -By default, all R packages attached to the current R session will also be -attached (with `library`) on each cluster node, though this can be modified with -the optional `pkgs` argument. - - -## Configuring SLURM options - -The `slurm_options` argument allows you to set any of the command line -options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the -SLURM `sbatch` command. It should be formatted as a named list, using the long -names of each option (e.g. "time" rather than "t"). Flags, i.e. command line -options that are toggled rather than set to a particular value, should be set to -`TRUE` in `slurm_options`. For example, the following code: -```{r} -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` -sets the command line options `--time=1:00:00 --share`. - - -## Generating scripts for later submission - -When working from a R session without direct access to the cluster, you can set -`submit = FALSE` within `slurm_apply`. The function will create the -*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without -submitting the job. You may then copy those files to the cluster and submit the -job manually by calling `sbatch submit.sh` from the command line. - - -## How it works / advanced customization - -As mentioned above, the `slurm_apply` function creates a job-specific folder. -This folder contains the parameters as a *.RDS* file and (if applicable) the objects -specified as `add_objects` saved together in a *.RData* file. The function also -generates a R script (`slurm_run.R`) to be run on each cluster node, as well -as a Bash script (`submit.sh`) to submit the job to SLURM. - -More specifically, the Bash script creates a SLURM job array, with each cluster -node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment -variable. This variable is read by `slurm_run.R`, which allows each instance of -the script to operate on a different parameter subset and write its output to -a different results file. The R script calls `parallel::mcMap` to parallelize -calculations on each node. - -Both `slurm_run.R` and `submit.sh` are generated from templates, using the -**whisker** package; these templates can be found in the `rslurm/templates` -subfolder in your R package library. There are two templates for each script, -one for `slurm_apply` and the other (with the word *single* in its title) for -`slurm_call`. - -While you should avoid changing any existing lines in the template scripts, you -may want to add `#SBATCH` lines to the `submit.sh` templates in order to -permanently set certain SLURM command line options and thus customize the package -to your particular cluster setup. \ No newline at end of file diff --git a/inst/doc/overview.html b/inst/doc/overview.html deleted file mode 100644 index 3afdbb5..0000000 --- a/inst/doc/overview.html +++ /dev/null @@ -1,188 +0,0 @@ -<!DOCTYPE html> - -<html xmlns="http://www.w3.org/1999/xhtml"> - -<head> - -<meta charset="utf-8"> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<meta name="generator" content="pandoc" /> - -<meta name="viewport" content="width=device-width, initial-scale=1"> - - - -<title>Parallelize R code on a SLURM cluster</title> - - - -<style type="text/css">code{white-space: pre;}</style> -<style type="text/css"> -div.sourceCode { overflow-x: auto; } -table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode { - margin: 0; padding: 0; vertical-align: baseline; border: none; } -table.sourceCode { width: 100%; line-height: 100%; } -td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; } -td.sourceCode { padding-left: 5px; } -code > span.kw { color: #007020; font-weight: bold; } /* Keyword */ -code > span.dt { color: #902000; } /* DataType */ -code > span.dv { color: #40a070; } /* DecVal */ -code > span.bn { color: #40a070; } /* BaseN */ -code > span.fl { color: #40a070; } /* Float */ -code > span.ch { color: #4070a0; } /* Char */ -code > span.st { color: #4070a0; } /* String */ -code > span.co { color: #60a0b0; font-style: italic; } /* Comment */ -code > span.ot { color: #007020; } /* Other */ -code > span.al { color: #ff0000; font-weight: bold; } /* Alert */ -code > span.fu { color: #06287e; } /* Function */ -code > span.er { color: #ff0000; font-weight: bold; } /* Error */ -code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */ -code > span.cn { color: #880000; } /* Constant */ -code > span.sc { color: #4070a0; } /* SpecialChar */ -code > span.vs { color: #4070a0; } /* VerbatimString */ -code > span.ss { color: #bb6688; } /* SpecialString */ -code > span.im { } /* Import */ -code > span.va { color: #19177c; } /* Variable */ -code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */ -code > span.op { color: #666666; } /* Operator */ -code > span.bu { } /* BuiltIn */ -code > span.ex { } /* Extension */ -code > span.pp { color: #bc7a00; } /* Preprocessor */ -code > span.at { color: #7d9029; } /* Attribute */ -code > span.do { color: #ba2121; font-style: italic; } /* Documentation */ -code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */ -code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */ -code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */ -</style> - - - -<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" /> - -</head> - -<body> - - - - -<h1 class="title toc-ignore">Parallelize R code on a SLURM cluster</h1> - - - -<p>Many computing-intensive processes in R involve the repeated evaluation of a function over many items or parameter sets. These so-called <a href="https://en.wikipedia.org/wiki/Embarrassingly_parallel">embarrassingly parallel</a> calculations can be run serially with the <code>lapply</code> or <code>Map</code> function, or in parallel on a single machine with <code>mclapply</code> or <code>mcMap</code> (from the <strong>parallel</strong> package).</p> -<p>The rslurm package simplifies the process of distributing this type of calculation across a computing cluster that uses the <a href="http://slurm.schedmd.com/">SLURM</a> workload manager. Its main function, <code>slurm_apply</code>, automatically divides the computation over multiple nodes and writes the necessary submission scripts. It also includes functions to retrieve and combine the output from different nodes, as well as wrappers for common SLURM commands.</p> -<div id="table-of-contents" class="section level3"> -<h3>Table of contents</h3> -<ul> -<li><a href="#basic-example">Basic example</a></li> -<li><a href="#single-function-evaluation">Single function evaluation</a></li> -<li><a href="#adding-auxiliary-data-and-functions">Adding auxiliary data and functions</a></li> -<li><a href="#configuring-slurm-options">Configuring SLURM options</a></li> -<li><a href="#generating-scripts-for-later-submission">Generating scripts for later submission</a></li> -<li><a href="#how-it-works-advanced-customization">How it works / advanced customization</a></li> -</ul> -</div> -<div id="basic-example" class="section level2"> -<h2>Basic example</h2> -<p>To illustrate a typical rslurm workflow, we use a simple function that takes a mean and standard deviation as parameters, generates a million normal deviates and returns the sample mean and standard deviation.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">test_func <-<span class="st"> </span>function(par_mu, par_sd) { - samp <-<span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">10</span>^<span class="dv">6</span>, par_mu, par_sd) - <span class="kw">c</span>(<span class="dt">s_mu =</span> <span class="kw">mean</span>(samp), <span class="dt">s_sd =</span> <span class="kw">sd</span>(samp)) -}</code></pre></div> -<p>We then create a parameter data frame where each row is a parameter set and each column matches an argument of the function.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">pars <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">par_mu =</span> <span class="dv">1</span>:<span class="dv">10</span>, - <span class="dt">par_sd =</span> <span class="kw">seq</span>(<span class="fl">0.1</span>, <span class="dv">1</span>, <span class="dt">length.out =</span> <span class="dv">10</span>)) -<span class="kw">head</span>(pars, <span class="dv">3</span>)</code></pre></div> -<pre><code>## par_mu par_sd -## 1 1 0.1 -## 2 2 0.2 -## 3 3 0.3</code></pre> -<p>We can now pass that function and the parameters data frame to <code>slurm_apply</code>, specifiying the number of cluster nodes to use and the number of CPUs per node. The latter (<code>cpus_per_node</code>) determines how many processes will be forked on each node, as the <code>mc.cores</code> argument of <code>parallel::mcMap</code>.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) -sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, - <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 550992</code></pre> -<p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> -<p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> -<p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> -<p>After the job has been submitted, you can call <code>print_job_status</code> to display its status (in queue, running or completed) or call <code>cancel_slurm</code> to cancel its execution. These functions are R wrappers for the SLURM command line functions <code>squeue</code> and <code>scancel</code>, respectively.</p> -<p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) -<span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 1.000042 0.1000799 -## 2 2.000120 0.2002692 -## 3 2.999651 0.2999178</code></pre> -<p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) -res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> -<pre><code>## [[1]] -## s_mu s_sd -## 1.0000423 0.1000799 -## -## [[2]] -## s_mu s_sd -## 2.0001201 0.2002692 -## -## [[3]] -## s_mu s_sd -## 2.9996509 0.2999178</code></pre> -<p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> -<pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" -## [5] "slurm_1.out" "slurm_run.R" "submit.sh"</code></pre> -<p>The utility function <code>cleanup_files</code> deletes the temporary folder for the specified <em>slurm_job</em>.</p> -</div> -<div id="single-function-evaluation" class="section level2"> -<h2>Single function evaluation</h2> -<p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 550996</code></pre> -<p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> -</div> -<div id="adding-auxiliary-data-and-functions" class="section level2"> -<h2>Adding auxiliary data and functions</h2> -<p>The function passed to <code>slurm_apply</code> can only receive atomic parameters stored within a data frame. Suppose we want instead to apply a function <code>func</code> to a list of complex R objects, <code>obj_list</code>. To use <code>slurm_apply</code> in this case, we can wrap <code>func</code> in an inline function that takes an integer parameter.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(function(i) <span class="kw">func</span>(obj_list[[i]]), - <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), - <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), - <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 550998</code></pre> -<p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> -<p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> -</div> -<div id="configuring-slurm-options" class="section level2"> -<h2>Configuring SLURM options</h2> -<p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, - <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551000</code></pre> -<p>sets the command line options <code>--time=1:00:00 --share</code>.</p> -</div> -<div id="generating-scripts-for-later-submission" class="section level2"> -<h2>Generating scripts for later submission</h2> -<p>When working from a R session without direct access to the cluster, you can set <code>submit = FALSE</code> within <code>slurm_apply</code>. The function will create the <em>_rslurm_[jobname]</em> folder and generate the scripts and .RData files, without submitting the job. You may then copy those files to the cluster and submit the job manually by calling <code>sbatch submit.sh</code> from the command line.</p> -</div> -<div id="how-it-works-advanced-customization" class="section level2"> -<h2>How it works / advanced customization</h2> -<p>As mentioned above, the <code>slurm_apply</code> function creates a job-specific folder. This folder contains the parameters as a <em>.RDS</em> file and (if applicable) the objects specified as <code>add_objects</code> saved together in a <em>.RData</em> file. The function also generates a R script (<code>slurm_run.R</code>) to be run on each cluster node, as well as a Bash script (<code>submit.sh</code>) to submit the job to SLURM.</p> -<p>More specifically, the Bash script creates a SLURM job array, with each cluster node receiving a different value of the <em>SLURM_ARRAY_TASK_ID</em> environment variable. This variable is read by <code>slurm_run.R</code>, which allows each instance of the script to operate on a different parameter subset and write its output to a different results file. The R script calls <code>parallel::mcMap</code> to parallelize calculations on each node.</p> -<p>Both <code>slurm_run.R</code> and <code>submit.sh</code> are generated from templates, using the <strong>whisker</strong> package; these templates can be found in the <code>rslurm/templates</code> subfolder in your R package library. There are two templates for each script, one for <code>slurm_apply</code> and the other (with the word <em>single</em> in its title) for <code>slurm_call</code>.</p> -<p>While you should avoid changing any existing lines in the template scripts, you may want to add <code>#SBATCH</code> lines to the <code>submit.sh</code> templates in order to permanently set certain SLURM command line options and thus customize the package to your particular cluster setup.</p> -</div> - - - -<!-- dynamically load mathjax for compatibility with self-contained --> -<script> - (function () { - var script = document.createElement("script"); - script.type = "text/javascript"; - script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; - document.getElementsByTagName("head")[0].appendChild(script); - })(); -</script> - -</body> -</html> diff --git a/rslurm.Rproj b/rslurm.Rproj index fdd5a94..828602d 100755 --- a/rslurm.Rproj +++ b/rslurm.Rproj @@ -15,5 +15,4 @@ LaTeX: pdfLaTeX BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source -PackageBuildArgs: --no-build-vignettes PackageRoxygenize: rd,collate,namespace From 33cf107dcc0e385bb7e31726572fcd2e812131f6 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 14:30:29 -0400 Subject: [PATCH 09/25] include build vignette --- inst/doc/overview.R | 56 +++++++++++ inst/doc/overview.Rmd | 214 +++++++++++++++++++++++++++++++++++++++++ inst/doc/overview.html | 188 ++++++++++++++++++++++++++++++++++++ 3 files changed, 458 insertions(+) create mode 100644 inst/doc/overview.R create mode 100644 inst/doc/overview.Rmd create mode 100644 inst/doc/overview.html diff --git a/inst/doc/overview.R b/inst/doc/overview.R new file mode 100644 index 0000000..f2f1994 --- /dev/null +++ b/inst/doc/overview.R @@ -0,0 +1,56 @@ +## ------------------------------------------------------------------------ +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} + +## ------------------------------------------------------------------------ +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) + +## ------------------------------------------------------------------------ +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) + +## ------------------------------------------------------------------------ +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) + +## ------------------------------------------------------------------------ +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] + +## ------------------------------------------------------------------------ +dir("_rslurm_test_job") + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ----echo=FALSE---------------------------------------------------------- +obj_list <- list(NULL) +func <- function(obj) {} + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + diff --git a/inst/doc/overview.Rmd b/inst/doc/overview.Rmd new file mode 100644 index 0000000..bdc2659 --- /dev/null +++ b/inst/doc/overview.Rmd @@ -0,0 +1,214 @@ +--- +title: "Parallelize R code on a SLURM cluster" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{overview} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +Many computing-intensive processes in R involve the repeated evaluation of +a function over many items or parameter sets. These so-called +[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) +calculations can be run serially with the `lapply` or `Map` function, or in parallel +on a single machine with `mclapply` or `mcMap` (from the **parallel** package). + +The rslurm package simplifies the process of distributing this type of calculation +across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) +workload manager. Its main function, `slurm_apply`, automatically divides the +computation over multiple nodes and writes the necessary submission scripts. +It also includes functions to retrieve and combine the output from different nodes, +as well as wrappers for common SLURM commands. + +### Table of contents + +- [Basic example](#basic-example) +- [Single function evaluation](#single-function-evaluation) +- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) +- [Configuring SLURM options](#configuring-slurm-options) +- [Generating scripts for later submission](#generating-scripts-for-later-submission) +- [How it works / advanced customization](#how-it-works-advanced-customization) + + +## Basic example + +To illustrate a typical rslurm workflow, we use a simple function that takes +a mean and standard deviation as parameters, generates a million normal deviates +and returns the sample mean and standard deviation. + +```{r} +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} +``` + +We then create a parameter data frame where each row is a parameter set and each +column matches an argument of the function. + +```{r} +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) +``` + +We can now pass that function and the parameters data frame to `slurm_apply`, +specifiying the number of cluster nodes to use and the number of CPUs per node. +The latter (`cpus_per_node`) determines how many processes will be forked on +each node, as the `mc.cores` argument of `parallel::mcMap`. +```{r} +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) +``` +The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of +information (job name and number of nodes) needed to retrieve the job's output. + +Assuming the function is run on a machine with access to the cluster, it also +prints a message confirming the job has been submitted to SLURM. + +Particular clusters may require the specification of additional SLURM options, +such as time and memory limits for the job. Also, when running R on a local +machine without direct cluster access, you may want to generate scripts to be +copied to the cluster and run at a later time. These topics are covered in +additional sections below this basic example. + +After the job has been submitted, you can call `print_job_status` to display its +status (in queue, running or completed) or call `cancel_slurm` to cancel its +execution. These functions are R wrappers for the SLURM command line functions +`squeue` and `scancel`, respectively. + +Once the job completes, `get_slurm_out` reads and combines the output from all +nodes. +```{r} +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) +``` + +When `outtype = "table"`, the outputs from each function evaluation are +row-bound into a single data frame; this is an appropriate format when the +function returns a simple vector. The default `outtype = "raw"` combines the +outputs into a list and can thus handle arbitrarily complex return objects. + +```{r} +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] +``` + +The files generated by `slurm_apply` are saved in a folder named +*\_rslurm_[jobname]* under the current working directory. + +```{r} +dir("_rslurm_test_job") +``` + +The utility function `cleanup_files` deletes the temporary folder for the +specified *slurm_job*. + +```{r echo=FALSE} +cleanup_files(sjob) +``` + + +## Single function evaluation + +In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which +sends a single function call to the cluster. It is analogous in syntax to the +base R function `do.call`, accepting a function and a named list of parameters +as arguments. + +```{r} +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +Because `slurm_call` involves a single process on a single node, it does not +recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the +same additional arguments (detailed in the sections below) as `slurm_apply`. + + +## Adding auxiliary data and functions + +The function passed to `slurm_apply` can only receive atomic parameters stored +within a data frame. Suppose we want instead to apply a function `func` to a list +of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap +`func` in an inline function that takes an integer parameter. + +```{r echo=FALSE} +obj_list <- list(NULL) +func <- function(obj) {} +``` +```{r} +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +The `add_objects` argument specifies the names of any R objects (besides the +parameters data frame) that must be accessed by the function passed to +`slurm_apply`. These objects are saved to a `.RData` file that is loaded +on each cluster node prior to evaluating the function in parallel. + +By default, all R packages attached to the current R session will also be +attached (with `library`) on each cluster node, though this can be modified with +the optional `pkgs` argument. + + +## Configuring SLURM options + +The `slurm_options` argument allows you to set any of the command line +options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the +SLURM `sbatch` command. It should be formatted as a named list, using the long +names of each option (e.g. "time" rather than "t"). Flags, i.e. command line +options that are toggled rather than set to a particular value, should be set to +`TRUE` in `slurm_options`. For example, the following code: +```{r} +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` +sets the command line options `--time=1:00:00 --share`. + + +## Generating scripts for later submission + +When working from a R session without direct access to the cluster, you can set +`submit = FALSE` within `slurm_apply`. The function will create the +*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without +submitting the job. You may then copy those files to the cluster and submit the +job manually by calling `sbatch submit.sh` from the command line. + + +## How it works / advanced customization + +As mentioned above, the `slurm_apply` function creates a job-specific folder. +This folder contains the parameters as a *.RDS* file and (if applicable) the objects +specified as `add_objects` saved together in a *.RData* file. The function also +generates a R script (`slurm_run.R`) to be run on each cluster node, as well +as a Bash script (`submit.sh`) to submit the job to SLURM. + +More specifically, the Bash script creates a SLURM job array, with each cluster +node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment +variable. This variable is read by `slurm_run.R`, which allows each instance of +the script to operate on a different parameter subset and write its output to +a different results file. The R script calls `parallel::mcMap` to parallelize +calculations on each node. + +Both `slurm_run.R` and `submit.sh` are generated from templates, using the +**whisker** package; these templates can be found in the `rslurm/templates` +subfolder in your R package library. There are two templates for each script, +one for `slurm_apply` and the other (with the word *single* in its title) for +`slurm_call`. + +While you should avoid changing any existing lines in the template scripts, you +may want to add `#SBATCH` lines to the `submit.sh` templates in order to +permanently set certain SLURM command line options and thus customize the package +to your particular cluster setup. \ No newline at end of file diff --git a/inst/doc/overview.html b/inst/doc/overview.html new file mode 100644 index 0000000..e13f9e3 --- /dev/null +++ b/inst/doc/overview.html @@ -0,0 +1,188 @@ +<!DOCTYPE html> + +<html xmlns="http://www.w3.org/1999/xhtml"> + +<head> + +<meta charset="utf-8"> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="pandoc" /> + +<meta name="viewport" content="width=device-width, initial-scale=1"> + + + +<title>Parallelize R code on a SLURM cluster</title> + + + +<style type="text/css">code{white-space: pre;}</style> +<style type="text/css"> +div.sourceCode { overflow-x: auto; } +table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode { + margin: 0; padding: 0; vertical-align: baseline; border: none; } +table.sourceCode { width: 100%; line-height: 100%; } +td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; } +td.sourceCode { padding-left: 5px; } +code > span.kw { color: #007020; font-weight: bold; } /* Keyword */ +code > span.dt { color: #902000; } /* DataType */ +code > span.dv { color: #40a070; } /* DecVal */ +code > span.bn { color: #40a070; } /* BaseN */ +code > span.fl { color: #40a070; } /* Float */ +code > span.ch { color: #4070a0; } /* Char */ +code > span.st { color: #4070a0; } /* String */ +code > span.co { color: #60a0b0; font-style: italic; } /* Comment */ +code > span.ot { color: #007020; } /* Other */ +code > span.al { color: #ff0000; font-weight: bold; } /* Alert */ +code > span.fu { color: #06287e; } /* Function */ +code > span.er { color: #ff0000; font-weight: bold; } /* Error */ +code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */ +code > span.cn { color: #880000; } /* Constant */ +code > span.sc { color: #4070a0; } /* SpecialChar */ +code > span.vs { color: #4070a0; } /* VerbatimString */ +code > span.ss { color: #bb6688; } /* SpecialString */ +code > span.im { } /* Import */ +code > span.va { color: #19177c; } /* Variable */ +code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */ +code > span.op { color: #666666; } /* Operator */ +code > span.bu { } /* BuiltIn */ +code > span.ex { } /* Extension */ +code > span.pp { color: #bc7a00; } /* Preprocessor */ +code > span.at { color: #7d9029; } /* Attribute */ +code > span.do { color: #ba2121; font-style: italic; } /* Documentation */ +code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */ +code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */ +code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */ +</style> + + + +<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" /> + +</head> + +<body> + + + + +<h1 class="title toc-ignore">Parallelize R code on a SLURM cluster</h1> + + + +<p>Many computing-intensive processes in R involve the repeated evaluation of a function over many items or parameter sets. These so-called <a href="https://en.wikipedia.org/wiki/Embarrassingly_parallel">embarrassingly parallel</a> calculations can be run serially with the <code>lapply</code> or <code>Map</code> function, or in parallel on a single machine with <code>mclapply</code> or <code>mcMap</code> (from the <strong>parallel</strong> package).</p> +<p>The rslurm package simplifies the process of distributing this type of calculation across a computing cluster that uses the <a href="http://slurm.schedmd.com/">SLURM</a> workload manager. Its main function, <code>slurm_apply</code>, automatically divides the computation over multiple nodes and writes the necessary submission scripts. It also includes functions to retrieve and combine the output from different nodes, as well as wrappers for common SLURM commands.</p> +<div id="table-of-contents" class="section level3"> +<h3>Table of contents</h3> +<ul> +<li><a href="#basic-example">Basic example</a></li> +<li><a href="#single-function-evaluation">Single function evaluation</a></li> +<li><a href="#adding-auxiliary-data-and-functions">Adding auxiliary data and functions</a></li> +<li><a href="#configuring-slurm-options">Configuring SLURM options</a></li> +<li><a href="#generating-scripts-for-later-submission">Generating scripts for later submission</a></li> +<li><a href="#how-it-works-advanced-customization">How it works / advanced customization</a></li> +</ul> +</div> +<div id="basic-example" class="section level2"> +<h2>Basic example</h2> +<p>To illustrate a typical rslurm workflow, we use a simple function that takes a mean and standard deviation as parameters, generates a million normal deviates and returns the sample mean and standard deviation.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">test_func <-<span class="st"> </span>function(par_mu, par_sd) { + samp <-<span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">10</span>^<span class="dv">6</span>, par_mu, par_sd) + <span class="kw">c</span>(<span class="dt">s_mu =</span> <span class="kw">mean</span>(samp), <span class="dt">s_sd =</span> <span class="kw">sd</span>(samp)) +}</code></pre></div> +<p>We then create a parameter data frame where each row is a parameter set and each column matches an argument of the function.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">pars <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">par_mu =</span> <span class="dv">1</span>:<span class="dv">10</span>, + <span class="dt">par_sd =</span> <span class="kw">seq</span>(<span class="fl">0.1</span>, <span class="dv">1</span>, <span class="dt">length.out =</span> <span class="dv">10</span>)) +<span class="kw">head</span>(pars, <span class="dv">3</span>)</code></pre></div> +<pre><code>## par_mu par_sd +## 1 1 0.1 +## 2 2 0.2 +## 3 3 0.3</code></pre> +<p>We can now pass that function and the parameters data frame to <code>slurm_apply</code>, specifiying the number of cluster nodes to use and the number of CPUs per node. The latter (<code>cpus_per_node</code>) determines how many processes will be forked on each node, as the <code>mc.cores</code> argument of <code>parallel::mcMap</code>.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) +sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 551091</code></pre> +<p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> +<p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> +<p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> +<p>After the job has been submitted, you can call <code>print_job_status</code> to display its status (in queue, running or completed) or call <code>cancel_slurm</code> to cancel its execution. These functions are R wrappers for the SLURM command line functions <code>squeue</code> and <code>scancel</code>, respectively.</p> +<p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) +<span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> +<pre><code>## s_mu s_sd +## 1 0.9999199 0.1000873 +## 2 2.0000448 0.1999509 +## 3 3.0001045 0.2996829</code></pre> +<p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) +res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> +<pre><code>## [[1]] +## s_mu s_sd +## 0.9999199 0.1000873 +## +## [[2]] +## s_mu s_sd +## 2.0000448 0.1999509 +## +## [[3]] +## s_mu s_sd +## 3.0001045 0.2996829</code></pre> +<p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> +<pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" +## [5] "slurm_1.out" "slurm_run.R" "submit.sh"</code></pre> +<p>The utility function <code>cleanup_files</code> deletes the temporary folder for the specified <em>slurm_job</em>.</p> +</div> +<div id="single-function-evaluation" class="section level2"> +<h2>Single function evaluation</h2> +<p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> +<pre><code>## Submitted batch job 551095</code></pre> +<p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> +</div> +<div id="adding-auxiliary-data-and-functions" class="section level2"> +<h2>Adding auxiliary data and functions</h2> +<p>The function passed to <code>slurm_apply</code> can only receive atomic parameters stored within a data frame. Suppose we want instead to apply a function <code>func</code> to a list of complex R objects, <code>obj_list</code>. To use <code>slurm_apply</code> in this case, we can wrap <code>func</code> in an inline function that takes an integer parameter.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(function(i) <span class="kw">func</span>(obj_list[[i]]), + <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), + <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 551097</code></pre> +<p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> +<p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> +</div> +<div id="configuring-slurm-options" class="section level2"> +<h2>Configuring SLURM options</h2> +<p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, + <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> +<pre><code>## Submitted batch job 551099</code></pre> +<p>sets the command line options <code>--time=1:00:00 --share</code>.</p> +</div> +<div id="generating-scripts-for-later-submission" class="section level2"> +<h2>Generating scripts for later submission</h2> +<p>When working from a R session without direct access to the cluster, you can set <code>submit = FALSE</code> within <code>slurm_apply</code>. The function will create the <em>_rslurm_[jobname]</em> folder and generate the scripts and .RData files, without submitting the job. You may then copy those files to the cluster and submit the job manually by calling <code>sbatch submit.sh</code> from the command line.</p> +</div> +<div id="how-it-works-advanced-customization" class="section level2"> +<h2>How it works / advanced customization</h2> +<p>As mentioned above, the <code>slurm_apply</code> function creates a job-specific folder. This folder contains the parameters as a <em>.RDS</em> file and (if applicable) the objects specified as <code>add_objects</code> saved together in a <em>.RData</em> file. The function also generates a R script (<code>slurm_run.R</code>) to be run on each cluster node, as well as a Bash script (<code>submit.sh</code>) to submit the job to SLURM.</p> +<p>More specifically, the Bash script creates a SLURM job array, with each cluster node receiving a different value of the <em>SLURM_ARRAY_TASK_ID</em> environment variable. This variable is read by <code>slurm_run.R</code>, which allows each instance of the script to operate on a different parameter subset and write its output to a different results file. The R script calls <code>parallel::mcMap</code> to parallelize calculations on each node.</p> +<p>Both <code>slurm_run.R</code> and <code>submit.sh</code> are generated from templates, using the <strong>whisker</strong> package; these templates can be found in the <code>rslurm/templates</code> subfolder in your R package library. There are two templates for each script, one for <code>slurm_apply</code> and the other (with the word <em>single</em> in its title) for <code>slurm_call</code>.</p> +<p>While you should avoid changing any existing lines in the template scripts, you may want to add <code>#SBATCH</code> lines to the <code>submit.sh</code> templates in order to permanently set certain SLURM command line options and thus customize the package to your particular cluster setup.</p> +</div> + + + +<!-- dynamically load mathjax for compatibility with self-contained --> +<script> + (function () { + var script = document.createElement("script"); + script.type = "text/javascript"; + script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; + document.getElementsByTagName("head")[0].appendChild(script); + })(); +</script> + +</body> +</html> From 1ceda733678013f58dd96161bc4074430dc1689c Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 14:33:12 -0400 Subject: [PATCH 10/25] passed win-builder check without vignette --- cran-comments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cran-comments.md b/cran-comments.md index 8a6fd84..8ee50fa 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -7,8 +7,8 @@ The 'overview.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on Ubuntu 14.04 with R 3.3 (rstudio.sesync.org, SLURM head node) Ubuntu 12.04 with R 3.3 (travis-ci) macOS 10.12 with R 3.3 (local machine, R CMD build with '--no-build-vignettes', R CMD check with '--no-vignettes') -?win-builder (release and devel, after manual removal of 'overview.Rmd' vignette) +win-builder (release and devel, after manual removal of 'overview.Rmd' vignette) ## R CMD check -0 errors | 0 warnings | 0 notes \ No newline at end of file +0 errors | 0 warnings | 0 notes From 5e6831a0dbb9dd5ace486450da802dbd383aad26 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 21:32:42 -0400 Subject: [PATCH 11/25] test vignette buildignore on travis --- .Rbuildignore | 2 + R/rslurm.R | 7 +- README.html | 94 +++++++++++++++++++++++++ README.md | 53 ++++++++++++-- inst/doc/{overview.R => rslurm.R} | 0 inst/doc/{overview.Rmd => rslurm.Rmd} | 0 inst/doc/{overview.html => rslurm.html} | 24 +++---- man/rslurm-package.Rd | 30 ++++++++ vignettes/{overview.Rmd => rslurm.Rmd} | 0 9 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 README.html rename inst/doc/{overview.R => rslurm.R} (100%) rename inst/doc/{overview.Rmd => rslurm.Rmd} (100%) rename inst/doc/{overview.html => rslurm.html} (97%) rename vignettes/{overview.Rmd => rslurm.Rmd} (100%) diff --git a/.Rbuildignore b/.Rbuildignore index e5afaad..51cd730 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,3 +2,5 @@ ^\.Rproj\.user$ ^\.travis\.yml$ cran-comments.md +vignettes/rslurm.Rmd +inst/doc/rslurm.Rmd diff --git a/R/rslurm.R b/R/rslurm.R index 214d21b..ef70ec2 100755 --- a/R/rslurm.R +++ b/R/rslurm.R @@ -65,9 +65,14 @@ #' cleanup_files(sjob1) #' } #' +#' @section Acknowledgement: +#' Development of this R package was supported by the National +#' Socio-Environmental Synthesis Center (SESYNC) under funding received +#' from the National Science Foundation DBI-1052875. +#' #' @docType package #' @name rslurm-package #' @aliases rslurm #' #' @importFrom utils capture.output -NULL +"_PACKAGE" diff --git a/README.html b/README.html new file mode 100644 index 0000000..c6d3d32 --- /dev/null +++ b/README.html @@ -0,0 +1,94 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><title>R: rslurm: Submit R calculations to a SLURM cluster</title> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<link rel="stylesheet" type="text/css" href="R.css" /> +</head><body> + +<table width="100%" summary="page for rslurm-package"><tr><td>rslurm-package</td><td style="text-align: right;">R Documentation</td></tr></table> + +<h2>rslurm: Submit R calculations to a SLURM cluster</h2> + +<h3>Description</h3> + +<p>This package automates the process of sending simple function calls or +parallel calculations to a cluster using the SLURM workload manager. +</p> + + +<h3>Overview</h3> + +<p>This package includes two core functions used to send computations to a +SLURM cluster. While <code>slurm_call</code> executes a function using a +single set of parameters (passed as a list), <code>slurm_apply</code> +evaluates the function in parallel for multiple sets of parameters grouped +in a data frame. <code>slurm_apply</code> automatically splits the parameter +sets into equal-size chunks, each chunk to be processed by a separate +cluster node. It uses functions from the <code>parallel</code> +package to parallelize computations within each node. +</p> +<p>The output of <code>slurm_apply</code> or <code>slurm_call</code> is a +<code>slurm_job</code> object that serves as an input to the other functions +in the package: <code>print_job_status</code>, <code>cancel_slurm</code>, +<code>get_slurm_out</code> and <code>cleanup_files</code>. +</p> +<p>For bug reports or questions about this package, contact +Ian Carroll(icarroll@sesync.org). +</p> + + +<h3>Function Specification</h3> + +<p>To be compatible with <code>slurm_apply</code>, a function may accept +any number of single value parameters. The names of these parameters must +match the column names of the <code>params</code> data frame supplied. There are no +restrictions on the types of parameters passed as a list to +<code>slurm_call</code>. +</p> +<p>If the function passed to <code>slurm_call</code> or <code>slurm_apply</code> requires +knowledge of any R objects (data, custom helper functions) besides <code>params</code>, +a character vector corresponding to their names should be passed to the +optional <code>add_objects</code> argument. +</p> +<p>When parallelizing a function, since any error will interrupt all calculations +for the current node, it may be useful to wrap expressions which may generate +errors into a <code>try</code> or <code>tryCatch</code> function. +This will ensure the computation continues with the next parameter set after +reporting the error. +</p> + + +<h3>Output Format</h3> + +<p>The default output format for <code>get_slurm_out</code> (<code>outtype = "raw"</code>) +is a list where each element is the return value of one function call. If the +function passed to <code>slurm_apply</code> produces a vector output, you may use +<code>outtype = "table"</code> to collect the output in a single data frame, with +one row by function call. +</p> + + +<h3>Examples</h3> + +<pre> +## Not run: +# Create a data frame of mean/sd values for normal distributions +pars <- data.frame(par_m = seq(-10, 10, length.out = 1000), + par_sd = seq(0.1, 10, length.out = 1000)) + +# Create a function to parallelize +ftest <- function(par_m, par_sd) { + samp <- rnorm(10^7, par_m, par_sd) + c(s_m = mean(samp), s_sd = sd(samp)) +} + +sjob1 <- slurm_apply(ftest, pars) +print_job_status(sjob1) +res <- get_slurm_out(sjob1, "table") +all.equal(pars, res) # Confirm correct output +cleanup_files(sjob1) + +## End(Not run) + +</pre> + + +</body></html> diff --git a/README.md b/README.md index 44229ef..64c37c5 100755 --- a/README.md +++ b/README.md @@ -1,11 +1,52 @@ -# rslurm +| | | +|----------------|-----------------| +| rslurm-package | R Documentation | -[![Travis-CI Build Status](https://travis-ci.org/SESYNC-ci/rslurm.svg?branch=master)](https://travis-ci.org/SESYNC-ci/rslurm) +rslurm: Submit R calculations to a SLURM cluster +------------------------------------------------ -## Overview +### Description -See [inst/overview.html] +This package automates the process of sending simple function calls or parallel calculations to a cluster using the SLURM workload manager. -## Acknowledgement +### Overview + +This package includes two core functions used to send computations to a SLURM cluster. While `slurm_call` executes a function using a single set of parameters (passed as a list), `slurm_apply` evaluates the function in parallel for multiple sets of parameters grouped in a data frame. `slurm_apply` automatically splits the parameter sets into equal-size chunks, each chunk to be processed by a separate cluster node. It uses functions from the `parallel` package to parallelize computations within each node. + +The output of `slurm_apply` or `slurm_call` is a `slurm_job` object that serves as an input to the other functions in the package: `print_job_status`, `cancel_slurm`, `get_slurm_out` and `cleanup_files`. + +For bug reports or questions about this package, contact Ian Carroll(icarroll@sesync.org). + +### Function Specification + +To be compatible with `slurm_apply`, a function may accept any number of single value parameters. The names of these parameters must match the column names of the `params` data frame supplied. There are no restrictions on the types of parameters passed as a list to `slurm_call`. + +If the function passed to `slurm_call` or `slurm_apply` requires knowledge of any R objects (data, custom helper functions) besides `params`, a character vector corresponding to their names should be passed to the optional `add_objects` argument. + +When parallelizing a function, since any error will interrupt all calculations for the current node, it may be useful to wrap expressions which may generate errors into a `try` or `tryCatch` function. This will ensure the computation continues with the next parameter set after reporting the error. + +### Output Format + +The default output format for `get_slurm_out` (`outtype = "raw"`) is a list where each element is the return value of one function call. If the function passed to `slurm_apply` produces a vector output, you may use `outtype = "table"` to collect the output in a single data frame, with one row by function call. + +### Examples + + ## Not run: + # Create a data frame of mean/sd values for normal distributions + pars <- data.frame(par_m = seq(-10, 10, length.out = 1000), + par_sd = seq(0.1, 10, length.out = 1000)) + + # Create a function to parallelize + ftest <- function(par_m, par_sd) { + samp <- rnorm(10^7, par_m, par_sd) + c(s_m = mean(samp), s_sd = sd(samp)) + } + + sjob1 <- slurm_apply(ftest, pars) + print_job_status(sjob1) + res <- get_slurm_out(sjob1, "table") + all.equal(pars, res) # Confirm correct output + cleanup_files(sjob1) + + ## End(Not run) -Development of this R package was supported by the National Socio-Environmental Synthesis Center (SESYNC) under funding received from the National Science Foundation DBI-1052875. diff --git a/inst/doc/overview.R b/inst/doc/rslurm.R similarity index 100% rename from inst/doc/overview.R rename to inst/doc/rslurm.R diff --git a/inst/doc/overview.Rmd b/inst/doc/rslurm.Rmd similarity index 100% rename from inst/doc/overview.Rmd rename to inst/doc/rslurm.Rmd diff --git a/inst/doc/overview.html b/inst/doc/rslurm.html similarity index 97% rename from inst/doc/overview.html rename to inst/doc/rslurm.html index e13f9e3..0538996 100644 --- a/inst/doc/overview.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551091</code></pre> +<pre><code>## Submitted batch job 551113</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9999199 0.1000873 -## 2 2.0000448 0.1999509 -## 3 3.0001045 0.2996829</code></pre> +<pre><code>## s_mu s_sd +## 1 0.9999847 0.09997995 +## 2 2.0001498 0.19994611 +## 3 3.0003275 0.30017820</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] -## s_mu s_sd -## 0.9999199 0.1000873 +## s_mu s_sd +## 0.99998468 0.09997995 ## ## [[2]] ## s_mu s_sd -## 2.0000448 0.1999509 +## 2.0001498 0.1999461 ## ## [[3]] ## s_mu s_sd -## 3.0001045 0.2996829</code></pre> +## 3.0003275 0.3001782</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551095</code></pre> +<pre><code>## Submitted batch job 551117</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551097</code></pre> +<pre><code>## Submitted batch job 551119</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551099</code></pre> +<pre><code>## Submitted batch job 551121</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 1db92f3..56f7295 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -3,6 +3,7 @@ \docType{package} \name{rslurm-package} \alias{rslurm-package} +\alias{_PACKAGE} \alias{rslurm} \title{rslurm: Submit R calculations to a SLURM cluster} \description{ @@ -58,6 +59,13 @@ function passed to \code{slurm_apply} produces a vector output, you may use one row by function call. } +\section{Acknowledgement}{ + +Development of this R package was supported by the National +Socio-Environmental Synthesis Center (SESYNC) under funding received +from the National Science Foundation DBI-1052875. +} + \examples{ \dontrun{ # Create a data frame of mean/sd values for normal distributions @@ -78,3 +86,25 @@ cleanup_files(sjob1) } } +\seealso{ +Useful links: +\itemize{ + \item \url{https://github.com/SESYNC-ci/rslurm} + \item Report bugs at \url{https://github.com/SESYNC-ci/rslurm/issues} +} + +} +\author{ +\strong{Maintainer}: Ian Carroll \email{icarroll@sesync.org} [contributor] + +Authors: +\itemize{ + \item Philippe Marchand +} + +Other contributors: +\itemize{ + \item Mike Smorul [contributor] +} + +} diff --git a/vignettes/overview.Rmd b/vignettes/rslurm.Rmd similarity index 100% rename from vignettes/overview.Rmd rename to vignettes/rslurm.Rmd From a514b62f47df7b5e983f85c095b27ca19f5ee8e2 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 21:50:18 -0400 Subject: [PATCH 12/25] remove travis build conf --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7fd203..b79e15a 100755 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: r warnings_are_errors: true sudo: required -r_build_args: "--no-build-vignettes" -r_check_args: "--no-vignettes" env: global: From a401b6c7a61a9d7669960db759e10486ab9d6c6b Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 23:07:41 -0400 Subject: [PATCH 13/25] try to disable vignette build with DESCRIPTION --- .Rbuildignore | 2 -- DESCRIPTION | 1 + README.html | 94 --------------------------------------------------- 3 files changed, 1 insertion(+), 96 deletions(-) delete mode 100644 README.html diff --git a/.Rbuildignore b/.Rbuildignore index 51cd730..e5afaad 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,5 +2,3 @@ ^\.Rproj\.user$ ^\.travis\.yml$ cran-comments.md -vignettes/rslurm.Rmd -inst/doc/rslurm.Rmd diff --git a/DESCRIPTION b/DESCRIPTION index c04336a..4d2287b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -23,3 +23,4 @@ Suggests: knitr, rmarkdown VignetteBuilder: knitr +BuildVignettes: no diff --git a/README.html b/README.html deleted file mode 100644 index c6d3d32..0000000 --- a/README.html +++ /dev/null @@ -1,94 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><title>R: rslurm: Submit R calculations to a SLURM cluster</title> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<link rel="stylesheet" type="text/css" href="R.css" /> -</head><body> - -<table width="100%" summary="page for rslurm-package"><tr><td>rslurm-package</td><td style="text-align: right;">R Documentation</td></tr></table> - -<h2>rslurm: Submit R calculations to a SLURM cluster</h2> - -<h3>Description</h3> - -<p>This package automates the process of sending simple function calls or -parallel calculations to a cluster using the SLURM workload manager. -</p> - - -<h3>Overview</h3> - -<p>This package includes two core functions used to send computations to a -SLURM cluster. While <code>slurm_call</code> executes a function using a -single set of parameters (passed as a list), <code>slurm_apply</code> -evaluates the function in parallel for multiple sets of parameters grouped -in a data frame. <code>slurm_apply</code> automatically splits the parameter -sets into equal-size chunks, each chunk to be processed by a separate -cluster node. It uses functions from the <code>parallel</code> -package to parallelize computations within each node. -</p> -<p>The output of <code>slurm_apply</code> or <code>slurm_call</code> is a -<code>slurm_job</code> object that serves as an input to the other functions -in the package: <code>print_job_status</code>, <code>cancel_slurm</code>, -<code>get_slurm_out</code> and <code>cleanup_files</code>. -</p> -<p>For bug reports or questions about this package, contact -Ian Carroll(icarroll@sesync.org). -</p> - - -<h3>Function Specification</h3> - -<p>To be compatible with <code>slurm_apply</code>, a function may accept -any number of single value parameters. The names of these parameters must -match the column names of the <code>params</code> data frame supplied. There are no -restrictions on the types of parameters passed as a list to -<code>slurm_call</code>. -</p> -<p>If the function passed to <code>slurm_call</code> or <code>slurm_apply</code> requires -knowledge of any R objects (data, custom helper functions) besides <code>params</code>, -a character vector corresponding to their names should be passed to the -optional <code>add_objects</code> argument. -</p> -<p>When parallelizing a function, since any error will interrupt all calculations -for the current node, it may be useful to wrap expressions which may generate -errors into a <code>try</code> or <code>tryCatch</code> function. -This will ensure the computation continues with the next parameter set after -reporting the error. -</p> - - -<h3>Output Format</h3> - -<p>The default output format for <code>get_slurm_out</code> (<code>outtype = "raw"</code>) -is a list where each element is the return value of one function call. If the -function passed to <code>slurm_apply</code> produces a vector output, you may use -<code>outtype = "table"</code> to collect the output in a single data frame, with -one row by function call. -</p> - - -<h3>Examples</h3> - -<pre> -## Not run: -# Create a data frame of mean/sd values for normal distributions -pars <- data.frame(par_m = seq(-10, 10, length.out = 1000), - par_sd = seq(0.1, 10, length.out = 1000)) - -# Create a function to parallelize -ftest <- function(par_m, par_sd) { - samp <- rnorm(10^7, par_m, par_sd) - c(s_m = mean(samp), s_sd = sd(samp)) -} - -sjob1 <- slurm_apply(ftest, pars) -print_job_status(sjob1) -res <- get_slurm_out(sjob1, "table") -all.equal(pars, res) # Confirm correct output -cleanup_files(sjob1) - -## End(Not run) - -</pre> - - -</body></html> From bb6e09af34084fbdb8fe4b8e0172239c66f54247 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Thu, 6 Apr 2017 23:40:52 -0400 Subject: [PATCH 14/25] test .Rbuildignore on travis --- .Rbuildignore | 3 ++- DESCRIPTION | 6 +++--- R/rslurm.R | 7 +------ inst/doc/rslurm.html | 20 ++++++++++---------- man/rslurm-package.Rd | 7 ++++--- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index e5afaad..c81f3a3 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,4 +1,5 @@ ^.*\.Rproj$ ^\.Rproj\.user$ ^\.travis\.yml$ -cran-comments.md +^cran-comments\.md +^vignettes diff --git a/DESCRIPTION b/DESCRIPTION index 4d2287b..a36d623 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,9 @@ Package: rslurm Type: Package Title: Submit R Calculations to a 'SLURM' Cluster -Description: Functions that simplify the R interface the 'SLURM' cluster - workload manager, and automate the process of dividing a parallel calculation - across cluster nodes. +Description: Functions that simplify submitting R scripts to a 'SLURM' cluster + workload manager, in part by automating the division of emarassingly parallel + calculations across cluster nodes. Version: 0.3.2 License: GPL-3 URL: https://github.com/SESYNC-ci/rslurm diff --git a/R/rslurm.R b/R/rslurm.R index ef70ec2..b48ab16 100755 --- a/R/rslurm.R +++ b/R/rslurm.R @@ -1,8 +1,3 @@ -#' rslurm: Submit R calculations to a SLURM cluster -#' -#' This package automates the process of sending simple function calls or -#' parallel calculations to a cluster using the SLURM workload manager. -#' #' @section Overview: #' This package includes two core functions used to send computations to a #' SLURM cluster. While \code{\link{slurm_call}} executes a function using a @@ -75,4 +70,4 @@ #' @aliases rslurm #' #' @importFrom utils capture.output -"_PACKAGE" +"_PACKAGE" \ No newline at end of file diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html index 0538996..dbf1252 100644 --- a/inst/doc/rslurm.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551113</code></pre> +<pre><code>## Submitted batch job 551179</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -111,23 +111,23 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> <pre><code>## s_mu s_sd -## 1 0.9999847 0.09997995 -## 2 2.0001498 0.19994611 -## 3 3.0003275 0.30017820</code></pre> +## 1 0.9997706 0.09990867 +## 2 1.9997721 0.20004867 +## 3 2.9998451 0.30017665</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] ## s_mu s_sd -## 0.99998468 0.09997995 +## 0.99977062 0.09990867 ## ## [[2]] ## s_mu s_sd -## 2.0001498 0.1999461 +## 1.9997721 0.2000487 ## ## [[3]] ## s_mu s_sd -## 3.0003275 0.3001782</code></pre> +## 2.9998451 0.3001767</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551117</code></pre> +<pre><code>## Submitted batch job 551183</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551119</code></pre> +<pre><code>## Submitted batch job 551185</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551121</code></pre> +<pre><code>## Submitted batch job 551187</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 56f7295..b8a8598 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -5,10 +5,11 @@ \alias{rslurm-package} \alias{_PACKAGE} \alias{rslurm} -\title{rslurm: Submit R calculations to a SLURM cluster} +\title{rslurm: Submit R Calculations to a 'SLURM' Cluster} \description{ -This package automates the process of sending simple function calls or -parallel calculations to a cluster using the SLURM workload manager. +Functions that simplify submitting R scripts to a 'SLURM' cluster +workload manager, in part by automating the division of emarassingly parallel +calculations across cluster nodes. } \section{Overview}{ From ec3510b2dd51919fd9156d9ec11799212f950ff7 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 01:35:13 -0400 Subject: [PATCH 15/25] document.R to create docs --- .Rbuildignore | 1 + R/rslurm.R | 4 ---- README.md | 30 +++++++++++++++++++++++++++--- document.R | 18 ++++++++++++++++++ inst/doc/rslurm.Rmd | 2 +- inst/doc/rslurm.html | 22 +++++++++++----------- man/rslurm-package.Rd | 3 +-- vignettes/rslurm.Rmd | 2 +- 8 files changed, 60 insertions(+), 22 deletions(-) mode change 100755 => 100644 README.md create mode 100644 document.R diff --git a/.Rbuildignore b/.Rbuildignore index c81f3a3..e0e34c8 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,4 +2,5 @@ ^\.Rproj\.user$ ^\.travis\.yml$ ^cran-comments\.md +^document\.R ^vignettes diff --git a/R/rslurm.R b/R/rslurm.R index b48ab16..b4d3889 100755 --- a/R/rslurm.R +++ b/R/rslurm.R @@ -65,9 +65,5 @@ #' Socio-Environmental Synthesis Center (SESYNC) under funding received #' from the National Science Foundation DBI-1052875. #' -#' @docType package -#' @name rslurm-package -#' @aliases rslurm -#' #' @importFrom utils capture.output "_PACKAGE" \ No newline at end of file diff --git a/README.md b/README.md old mode 100755 new mode 100644 index 64c37c5..7339824 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ |----------------|-----------------| | rslurm-package | R Documentation | -rslurm: Submit R calculations to a SLURM cluster ------------------------------------------------- +rslurm: Submit R Calculations to a 'SLURM' Cluster +-------------------------------------------------- ### Description -This package automates the process of sending simple function calls or parallel calculations to a cluster using the SLURM workload manager. +Functions that simplify submitting R scripts to a 'SLURM' cluster workload manager, in part by automating the division of emarassingly parallel calculations across cluster nodes. ### Overview @@ -29,6 +29,30 @@ When parallelizing a function, since any error will interrupt all calculations f The default output format for `get_slurm_out` (`outtype = "raw"`) is a list where each element is the return value of one function call. If the function passed to `slurm_apply` produces a vector output, you may use `outtype = "table"` to collect the output in a single data frame, with one row by function call. +### Acknowledgement + +Development of this R package was supported by the National Socio-Environmental Synthesis Center (SESYNC) under funding received from the National Science Foundation DBI-1052875. + +### Author(s) + +**Maintainer**: Ian Carroll <icarroll@sesync.org> \[contributor\] + +Authors: + +- Philippe Marchand + +Other contributors: + +- Mike Smorul \[contributor\] + +### See Also + +Useful links: + +- <https://github.com/SESYNC-ci/rslurm> + +- Report bugs at <https://github.com/SESYNC-ci/rslurm/issues> + ### Examples ## Not run: diff --git a/document.R b/document.R new file mode 100644 index 0000000..9b36c3d --- /dev/null +++ b/document.R @@ -0,0 +1,18 @@ +library(tools) +library(rmarkdown) +library(devtools) + +# Create a README.md for GitHub and CRAN from '../R/rlurm.R' +# by way of the '../man/rslurm-package.Rd' produced by roxygen2 +Rd2HTML(parse_Rd('man/rslurm-package.Rd'), out = 'README.html') +pandoc_convert(input='README.html', to='markdown_github', output='README.md') +unlink('README.html') + +# Build vignettes +build_vignettes() + +## TODO +# figure out how to include Travis badge +# +# how does rmarkdown have NEWS in package index? +# maybe by including inst/NEWS.Rd, seee tools:::news2rd diff --git a/inst/doc/rslurm.Rmd b/inst/doc/rslurm.Rmd index bdc2659..416ee47 100644 --- a/inst/doc/rslurm.Rmd +++ b/inst/doc/rslurm.Rmd @@ -2,7 +2,7 @@ title: "Parallelize R code on a SLURM cluster" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{overview} + %\VignetteIndexEntry{Parallelize R code on a SLURM cluster} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html index dbf1252..f742db5 100644 --- a/inst/doc/rslurm.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551179</code></pre> +<pre><code>## Submitted batch job 551278</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9997706 0.09990867 -## 2 1.9997721 0.20004867 -## 3 2.9998451 0.30017665</code></pre> +<pre><code>## s_mu s_sd +## 1 1.000131 0.09993934 +## 2 1.999793 0.19983189 +## 3 3.000108 0.29983346</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] ## s_mu s_sd -## 0.99977062 0.09990867 +## 1.00013089 0.09993934 ## ## [[2]] ## s_mu s_sd -## 1.9997721 0.2000487 +## 1.9997930 0.1998319 ## ## [[3]] ## s_mu s_sd -## 2.9998451 0.3001767</code></pre> +## 3.0001082 0.2998335</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551183</code></pre> +<pre><code>## Submitted batch job 551282</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551185</code></pre> +<pre><code>## Submitted batch job 551284</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551187</code></pre> +<pre><code>## Submitted batch job 551286</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index b8a8598..590fafb 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -2,9 +2,8 @@ % Please edit documentation in R/rslurm.R \docType{package} \name{rslurm-package} -\alias{rslurm-package} -\alias{_PACKAGE} \alias{rslurm} +\alias{rslurm-package} \title{rslurm: Submit R Calculations to a 'SLURM' Cluster} \description{ Functions that simplify submitting R scripts to a 'SLURM' cluster diff --git a/vignettes/rslurm.Rmd b/vignettes/rslurm.Rmd index bdc2659..416ee47 100644 --- a/vignettes/rslurm.Rmd +++ b/vignettes/rslurm.Rmd @@ -2,7 +2,7 @@ title: "Parallelize R code on a SLURM cluster" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{overview} + %\VignetteIndexEntry{Parallelize R code on a SLURM cluster} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- From 15994507e808eb7ab884921824e418b51ebceb3b Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 01:36:09 -0400 Subject: [PATCH 16/25] NEWS as Rd --- NEWS.md | 113 --------------------------------------------------- inst/NEWS.rd | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 113 deletions(-) delete mode 100755 NEWS.md create mode 100644 inst/NEWS.rd diff --git a/NEWS.md b/NEWS.md deleted file mode 100755 index b6388f8..0000000 --- a/NEWS.md +++ /dev/null @@ -1,113 +0,0 @@ -rslurm 0.3.2 -============ - -*2017-04-06* - -Minor update to include new feature and bug fixes: - -* 'Wait' feature: adds option to slurm_apply and slurm_call to block the -calling script until the submitted job completes. This option can be used to -allow immediate processing of a submitted job's output. - -* Use '.RDS' file extension, rather than '.RData', for serialized objects. - -* Minor bug fixes. - - -rslurm 0.3.1 -============ - -*2016-06-18* - -* Minor bug fix: specify full path of 'Rscript' when running batch scripts. - - -rslurm 0.3.0 -============ - -*2016-05-27* - -*First version on CRAN* - -Major update to the package interface and implementation: - -* Added a `submit` argument to `slurm_apply` and `slurm_call`. If `submit = FALSE`, -the submission scripts are created but not run. This is useful if the files need -to be transferred from a local machine to the cluster and run at a later time. - -* Added new optional arguments to `slurm_apply` and `slurm_call`, allowing users to give -informative names to SLURM jobs (`jobname`) and set any options understood by -`sbatch` (`slurm_options`). - -* The `data_file` arugment to `slurm_apply` and `slurm_call` is replaced with -`add_objects`, which accepts a vector of R object names from the active workspace -and automatically saves them in a .RData file to be loaded on each node. - -* `slurm_apply` and `slurm_call` now generate R and Bash scripts through -[whisker](https://github.com/edwindj/whisker) templates. Advanced users may want -to edit those templates in the `templates` folder of the installed R package -(e.g. to set default *SBATCH* options in `submit.sh`). - -* Files generated by the package (scripts, data files and output) are now saved -in a subfolder named `_rslurm_[jobname]` in the current working directory. - -* Minor updates, including reformatting the output of `print_job_status` and -removing this package's dependency on `stringr`. - - -rslurm 0.2.0 -============ - -*2015-11-23* - -* Changed the `slurm_apply` function to use `parallel::mcMap` instead of `mcmapply`, -which fixes a bug where list outputs (i.e. each function call returns a list) -would be collapsed in a single list (rather than returned as a list of lists). - -* Changed the interface so that the output type (table or raw) is now an argument -of `get_slurm_out` rather than of `slurm_apply`, and defaults to `raw`. - -* Added `cpus_per_node` argument to `slurm_apply`, indicating the number of -parallel processes to be run on each node. - - -rslurm 0.1.3 -============ - -*2015-07-13* - -* Added the `slurm_call` function, which submits a single function evaluation -on the cluster, with syntax similar to the base function `do.call`. - -* `get_slurm_out` can now process the output even if some filese are missing, -in which case it issues a warning. - - -rslurm 0.1.2 -============ - -*2015-06-29* - -* Added the optional argument `pkgs` to `slurm_apply`, indicating which packages -should be loaded on each node (by default, all packages currently attached to -the user's R session). - - -rslurm 0.1.1 -============ - -*2015-06-24* - -* Added the optional argument `output` to `slurm_apply`, which can take the -value `table` (each function evaluation returns a row, output is a data frame) or -`raw` (each function evaluation returns an arbitrary R object, output is a list). - -* Fixed a bug in the chunk size calculation for `slurm_apply`. - - -rslurm 0.1.0 -============ - -*2015-06-16* - -* First version of the package released on Github. diff --git a/inst/NEWS.rd b/inst/NEWS.rd new file mode 100644 index 0000000..3faa7a8 --- /dev/null +++ b/inst/NEWS.rd @@ -0,0 +1,88 @@ +\name{NEWS} +\title{News for Package \pkg{rslurm}} +\newcommand{\ghpr}{\href{https://github.com/sesync-ci/rslurm/pull/#1}{##1}} +\newcommand{\ghit}{\href{https://github.com/sesync-ci/rslurm/issues/#1}{##1}} + +\section{Changes in rslurm version 0.3.2 *2017-04-06*}{ +Minor update to include new feature and bug fixes. + \itemize{ + \item 'wait' argument adds option to slurm_apply and slurm_call to block the + calling script until the submitted job completes. This option can be used to allow immediate processing of a submitted job's output. + \item Use '.RDS' file extension, rather than '.RData', for serialized objects. + \item Minor bug fixes. + } +} +\section{Changes in rslurm version 0.3.1}{ + \itemize{ + \item *2016-06-18* + \item Minor bug fix: specify full path of 'Rscript' when running batch scripts. + } +} +\section{Changes in rslurm version 0.3.0}{ + \itemize{ + \item *2016-05-27* + *First version on CRAN* + Major update to the package interface and implementation: + \item Added a `submit` argument to `slurm_apply` and `slurm_call`. If `submit = FALSE`, + the submission scripts are created but not run. This is useful if the files need + to be transferred from a local machine to the cluster and run at a later time. + \item Added new optional arguments to `slurm_apply` and `slurm_call`, allowing users to give + informative names to SLURM jobs (`jobname`) and set any options understood by + `sbatch` (`slurm_options`). + \item The `data_file` arugment to `slurm_apply` and `slurm_call` is replaced with + `add_objects`, which accepts a vector of R object names from the active workspace + and automatically saves them in a .RData file to be loaded on each node. + \item `slurm_apply` and `slurm_call` now generate R and Bash scripts through + [whisker](https://github.com/edwindj/whisker) templates. Advanced users may want + to edit those templates in the `templates` folder of the installed R package + (e.g. to set default *SBATCH* options in `submit.sh`). + \item Files generated by the package (scripts, data files and output) are now saved + in a subfolder named `_rslurm_[jobname]` in the current working directory. + \item Minor updates, including reformatting the output of `print_job_status` and + removing this package's dependency on `stringr`. + } +} +\section{Changes in rslurm version 0.2.0}{ + \itemize{ + \item *2015-11-23* + \item Changed the `slurm_apply` function to use `parallel::mcMap` instead of `mcmapply`, + which fixes a bug where list outputs (i.e. each function call returns a list) + would be collapsed in a single list (rather than returned as a list of lists). + \item Changed the interface so that the output type (table or raw) is now an argument + of `get_slurm_out` rather than of `slurm_apply`, and defaults to `raw`. + \item Added `cpus_per_node` argument to `slurm_apply`, indicating the number of + parallel processes to be run on each node. + } +} +\section{Changes in rslurm version 0.1.3}{ + \itemize{ + \item *2015-07-13* + \item Added the `slurm_call` function, which submits a single function evaluation + on the cluster, with syntax similar to the base function `do.call`. + \item `get_slurm_out` can now process the output even if some filese are missing, + in which case it issues a warning. + } +} +\section{Changes in rslurm version 0.1.2}{ + \itemize{ + \item *2015-06-29* + \item Added the optional argument `pkgs` to `slurm_apply`, indicating which packages + should be loaded on each node (by default, all packages currently attached to + the user's R session). + } +} +\section{Changes in rslurm version 0.1.1}{ + \itemize{ + \item *2015-06-24* + \item Added the optional argument `output` to `slurm_apply`, which can take the + value `table` (each function evaluation returns a row, output is a data frame) or + `raw` (each function evaluation returns an arbitrary R object, output is a list). + \item Fixed a bug in the chunk size calculation for `slurm_apply`. + } +} +\section{Changes in rslurm version 0.1.0}{ + \itemize{ + \item *2015-06-16* + \item First version of the package released on Github. + } +} From 63c8ce47e1a0c4dd5b6b2595cd0da235c8b17884 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 01:49:22 -0400 Subject: [PATCH 17/25] case matters --- inst/NEWS.Rd | 61 ++++++++++++++++++++++++++++++++++++ inst/NEWS.rd | 88 ---------------------------------------------------- 2 files changed, 61 insertions(+), 88 deletions(-) create mode 100644 inst/NEWS.Rd delete mode 100644 inst/NEWS.rd diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd new file mode 100644 index 0000000..1d6abfd --- /dev/null +++ b/inst/NEWS.Rd @@ -0,0 +1,61 @@ +\name{NEWS} +\title{News for Package \pkg{rslurm}} +\newcommand{\ghpr}{\href{https://github.com/sesync-ci/rslurm/pull/#1}{##1}} +\newcommand{\ghit}{\href{https://github.com/sesync-ci/rslurm/issues/#1}{##1}} + +\section{Changes in version 0.3.2 from 2017-04-06}{ +Minor update to include new feature and bug fixes. + \itemize{ + \item 'wait' argument adds option to slurm_apply and slurm_call to block the + calling script until the submitted job completes. This option can be used to allow immediate processing of a submitted job's output (\ghpr{2}). + \item Use '.RDS' file extension, rather than '.RData', for serialized objects (\ghpr{4}). + \item Minor bug fixes (\ghpr{4}). + } +} +\section{Changes in version 0.3.1 from 2016-06-18}{ +Minor bug fix. + \itemize{ + \item specify full path of 'Rscript' when running batch scripts. + } +} +\section{Changes in version 0.3.0 from 2016-05-27}{ +Major update to the package interface and implementation. + \itemize{ + \item First version on CRAN + \item Added a `submit` argument to `slurm_apply` and `slurm_call`. If `submit = FALSE`, the submission scripts are created but not run. This is useful if the files need to be transferred from a local machine to the cluster and run at a later time. + \item Added new optional arguments to `slurm_apply` and `slurm_call`, allowing users to give informative names to SLURM jobs (`jobname`) and set any options understood by `sbatch` (`slurm_options`). + \item The `data_file` arugment to `slurm_apply` and `slurm_call` is replaced with `add_objects`, which accepts a vector of R object names from the active workspace and automatically saves them in a .RData file to be loaded on each node. + \item `slurm_apply` and `slurm_call` now generate R and Bash scripts through [whisker](https://github.com/edwindj/whisker) templates. Advanced users may want to edit those templates in the `templates` folder of the installed R package (e.g. to set default *SBATCH* options in `submit.sh`). + \item Files generated by the package (scripts, data files and output) are now saved in a subfolder named `_rslurm_[jobname]` in the current working directory. + \item Minor updates, including reformatting the output of `print_job_status` and removing this package's dependency on `stringr`. + } +} +\section{Changes in version 0.2.0 from 2015-11-23}{ + \itemize{ + \item Changed the `slurm_apply` function to use `parallel::mcMap` instead of `mcmapply`, which fixes a bug where list outputs (i.e. each function call returns a list) would be collapsed in a single list (rather than returned as a list of lists). + \item Changed the interface so that the output type (table or raw) is now an argument of `get_slurm_out` rather than of `slurm_apply`, and defaults to `raw`. + \item Added `cpus_per_node` argument to `slurm_apply`, indicating the number of parallel processes to be run on each node. + } +} +\section{Changes in version 0.1.3 from 2015-07-13}{ + \itemize{ + \item Added the `slurm_call` function, which submits a single function evaluation on the cluster, with syntax similar to the base function `do.call`. + \item `get_slurm_out` can now process the output even if some filese are missing, in which case it issues a warning. + } +} +\section{Changes in version 0.1.2 from 2015-06-29}{ + \itemize{ + \item Added the optional argument `pkgs` to `slurm_apply`, indicating which packages should be loaded on each node (by default, all packages currently attached to the user's R session). + } +} +\section{Changes in version 0.1.1 from 2015-06-24}{ + \itemize{ + \item Added the optional argument `output` to `slurm_apply`, which can take the value `table` (each function evaluation returns a row, output is a data frame) or `raw` (each function evaluation returns an arbitrary R object, output is a list). + \item Fixed a bug in the chunk size calculation for `slurm_apply`. + } +} +\section{Changes in version 0.1.0 2015-06-16}{ + \itemize{ + \item First version of the package released on Github. + } +} diff --git a/inst/NEWS.rd b/inst/NEWS.rd deleted file mode 100644 index 3faa7a8..0000000 --- a/inst/NEWS.rd +++ /dev/null @@ -1,88 +0,0 @@ -\name{NEWS} -\title{News for Package \pkg{rslurm}} -\newcommand{\ghpr}{\href{https://github.com/sesync-ci/rslurm/pull/#1}{##1}} -\newcommand{\ghit}{\href{https://github.com/sesync-ci/rslurm/issues/#1}{##1}} - -\section{Changes in rslurm version 0.3.2 *2017-04-06*}{ -Minor update to include new feature and bug fixes. - \itemize{ - \item 'wait' argument adds option to slurm_apply and slurm_call to block the - calling script until the submitted job completes. This option can be used to allow immediate processing of a submitted job's output. - \item Use '.RDS' file extension, rather than '.RData', for serialized objects. - \item Minor bug fixes. - } -} -\section{Changes in rslurm version 0.3.1}{ - \itemize{ - \item *2016-06-18* - \item Minor bug fix: specify full path of 'Rscript' when running batch scripts. - } -} -\section{Changes in rslurm version 0.3.0}{ - \itemize{ - \item *2016-05-27* - *First version on CRAN* - Major update to the package interface and implementation: - \item Added a `submit` argument to `slurm_apply` and `slurm_call`. If `submit = FALSE`, - the submission scripts are created but not run. This is useful if the files need - to be transferred from a local machine to the cluster and run at a later time. - \item Added new optional arguments to `slurm_apply` and `slurm_call`, allowing users to give - informative names to SLURM jobs (`jobname`) and set any options understood by - `sbatch` (`slurm_options`). - \item The `data_file` arugment to `slurm_apply` and `slurm_call` is replaced with - `add_objects`, which accepts a vector of R object names from the active workspace - and automatically saves them in a .RData file to be loaded on each node. - \item `slurm_apply` and `slurm_call` now generate R and Bash scripts through - [whisker](https://github.com/edwindj/whisker) templates. Advanced users may want - to edit those templates in the `templates` folder of the installed R package - (e.g. to set default *SBATCH* options in `submit.sh`). - \item Files generated by the package (scripts, data files and output) are now saved - in a subfolder named `_rslurm_[jobname]` in the current working directory. - \item Minor updates, including reformatting the output of `print_job_status` and - removing this package's dependency on `stringr`. - } -} -\section{Changes in rslurm version 0.2.0}{ - \itemize{ - \item *2015-11-23* - \item Changed the `slurm_apply` function to use `parallel::mcMap` instead of `mcmapply`, - which fixes a bug where list outputs (i.e. each function call returns a list) - would be collapsed in a single list (rather than returned as a list of lists). - \item Changed the interface so that the output type (table or raw) is now an argument - of `get_slurm_out` rather than of `slurm_apply`, and defaults to `raw`. - \item Added `cpus_per_node` argument to `slurm_apply`, indicating the number of - parallel processes to be run on each node. - } -} -\section{Changes in rslurm version 0.1.3}{ - \itemize{ - \item *2015-07-13* - \item Added the `slurm_call` function, which submits a single function evaluation - on the cluster, with syntax similar to the base function `do.call`. - \item `get_slurm_out` can now process the output even if some filese are missing, - in which case it issues a warning. - } -} -\section{Changes in rslurm version 0.1.2}{ - \itemize{ - \item *2015-06-29* - \item Added the optional argument `pkgs` to `slurm_apply`, indicating which packages - should be loaded on each node (by default, all packages currently attached to - the user's R session). - } -} -\section{Changes in rslurm version 0.1.1}{ - \itemize{ - \item *2015-06-24* - \item Added the optional argument `output` to `slurm_apply`, which can take the - value `table` (each function evaluation returns a row, output is a data frame) or - `raw` (each function evaluation returns an arbitrary R object, output is a list). - \item Fixed a bug in the chunk size calculation for `slurm_apply`. - } -} -\section{Changes in rslurm version 0.1.0}{ - \itemize{ - \item *2015-06-16* - \item First version of the package released on Github. - } -} From 6d0a0b924d6d3c1772619d37c8c333008cb79909 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 02:54:18 -0400 Subject: [PATCH 18/25] update README and cran-comments --- DESCRIPTION | 2 +- README.md | 8 +++----- cran-comments.md | 18 ++++++++++++------ document.R | 13 +++++++++---- inst/doc/rslurm.html | 22 +++++++++++----------- man/rslurm-package.Rd | 2 +- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index a36d623..9b26b24 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: rslurm Type: Package Title: Submit R Calculations to a 'SLURM' Cluster Description: Functions that simplify submitting R scripts to a 'SLURM' cluster - workload manager, in part by automating the division of emarassingly parallel + workload manager, in part by automating the division of embarrassingly parallel calculations across cluster nodes. Version: 0.3.2 License: GPL-3 diff --git a/README.md b/README.md index 7339824..460e166 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,11 @@ -| | | -|----------------|-----------------| -| rslurm-package | R Documentation | - rslurm: Submit R Calculations to a 'SLURM' Cluster -------------------------------------------------- +![](https://travis-ci.org/SESYNC-ci/rslurm.svg?branch=master) + ### Description -Functions that simplify submitting R scripts to a 'SLURM' cluster workload manager, in part by automating the division of emarassingly parallel calculations across cluster nodes. +Functions that simplify submitting R scripts to a 'SLURM' cluster workload manager, in part by automating the division of embarrassingly parallel calculations across cluster nodes. ### Overview diff --git a/cran-comments.md b/cran-comments.md index 8ee50fa..02c4ea1 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,14 +1,20 @@ -This is a minor update to the package to add a new feature, update the vignette, and fix bugs. +This is a minor update to the package to add a new feature, update the vignette, and fix structure and bugs. -The 'overview.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. +The 'rslurm.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. To prevent build errors, 'BuildVignette: no' is in the description and vignettes are matched in .Buildignore. This gives the Note, but I see no way around it. ## Tested on -Ubuntu 14.04 with R 3.3 (rstudio.sesync.org, SLURM head node) +Ubuntu 14.04 with R 3.3 (SLURM head node) Ubuntu 12.04 with R 3.3 (travis-ci) -macOS 10.12 with R 3.3 (local machine, R CMD build with '--no-build-vignettes', R CMD check with '--no-vignettes') -win-builder (release and devel, after manual removal of 'overview.Rmd' vignette) +macOS 10.12 with R 3.3 (local machine) +win-builder (release and devel) ## R CMD check -0 errors | 0 warnings | 0 notes +0 errors | 0 warnings | 1 notes + +* checking for old-style vignette sources ... NOTE +Vignette sources only in ‘inst/doc’: + ‘rslurm.Rmd’ +A ‘vignettes’ directory is required as from R 3.1.0 +and these will not be indexed nor checked \ No newline at end of file diff --git a/document.R b/document.R index 9b36c3d..5572447 100644 --- a/document.R +++ b/document.R @@ -1,10 +1,18 @@ library(tools) library(rmarkdown) library(devtools) +library(xml2) # Create a README.md for GitHub and CRAN from '../R/rlurm.R' # by way of the '../man/rslurm-package.Rd' produced by roxygen2 Rd2HTML(parse_Rd('man/rslurm-package.Rd'), out = 'README.html') +html <- read_html('README.html') +table <- xml_find_first(html, '//table') +xml_remove(table) +h2 <- xml_find_first(html, '//h2') +img <- read_xml('<img src="https://travis-ci.org/SESYNC-ci/rslurm.svg?branch=master"/>') +xml_add_sibling(h2, img, where='after') +write_html(html, 'README.html') pandoc_convert(input='README.html', to='markdown_github', output='README.md') unlink('README.html') @@ -12,7 +20,4 @@ unlink('README.html') build_vignettes() ## TODO -# figure out how to include Travis badge -# -# how does rmarkdown have NEWS in package index? -# maybe by including inst/NEWS.Rd, seee tools:::news2rd +# why is rslurm and rslurm-package in index diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html index f742db5..75b52fe 100644 --- a/inst/doc/rslurm.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551278</code></pre> +<pre><code>## Submitted batch job 551300</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 1.000131 0.09993934 -## 2 1.999793 0.19983189 -## 3 3.000108 0.29983346</code></pre> +<pre><code>## s_mu s_sd +## 1 0.9998704 0.09999728 +## 2 1.9999518 0.19974747 +## 3 3.0000175 0.30017537</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] ## s_mu s_sd -## 1.00013089 0.09993934 +## 0.99987045 0.09999728 ## ## [[2]] ## s_mu s_sd -## 1.9997930 0.1998319 +## 1.9999518 0.1997475 ## ## [[3]] ## s_mu s_sd -## 3.0001082 0.2998335</code></pre> +## 3.0000175 0.3001754</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551282</code></pre> +<pre><code>## Submitted batch job 551304</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551284</code></pre> +<pre><code>## Submitted batch job 551306</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551286</code></pre> +<pre><code>## Submitted batch job 551308</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index 590fafb..dbc20e3 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -7,7 +7,7 @@ \title{rslurm: Submit R Calculations to a 'SLURM' Cluster} \description{ Functions that simplify submitting R scripts to a 'SLURM' cluster -workload manager, in part by automating the division of emarassingly parallel +workload manager, in part by automating the division of embarrassingly parallel calculations across cluster nodes. } \section{Overview}{ From 91ef1898cfbe26ee33fd7420ba5de90c5fc5ea50 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 12:16:42 -0400 Subject: [PATCH 19/25] fix duplicate help pages in index --- .Rbuildignore | 2 +- document.R | 9 ++++++--- inst/NEWS.Rd | 14 +++++++++++--- man/rslurm-package.Rd | 1 - 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index e0e34c8..5052a1d 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,4 +3,4 @@ ^\.travis\.yml$ ^cran-comments\.md ^document\.R -^vignettes +^vignettes\rslurm\.Rmd diff --git a/document.R b/document.R index 5572447..57806ea 100644 --- a/document.R +++ b/document.R @@ -3,6 +3,9 @@ library(rmarkdown) library(devtools) library(xml2) +# Document from R/*.R +document() + # Create a README.md for GitHub and CRAN from '../R/rlurm.R' # by way of the '../man/rslurm-package.Rd' produced by roxygen2 Rd2HTML(parse_Rd('man/rslurm-package.Rd'), out = 'README.html') @@ -16,8 +19,8 @@ write_html(html, 'README.html') pandoc_convert(input='README.html', to='markdown_github', output='README.md') unlink('README.html') +# Remove duplicate documentation for rslurm and rslurm-package from index and search +system('sed -i "/alias{rslurm-package}/d" man/rslurm-package.Rd') + # Build vignettes build_vignettes() - -## TODO -# why is rslurm and rslurm-package in index diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd index 1d6abfd..ac76066 100644 --- a/inst/NEWS.Rd +++ b/inst/NEWS.Rd @@ -3,19 +3,27 @@ \newcommand{\ghpr}{\href{https://github.com/sesync-ci/rslurm/pull/#1}{##1}} \newcommand{\ghit}{\href{https://github.com/sesync-ci/rslurm/issues/#1}{##1}} +\section{Changes in version 0.3.3 from 2017-04-07}{ +Minor update to repair README. + \itemize{ + \item Modify DESCRIPTION with "BuildVignette: no". + \item Create README from R/slurm.R. + } +} + \section{Changes in version 0.3.2 from 2017-04-06}{ Minor update to include new feature and bug fixes. \itemize{ - \item 'wait' argument adds option to slurm_apply and slurm_call to block the + \item "wait" argument adds option to slurm_apply and slurm_call to block the calling script until the submitted job completes. This option can be used to allow immediate processing of a submitted job's output (\ghpr{2}). - \item Use '.RDS' file extension, rather than '.RData', for serialized objects (\ghpr{4}). + \item Use ".RDS" file extension, rather than ".RData", for serialized objects (\ghpr{4}). \item Minor bug fixes (\ghpr{4}). } } \section{Changes in version 0.3.1 from 2016-06-18}{ Minor bug fix. \itemize{ - \item specify full path of 'Rscript' when running batch scripts. + \item specify full path of `Rscript` when running batch scripts. } } \section{Changes in version 0.3.0 from 2016-05-27}{ diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd index dbc20e3..8c76085 100755 --- a/man/rslurm-package.Rd +++ b/man/rslurm-package.Rd @@ -3,7 +3,6 @@ \docType{package} \name{rslurm-package} \alias{rslurm} -\alias{rslurm-package} \title{rslurm: Submit R Calculations to a 'SLURM' Cluster} \description{ Functions that simplify submitting R scripts to a 'SLURM' cluster From 3f229ecb8464f32f309d09d35030accb8ccd2539 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 16:07:58 -0400 Subject: [PATCH 20/25] bump version; rearrange build --- .Rbuildignore | 4 +- DESCRIPTION | 3 +- cran-comments.md | 4 +- inst/doc/rslurm.R | 56 ----------- inst/doc/rslurm.Rmd | 214 ---------------------------------------- inst/doc/rslurm.html | 188 ----------------------------------- man/rslurm-package.Rd | 0 document.R => release.R | 5 +- 8 files changed, 8 insertions(+), 466 deletions(-) delete mode 100644 inst/doc/rslurm.R delete mode 100644 inst/doc/rslurm.Rmd delete mode 100644 inst/doc/rslurm.html mode change 100755 => 100644 man/rslurm-package.Rd rename document.R => release.R (95%) diff --git a/.Rbuildignore b/.Rbuildignore index 5052a1d..0362ee0 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,5 +2,5 @@ ^\.Rproj\.user$ ^\.travis\.yml$ ^cran-comments\.md -^document\.R -^vignettes\rslurm\.Rmd +^release\.R +^vignettes/rslurm\.Rmd diff --git a/DESCRIPTION b/DESCRIPTION index 9b26b24..2758526 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -4,7 +4,7 @@ Title: Submit R Calculations to a 'SLURM' Cluster Description: Functions that simplify submitting R scripts to a 'SLURM' cluster workload manager, in part by automating the division of embarrassingly parallel calculations across cluster nodes. -Version: 0.3.2 +Version: 0.3.3 License: GPL-3 URL: https://github.com/SESYNC-ci/rslurm BugReports: https://github.com/SESYNC-ci/rslurm/issues @@ -23,4 +23,3 @@ Suggests: knitr, rmarkdown VignetteBuilder: knitr -BuildVignettes: no diff --git a/cran-comments.md b/cran-comments.md index 02c4ea1..7db3606 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,6 +1,6 @@ -This is a minor update to the package to add a new feature, update the vignette, and fix structure and bugs. +This is a minor update to the package to update the README. -The 'rslurm.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. To prevent build errors, 'BuildVignette: no' is in the description and vignettes are matched in .Buildignore. This gives the Note, but I see no way around it. +The 'rslurm.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. To prevent build errors, 'BuildVignette: no' is in the description and vignettes are matched in .Buildignore. The source for the vignette is copied into 'inst/doc' for FOSS compliance. This gives the NOTE, but I see no way around it. ## Tested on diff --git a/inst/doc/rslurm.R b/inst/doc/rslurm.R deleted file mode 100644 index f2f1994..0000000 --- a/inst/doc/rslurm.R +++ /dev/null @@ -1,56 +0,0 @@ -## ------------------------------------------------------------------------ -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} - -## ------------------------------------------------------------------------ -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) - -## ------------------------------------------------------------------------ -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) - -## ------------------------------------------------------------------------ -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) - -## ------------------------------------------------------------------------ -res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) -res_raw[1:3] - -## ------------------------------------------------------------------------ -dir("_rslurm_test_job") - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ------------------------------------------------------------------------ -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ----echo=FALSE---------------------------------------------------------- -obj_list <- list(NULL) -func <- function(obj) {} - -## ------------------------------------------------------------------------ -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - -## ------------------------------------------------------------------------ -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) - -## ----echo=FALSE---------------------------------------------------------- -cleanup_files(sjob) - diff --git a/inst/doc/rslurm.Rmd b/inst/doc/rslurm.Rmd deleted file mode 100644 index 416ee47..0000000 --- a/inst/doc/rslurm.Rmd +++ /dev/null @@ -1,214 +0,0 @@ ---- -title: "Parallelize R code on a SLURM cluster" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{Parallelize R code on a SLURM cluster} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -Many computing-intensive processes in R involve the repeated evaluation of -a function over many items or parameter sets. These so-called -[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) -calculations can be run serially with the `lapply` or `Map` function, or in parallel -on a single machine with `mclapply` or `mcMap` (from the **parallel** package). - -The rslurm package simplifies the process of distributing this type of calculation -across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) -workload manager. Its main function, `slurm_apply`, automatically divides the -computation over multiple nodes and writes the necessary submission scripts. -It also includes functions to retrieve and combine the output from different nodes, -as well as wrappers for common SLURM commands. - -### Table of contents - -- [Basic example](#basic-example) -- [Single function evaluation](#single-function-evaluation) -- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) -- [Configuring SLURM options](#configuring-slurm-options) -- [Generating scripts for later submission](#generating-scripts-for-later-submission) -- [How it works / advanced customization](#how-it-works-advanced-customization) - - -## Basic example - -To illustrate a typical rslurm workflow, we use a simple function that takes -a mean and standard deviation as parameters, generates a million normal deviates -and returns the sample mean and standard deviation. - -```{r} -test_func <- function(par_mu, par_sd) { - samp <- rnorm(10^6, par_mu, par_sd) - c(s_mu = mean(samp), s_sd = sd(samp)) -} -``` - -We then create a parameter data frame where each row is a parameter set and each -column matches an argument of the function. - -```{r} -pars <- data.frame(par_mu = 1:10, - par_sd = seq(0.1, 1, length.out = 10)) -head(pars, 3) -``` - -We can now pass that function and the parameters data frame to `slurm_apply`, -specifiying the number of cluster nodes to use and the number of CPUs per node. -The latter (`cpus_per_node`) determines how many processes will be forked on -each node, as the `mc.cores` argument of `parallel::mcMap`. -```{r} -library(rslurm) -sjob <- slurm_apply(test_func, pars, jobname = "test_job", - nodes = 2, cpus_per_node = 2) -``` -The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of -information (job name and number of nodes) needed to retrieve the job's output. - -Assuming the function is run on a machine with access to the cluster, it also -prints a message confirming the job has been submitted to SLURM. - -Particular clusters may require the specification of additional SLURM options, -such as time and memory limits for the job. Also, when running R on a local -machine without direct cluster access, you may want to generate scripts to be -copied to the cluster and run at a later time. These topics are covered in -additional sections below this basic example. - -After the job has been submitted, you can call `print_job_status` to display its -status (in queue, running or completed) or call `cancel_slurm` to cancel its -execution. These functions are R wrappers for the SLURM command line functions -`squeue` and `scancel`, respectively. - -Once the job completes, `get_slurm_out` reads and combines the output from all -nodes. -```{r} -res <- get_slurm_out(sjob, outtype = "table") -head(res, 3) -``` - -When `outtype = "table"`, the outputs from each function evaluation are -row-bound into a single data frame; this is an appropriate format when the -function returns a simple vector. The default `outtype = "raw"` combines the -outputs into a list and can thus handle arbitrarily complex return objects. - -```{r} -res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) -res_raw[1:3] -``` - -The files generated by `slurm_apply` are saved in a folder named -*\_rslurm_[jobname]* under the current working directory. - -```{r} -dir("_rslurm_test_job") -``` - -The utility function `cleanup_files` deletes the temporary folder for the -specified *slurm_job*. - -```{r echo=FALSE} -cleanup_files(sjob) -``` - - -## Single function evaluation - -In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which -sends a single function call to the cluster. It is analogous in syntax to the -base R function `do.call`, accepting a function and a named list of parameters -as arguments. - -```{r} -sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` - -Because `slurm_call` involves a single process on a single node, it does not -recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the -same additional arguments (detailed in the sections below) as `slurm_apply`. - - -## Adding auxiliary data and functions - -The function passed to `slurm_apply` can only receive atomic parameters stored -within a data frame. Suppose we want instead to apply a function `func` to a list -of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap -`func` in an inline function that takes an integer parameter. - -```{r echo=FALSE} -obj_list <- list(NULL) -func <- function(obj) {} -``` -```{r} -sjob <- slurm_apply(function(i) func(obj_list[[i]]), - data.frame(i = seq_along(obj_list)), - add_objects = c("func", "obj_list"), - nodes = 2, cpus_per_node = 2) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` - -The `add_objects` argument specifies the names of any R objects (besides the -parameters data frame) that must be accessed by the function passed to -`slurm_apply`. These objects are saved to a `.RData` file that is loaded -on each cluster node prior to evaluating the function in parallel. - -By default, all R packages attached to the current R session will also be -attached (with `library`) on each cluster node, though this can be modified with -the optional `pkgs` argument. - - -## Configuring SLURM options - -The `slurm_options` argument allows you to set any of the command line -options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the -SLURM `sbatch` command. It should be formatted as a named list, using the long -names of each option (e.g. "time" rather than "t"). Flags, i.e. command line -options that are toggled rather than set to a particular value, should be set to -`TRUE` in `slurm_options`. For example, the following code: -```{r} -sjob <- slurm_apply(test_func, pars, - slurm_options = list(time = "1:00:00", share = TRUE)) -``` -```{r echo=FALSE} -cleanup_files(sjob) -``` -sets the command line options `--time=1:00:00 --share`. - - -## Generating scripts for later submission - -When working from a R session without direct access to the cluster, you can set -`submit = FALSE` within `slurm_apply`. The function will create the -*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without -submitting the job. You may then copy those files to the cluster and submit the -job manually by calling `sbatch submit.sh` from the command line. - - -## How it works / advanced customization - -As mentioned above, the `slurm_apply` function creates a job-specific folder. -This folder contains the parameters as a *.RDS* file and (if applicable) the objects -specified as `add_objects` saved together in a *.RData* file. The function also -generates a R script (`slurm_run.R`) to be run on each cluster node, as well -as a Bash script (`submit.sh`) to submit the job to SLURM. - -More specifically, the Bash script creates a SLURM job array, with each cluster -node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment -variable. This variable is read by `slurm_run.R`, which allows each instance of -the script to operate on a different parameter subset and write its output to -a different results file. The R script calls `parallel::mcMap` to parallelize -calculations on each node. - -Both `slurm_run.R` and `submit.sh` are generated from templates, using the -**whisker** package; these templates can be found in the `rslurm/templates` -subfolder in your R package library. There are two templates for each script, -one for `slurm_apply` and the other (with the word *single* in its title) for -`slurm_call`. - -While you should avoid changing any existing lines in the template scripts, you -may want to add `#SBATCH` lines to the `submit.sh` templates in order to -permanently set certain SLURM command line options and thus customize the package -to your particular cluster setup. \ No newline at end of file diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html deleted file mode 100644 index 75b52fe..0000000 --- a/inst/doc/rslurm.html +++ /dev/null @@ -1,188 +0,0 @@ -<!DOCTYPE html> - -<html xmlns="http://www.w3.org/1999/xhtml"> - -<head> - -<meta charset="utf-8"> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<meta name="generator" content="pandoc" /> - -<meta name="viewport" content="width=device-width, initial-scale=1"> - - - -<title>Parallelize R code on a SLURM cluster</title> - - - -<style type="text/css">code{white-space: pre;}</style> -<style type="text/css"> -div.sourceCode { overflow-x: auto; } -table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode { - margin: 0; padding: 0; vertical-align: baseline; border: none; } -table.sourceCode { width: 100%; line-height: 100%; } -td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; } -td.sourceCode { padding-left: 5px; } -code > span.kw { color: #007020; font-weight: bold; } /* Keyword */ -code > span.dt { color: #902000; } /* DataType */ -code > span.dv { color: #40a070; } /* DecVal */ -code > span.bn { color: #40a070; } /* BaseN */ -code > span.fl { color: #40a070; } /* Float */ -code > span.ch { color: #4070a0; } /* Char */ -code > span.st { color: #4070a0; } /* String */ -code > span.co { color: #60a0b0; font-style: italic; } /* Comment */ -code > span.ot { color: #007020; } /* Other */ -code > span.al { color: #ff0000; font-weight: bold; } /* Alert */ -code > span.fu { color: #06287e; } /* Function */ -code > span.er { color: #ff0000; font-weight: bold; } /* Error */ -code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */ -code > span.cn { color: #880000; } /* Constant */ -code > span.sc { color: #4070a0; } /* SpecialChar */ -code > span.vs { color: #4070a0; } /* VerbatimString */ -code > span.ss { color: #bb6688; } /* SpecialString */ -code > span.im { } /* Import */ -code > span.va { color: #19177c; } /* Variable */ -code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */ -code > span.op { color: #666666; } /* Operator */ -code > span.bu { } /* BuiltIn */ -code > span.ex { } /* Extension */ -code > span.pp { color: #bc7a00; } /* Preprocessor */ -code > span.at { color: #7d9029; } /* Attribute */ -code > span.do { color: #ba2121; font-style: italic; } /* Documentation */ -code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */ -code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */ -code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */ -</style> - - - -<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" /> - -</head> - -<body> - - - - -<h1 class="title toc-ignore">Parallelize R code on a SLURM cluster</h1> - - - -<p>Many computing-intensive processes in R involve the repeated evaluation of a function over many items or parameter sets. These so-called <a href="https://en.wikipedia.org/wiki/Embarrassingly_parallel">embarrassingly parallel</a> calculations can be run serially with the <code>lapply</code> or <code>Map</code> function, or in parallel on a single machine with <code>mclapply</code> or <code>mcMap</code> (from the <strong>parallel</strong> package).</p> -<p>The rslurm package simplifies the process of distributing this type of calculation across a computing cluster that uses the <a href="http://slurm.schedmd.com/">SLURM</a> workload manager. Its main function, <code>slurm_apply</code>, automatically divides the computation over multiple nodes and writes the necessary submission scripts. It also includes functions to retrieve and combine the output from different nodes, as well as wrappers for common SLURM commands.</p> -<div id="table-of-contents" class="section level3"> -<h3>Table of contents</h3> -<ul> -<li><a href="#basic-example">Basic example</a></li> -<li><a href="#single-function-evaluation">Single function evaluation</a></li> -<li><a href="#adding-auxiliary-data-and-functions">Adding auxiliary data and functions</a></li> -<li><a href="#configuring-slurm-options">Configuring SLURM options</a></li> -<li><a href="#generating-scripts-for-later-submission">Generating scripts for later submission</a></li> -<li><a href="#how-it-works-advanced-customization">How it works / advanced customization</a></li> -</ul> -</div> -<div id="basic-example" class="section level2"> -<h2>Basic example</h2> -<p>To illustrate a typical rslurm workflow, we use a simple function that takes a mean and standard deviation as parameters, generates a million normal deviates and returns the sample mean and standard deviation.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">test_func <-<span class="st"> </span>function(par_mu, par_sd) { - samp <-<span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">10</span>^<span class="dv">6</span>, par_mu, par_sd) - <span class="kw">c</span>(<span class="dt">s_mu =</span> <span class="kw">mean</span>(samp), <span class="dt">s_sd =</span> <span class="kw">sd</span>(samp)) -}</code></pre></div> -<p>We then create a parameter data frame where each row is a parameter set and each column matches an argument of the function.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">pars <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">par_mu =</span> <span class="dv">1</span>:<span class="dv">10</span>, - <span class="dt">par_sd =</span> <span class="kw">seq</span>(<span class="fl">0.1</span>, <span class="dv">1</span>, <span class="dt">length.out =</span> <span class="dv">10</span>)) -<span class="kw">head</span>(pars, <span class="dv">3</span>)</code></pre></div> -<pre><code>## par_mu par_sd -## 1 1 0.1 -## 2 2 0.2 -## 3 3 0.3</code></pre> -<p>We can now pass that function and the parameters data frame to <code>slurm_apply</code>, specifiying the number of cluster nodes to use and the number of CPUs per node. The latter (<code>cpus_per_node</code>) determines how many processes will be forked on each node, as the <code>mc.cores</code> argument of <code>parallel::mcMap</code>.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) -sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, - <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551300</code></pre> -<p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> -<p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> -<p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> -<p>After the job has been submitted, you can call <code>print_job_status</code> to display its status (in queue, running or completed) or call <code>cancel_slurm</code> to cancel its execution. These functions are R wrappers for the SLURM command line functions <code>squeue</code> and <code>scancel</code>, respectively.</p> -<p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) -<span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9998704 0.09999728 -## 2 1.9999518 0.19974747 -## 3 3.0000175 0.30017537</code></pre> -<p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) -res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> -<pre><code>## [[1]] -## s_mu s_sd -## 0.99987045 0.09999728 -## -## [[2]] -## s_mu s_sd -## 1.9999518 0.1997475 -## -## [[3]] -## s_mu s_sd -## 3.0000175 0.3001754</code></pre> -<p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> -<pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" -## [5] "slurm_1.out" "slurm_run.R" "submit.sh"</code></pre> -<p>The utility function <code>cleanup_files</code> deletes the temporary folder for the specified <em>slurm_job</em>.</p> -</div> -<div id="single-function-evaluation" class="section level2"> -<h2>Single function evaluation</h2> -<p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551304</code></pre> -<p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> -</div> -<div id="adding-auxiliary-data-and-functions" class="section level2"> -<h2>Adding auxiliary data and functions</h2> -<p>The function passed to <code>slurm_apply</code> can only receive atomic parameters stored within a data frame. Suppose we want instead to apply a function <code>func</code> to a list of complex R objects, <code>obj_list</code>. To use <code>slurm_apply</code> in this case, we can wrap <code>func</code> in an inline function that takes an integer parameter.</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(function(i) <span class="kw">func</span>(obj_list[[i]]), - <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), - <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), - <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551306</code></pre> -<p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> -<p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> -</div> -<div id="configuring-slurm-options" class="section level2"> -<h2>Configuring SLURM options</h2> -<p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> -<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, - <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551308</code></pre> -<p>sets the command line options <code>--time=1:00:00 --share</code>.</p> -</div> -<div id="generating-scripts-for-later-submission" class="section level2"> -<h2>Generating scripts for later submission</h2> -<p>When working from a R session without direct access to the cluster, you can set <code>submit = FALSE</code> within <code>slurm_apply</code>. The function will create the <em>_rslurm_[jobname]</em> folder and generate the scripts and .RData files, without submitting the job. You may then copy those files to the cluster and submit the job manually by calling <code>sbatch submit.sh</code> from the command line.</p> -</div> -<div id="how-it-works-advanced-customization" class="section level2"> -<h2>How it works / advanced customization</h2> -<p>As mentioned above, the <code>slurm_apply</code> function creates a job-specific folder. This folder contains the parameters as a <em>.RDS</em> file and (if applicable) the objects specified as <code>add_objects</code> saved together in a <em>.RData</em> file. The function also generates a R script (<code>slurm_run.R</code>) to be run on each cluster node, as well as a Bash script (<code>submit.sh</code>) to submit the job to SLURM.</p> -<p>More specifically, the Bash script creates a SLURM job array, with each cluster node receiving a different value of the <em>SLURM_ARRAY_TASK_ID</em> environment variable. This variable is read by <code>slurm_run.R</code>, which allows each instance of the script to operate on a different parameter subset and write its output to a different results file. The R script calls <code>parallel::mcMap</code> to parallelize calculations on each node.</p> -<p>Both <code>slurm_run.R</code> and <code>submit.sh</code> are generated from templates, using the <strong>whisker</strong> package; these templates can be found in the <code>rslurm/templates</code> subfolder in your R package library. There are two templates for each script, one for <code>slurm_apply</code> and the other (with the word <em>single</em> in its title) for <code>slurm_call</code>.</p> -<p>While you should avoid changing any existing lines in the template scripts, you may want to add <code>#SBATCH</code> lines to the <code>submit.sh</code> templates in order to permanently set certain SLURM command line options and thus customize the package to your particular cluster setup.</p> -</div> - - - -<!-- dynamically load mathjax for compatibility with self-contained --> -<script> - (function () { - var script = document.createElement("script"); - script.type = "text/javascript"; - script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; - document.getElementsByTagName("head")[0].appendChild(script); - })(); -</script> - -</body> -</html> diff --git a/man/rslurm-package.Rd b/man/rslurm-package.Rd old mode 100755 new mode 100644 diff --git a/document.R b/release.R similarity index 95% rename from document.R rename to release.R index 57806ea..0330527 100644 --- a/document.R +++ b/release.R @@ -22,5 +22,6 @@ unlink('README.html') # Remove duplicate documentation for rslurm and rslurm-package from index and search system('sed -i "/alias{rslurm-package}/d" man/rslurm-package.Rd') -# Build vignettes -build_vignettes() +# Build +pkg <- build() +check_built(pkg) From 9af5efb92241d9e3bf4d82c590b90ca044c4b0dc Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 16:17:36 -0400 Subject: [PATCH 21/25] travis: no build vignettes --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b79e15a..9699bf9 100755 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: r warnings_are_errors: true sudo: required +r_build_args: "--no-build-vignettes" env: global: From 0d50d3ff7416260e6efc42427cdeae0f9ef998aa Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 17:03:50 -0400 Subject: [PATCH 22/25] add vignette index.html --- inst/doc/index.html | 29 ++++++ inst/doc/rslurm.R | 56 +++++++++++ inst/doc/rslurm.Rmd | 214 +++++++++++++++++++++++++++++++++++++++++++ inst/doc/rslurm.html | 188 +++++++++++++++++++++++++++++++++++++ release.R | 13 ++- 5 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 inst/doc/index.html create mode 100644 inst/doc/rslurm.R create mode 100644 inst/doc/rslurm.Rmd create mode 100644 inst/doc/rslurm.html diff --git a/inst/doc/index.html b/inst/doc/index.html new file mode 100644 index 0000000..9afa307 --- /dev/null +++ b/inst/doc/index.html @@ -0,0 +1,29 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> + <title>rslurm User Guides</title> + <link rel="stylesheet" type="text/css" href="../../R.css"> +</head> +<body> + +<h1>Submit R Calculations to a 'SLURM' Cluster <img class="toplogo" src="../../../doc/html/logo.jpg" alt="[R logo]" width="101" height="77"></h1> +<hr> + +<h2>User Guides and Package Vignettes</h2> +<ul> + <li> + <strong>Parallelize R code on a SLURM cluster</strong> + [<a href="./rslurm.html">HTML</a>] + [<a href="./rslurm.Rmd">source</a>] + [<a href="./rslurm.R">R code</a>] + </li> +</ul> +<br> + +<hr> +<div align="center"><a href="../html/00Index.html">[Package Contents]</a></div> +<br> +<div align="center"><small>This page was generated by the <a href="http://cran.r-project.org/web/packages/R.rsp/">R.rsp</a> package.</small></div> + +</body> +</html> diff --git a/inst/doc/rslurm.R b/inst/doc/rslurm.R new file mode 100644 index 0000000..f2f1994 --- /dev/null +++ b/inst/doc/rslurm.R @@ -0,0 +1,56 @@ +## ------------------------------------------------------------------------ +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} + +## ------------------------------------------------------------------------ +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) + +## ------------------------------------------------------------------------ +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) + +## ------------------------------------------------------------------------ +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) + +## ------------------------------------------------------------------------ +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] + +## ------------------------------------------------------------------------ +dir("_rslurm_test_job") + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ----echo=FALSE---------------------------------------------------------- +obj_list <- list(NULL) +func <- function(obj) {} + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + +## ------------------------------------------------------------------------ +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) + +## ----echo=FALSE---------------------------------------------------------- +cleanup_files(sjob) + diff --git a/inst/doc/rslurm.Rmd b/inst/doc/rslurm.Rmd new file mode 100644 index 0000000..416ee47 --- /dev/null +++ b/inst/doc/rslurm.Rmd @@ -0,0 +1,214 @@ +--- +title: "Parallelize R code on a SLURM cluster" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Parallelize R code on a SLURM cluster} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +Many computing-intensive processes in R involve the repeated evaluation of +a function over many items or parameter sets. These so-called +[embarrassingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) +calculations can be run serially with the `lapply` or `Map` function, or in parallel +on a single machine with `mclapply` or `mcMap` (from the **parallel** package). + +The rslurm package simplifies the process of distributing this type of calculation +across a computing cluster that uses the [SLURM](http://slurm.schedmd.com/) +workload manager. Its main function, `slurm_apply`, automatically divides the +computation over multiple nodes and writes the necessary submission scripts. +It also includes functions to retrieve and combine the output from different nodes, +as well as wrappers for common SLURM commands. + +### Table of contents + +- [Basic example](#basic-example) +- [Single function evaluation](#single-function-evaluation) +- [Adding auxiliary data and functions](#adding-auxiliary-data-and-functions) +- [Configuring SLURM options](#configuring-slurm-options) +- [Generating scripts for later submission](#generating-scripts-for-later-submission) +- [How it works / advanced customization](#how-it-works-advanced-customization) + + +## Basic example + +To illustrate a typical rslurm workflow, we use a simple function that takes +a mean and standard deviation as parameters, generates a million normal deviates +and returns the sample mean and standard deviation. + +```{r} +test_func <- function(par_mu, par_sd) { + samp <- rnorm(10^6, par_mu, par_sd) + c(s_mu = mean(samp), s_sd = sd(samp)) +} +``` + +We then create a parameter data frame where each row is a parameter set and each +column matches an argument of the function. + +```{r} +pars <- data.frame(par_mu = 1:10, + par_sd = seq(0.1, 1, length.out = 10)) +head(pars, 3) +``` + +We can now pass that function and the parameters data frame to `slurm_apply`, +specifiying the number of cluster nodes to use and the number of CPUs per node. +The latter (`cpus_per_node`) determines how many processes will be forked on +each node, as the `mc.cores` argument of `parallel::mcMap`. +```{r} +library(rslurm) +sjob <- slurm_apply(test_func, pars, jobname = "test_job", + nodes = 2, cpus_per_node = 2) +``` +The output of `slurm_apply` is a *slurm_job* object that stores a few pieces of +information (job name and number of nodes) needed to retrieve the job's output. + +Assuming the function is run on a machine with access to the cluster, it also +prints a message confirming the job has been submitted to SLURM. + +Particular clusters may require the specification of additional SLURM options, +such as time and memory limits for the job. Also, when running R on a local +machine without direct cluster access, you may want to generate scripts to be +copied to the cluster and run at a later time. These topics are covered in +additional sections below this basic example. + +After the job has been submitted, you can call `print_job_status` to display its +status (in queue, running or completed) or call `cancel_slurm` to cancel its +execution. These functions are R wrappers for the SLURM command line functions +`squeue` and `scancel`, respectively. + +Once the job completes, `get_slurm_out` reads and combines the output from all +nodes. +```{r} +res <- get_slurm_out(sjob, outtype = "table") +head(res, 3) +``` + +When `outtype = "table"`, the outputs from each function evaluation are +row-bound into a single data frame; this is an appropriate format when the +function returns a simple vector. The default `outtype = "raw"` combines the +outputs into a list and can thus handle arbitrarily complex return objects. + +```{r} +res_raw <- get_slurm_out(sjob, outtype = "raw", wait = FALSE) +res_raw[1:3] +``` + +The files generated by `slurm_apply` are saved in a folder named +*\_rslurm_[jobname]* under the current working directory. + +```{r} +dir("_rslurm_test_job") +``` + +The utility function `cleanup_files` deletes the temporary folder for the +specified *slurm_job*. + +```{r echo=FALSE} +cleanup_files(sjob) +``` + + +## Single function evaluation + +In addition to `slurm_apply`, rslurm also defines a `slurm_call` function, which +sends a single function call to the cluster. It is analogous in syntax to the +base R function `do.call`, accepting a function and a named list of parameters +as arguments. + +```{r} +sjob <- slurm_call(test_func, list(par_mu = 5, par_sd = 1)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +Because `slurm_call` involves a single process on a single node, it does not +recognize the `nodes` and `cpus_per_node` arguments; otherwise, it accepts the +same additional arguments (detailed in the sections below) as `slurm_apply`. + + +## Adding auxiliary data and functions + +The function passed to `slurm_apply` can only receive atomic parameters stored +within a data frame. Suppose we want instead to apply a function `func` to a list +of complex R objects, `obj_list`. To use `slurm_apply` in this case, we can wrap +`func` in an inline function that takes an integer parameter. + +```{r echo=FALSE} +obj_list <- list(NULL) +func <- function(obj) {} +``` +```{r} +sjob <- slurm_apply(function(i) func(obj_list[[i]]), + data.frame(i = seq_along(obj_list)), + add_objects = c("func", "obj_list"), + nodes = 2, cpus_per_node = 2) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` + +The `add_objects` argument specifies the names of any R objects (besides the +parameters data frame) that must be accessed by the function passed to +`slurm_apply`. These objects are saved to a `.RData` file that is loaded +on each cluster node prior to evaluating the function in parallel. + +By default, all R packages attached to the current R session will also be +attached (with `library`) on each cluster node, though this can be modified with +the optional `pkgs` argument. + + +## Configuring SLURM options + +The `slurm_options` argument allows you to set any of the command line +options ([view list](http://slurm.schedmd.com/sbatch.html)) recognized by the +SLURM `sbatch` command. It should be formatted as a named list, using the long +names of each option (e.g. "time" rather than "t"). Flags, i.e. command line +options that are toggled rather than set to a particular value, should be set to +`TRUE` in `slurm_options`. For example, the following code: +```{r} +sjob <- slurm_apply(test_func, pars, + slurm_options = list(time = "1:00:00", share = TRUE)) +``` +```{r echo=FALSE} +cleanup_files(sjob) +``` +sets the command line options `--time=1:00:00 --share`. + + +## Generating scripts for later submission + +When working from a R session without direct access to the cluster, you can set +`submit = FALSE` within `slurm_apply`. The function will create the +*\_rslurm\_[jobname]* folder and generate the scripts and .RData files, without +submitting the job. You may then copy those files to the cluster and submit the +job manually by calling `sbatch submit.sh` from the command line. + + +## How it works / advanced customization + +As mentioned above, the `slurm_apply` function creates a job-specific folder. +This folder contains the parameters as a *.RDS* file and (if applicable) the objects +specified as `add_objects` saved together in a *.RData* file. The function also +generates a R script (`slurm_run.R`) to be run on each cluster node, as well +as a Bash script (`submit.sh`) to submit the job to SLURM. + +More specifically, the Bash script creates a SLURM job array, with each cluster +node receiving a different value of the *SLURM\_ARRAY\_TASK\_ID* environment +variable. This variable is read by `slurm_run.R`, which allows each instance of +the script to operate on a different parameter subset and write its output to +a different results file. The R script calls `parallel::mcMap` to parallelize +calculations on each node. + +Both `slurm_run.R` and `submit.sh` are generated from templates, using the +**whisker** package; these templates can be found in the `rslurm/templates` +subfolder in your R package library. There are two templates for each script, +one for `slurm_apply` and the other (with the word *single* in its title) for +`slurm_call`. + +While you should avoid changing any existing lines in the template scripts, you +may want to add `#SBATCH` lines to the `submit.sh` templates in order to +permanently set certain SLURM command line options and thus customize the package +to your particular cluster setup. \ No newline at end of file diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html new file mode 100644 index 0000000..ced4cf5 --- /dev/null +++ b/inst/doc/rslurm.html @@ -0,0 +1,188 @@ +<!DOCTYPE html> + +<html xmlns="http://www.w3.org/1999/xhtml"> + +<head> + +<meta charset="utf-8"> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="pandoc" /> + +<meta name="viewport" content="width=device-width, initial-scale=1"> + + + +<title>Parallelize R code on a SLURM cluster</title> + + + +<style type="text/css">code{white-space: pre;}</style> +<style type="text/css"> +div.sourceCode { overflow-x: auto; } +table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode { + margin: 0; padding: 0; vertical-align: baseline; border: none; } +table.sourceCode { width: 100%; line-height: 100%; } +td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; } +td.sourceCode { padding-left: 5px; } +code > span.kw { color: #007020; font-weight: bold; } /* Keyword */ +code > span.dt { color: #902000; } /* DataType */ +code > span.dv { color: #40a070; } /* DecVal */ +code > span.bn { color: #40a070; } /* BaseN */ +code > span.fl { color: #40a070; } /* Float */ +code > span.ch { color: #4070a0; } /* Char */ +code > span.st { color: #4070a0; } /* String */ +code > span.co { color: #60a0b0; font-style: italic; } /* Comment */ +code > span.ot { color: #007020; } /* Other */ +code > span.al { color: #ff0000; font-weight: bold; } /* Alert */ +code > span.fu { color: #06287e; } /* Function */ +code > span.er { color: #ff0000; font-weight: bold; } /* Error */ +code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */ +code > span.cn { color: #880000; } /* Constant */ +code > span.sc { color: #4070a0; } /* SpecialChar */ +code > span.vs { color: #4070a0; } /* VerbatimString */ +code > span.ss { color: #bb6688; } /* SpecialString */ +code > span.im { } /* Import */ +code > span.va { color: #19177c; } /* Variable */ +code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */ +code > span.op { color: #666666; } /* Operator */ +code > span.bu { } /* BuiltIn */ +code > span.ex { } /* Extension */ +code > span.pp { color: #bc7a00; } /* Preprocessor */ +code > span.at { color: #7d9029; } /* Attribute */ +code > span.do { color: #ba2121; font-style: italic; } /* Documentation */ +code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */ +code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */ +code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */ +</style> + + + +<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" /> + +</head> + +<body> + + + + +<h1 class="title toc-ignore">Parallelize R code on a SLURM cluster</h1> + + + +<p>Many computing-intensive processes in R involve the repeated evaluation of a function over many items or parameter sets. These so-called <a href="https://en.wikipedia.org/wiki/Embarrassingly_parallel">embarrassingly parallel</a> calculations can be run serially with the <code>lapply</code> or <code>Map</code> function, or in parallel on a single machine with <code>mclapply</code> or <code>mcMap</code> (from the <strong>parallel</strong> package).</p> +<p>The rslurm package simplifies the process of distributing this type of calculation across a computing cluster that uses the <a href="http://slurm.schedmd.com/">SLURM</a> workload manager. Its main function, <code>slurm_apply</code>, automatically divides the computation over multiple nodes and writes the necessary submission scripts. It also includes functions to retrieve and combine the output from different nodes, as well as wrappers for common SLURM commands.</p> +<div id="table-of-contents" class="section level3"> +<h3>Table of contents</h3> +<ul> +<li><a href="#basic-example">Basic example</a></li> +<li><a href="#single-function-evaluation">Single function evaluation</a></li> +<li><a href="#adding-auxiliary-data-and-functions">Adding auxiliary data and functions</a></li> +<li><a href="#configuring-slurm-options">Configuring SLURM options</a></li> +<li><a href="#generating-scripts-for-later-submission">Generating scripts for later submission</a></li> +<li><a href="#how-it-works-advanced-customization">How it works / advanced customization</a></li> +</ul> +</div> +<div id="basic-example" class="section level2"> +<h2>Basic example</h2> +<p>To illustrate a typical rslurm workflow, we use a simple function that takes a mean and standard deviation as parameters, generates a million normal deviates and returns the sample mean and standard deviation.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">test_func <-<span class="st"> </span>function(par_mu, par_sd) { + samp <-<span class="st"> </span><span class="kw">rnorm</span>(<span class="dv">10</span>^<span class="dv">6</span>, par_mu, par_sd) + <span class="kw">c</span>(<span class="dt">s_mu =</span> <span class="kw">mean</span>(samp), <span class="dt">s_sd =</span> <span class="kw">sd</span>(samp)) +}</code></pre></div> +<p>We then create a parameter data frame where each row is a parameter set and each column matches an argument of the function.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">pars <-<span class="st"> </span><span class="kw">data.frame</span>(<span class="dt">par_mu =</span> <span class="dv">1</span>:<span class="dv">10</span>, + <span class="dt">par_sd =</span> <span class="kw">seq</span>(<span class="fl">0.1</span>, <span class="dv">1</span>, <span class="dt">length.out =</span> <span class="dv">10</span>)) +<span class="kw">head</span>(pars, <span class="dv">3</span>)</code></pre></div> +<pre><code>## par_mu par_sd +## 1 1 0.1 +## 2 2 0.2 +## 3 3 0.3</code></pre> +<p>We can now pass that function and the parameters data frame to <code>slurm_apply</code>, specifiying the number of cluster nodes to use and the number of CPUs per node. The latter (<code>cpus_per_node</code>) determines how many processes will be forked on each node, as the <code>mc.cores</code> argument of <code>parallel::mcMap</code>.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) +sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 551564</code></pre> +<p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> +<p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> +<p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> +<p>After the job has been submitted, you can call <code>print_job_status</code> to display its status (in queue, running or completed) or call <code>cancel_slurm</code> to cancel its execution. These functions are R wrappers for the SLURM command line functions <code>squeue</code> and <code>scancel</code>, respectively.</p> +<p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) +<span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> +<pre><code>## s_mu s_sd +## 1 0.9999407 0.1001478 +## 2 2.0001493 0.2000370 +## 3 3.0005155 0.2999715</code></pre> +<p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) +res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> +<pre><code>## [[1]] +## s_mu s_sd +## 0.9999407 0.1001478 +## +## [[2]] +## s_mu s_sd +## 2.000149 0.200037 +## +## [[3]] +## s_mu s_sd +## 3.0005155 0.2999715</code></pre> +<p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> +<pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" +## [5] "slurm_1.out" "slurm_run.R" "submit.sh"</code></pre> +<p>The utility function <code>cleanup_files</code> deletes the temporary folder for the specified <em>slurm_job</em>.</p> +</div> +<div id="single-function-evaluation" class="section level2"> +<h2>Single function evaluation</h2> +<p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> +<pre><code>## Submitted batch job 551568</code></pre> +<p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> +</div> +<div id="adding-auxiliary-data-and-functions" class="section level2"> +<h2>Adding auxiliary data and functions</h2> +<p>The function passed to <code>slurm_apply</code> can only receive atomic parameters stored within a data frame. Suppose we want instead to apply a function <code>func</code> to a list of complex R objects, <code>obj_list</code>. To use <code>slurm_apply</code> in this case, we can wrap <code>func</code> in an inline function that takes an integer parameter.</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(function(i) <span class="kw">func</span>(obj_list[[i]]), + <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), + <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), + <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> +<pre><code>## Submitted batch job 551570</code></pre> +<p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> +<p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> +</div> +<div id="configuring-slurm-options" class="section level2"> +<h2>Configuring SLURM options</h2> +<p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> +<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, + <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> +<pre><code>## Submitted batch job 551572</code></pre> +<p>sets the command line options <code>--time=1:00:00 --share</code>.</p> +</div> +<div id="generating-scripts-for-later-submission" class="section level2"> +<h2>Generating scripts for later submission</h2> +<p>When working from a R session without direct access to the cluster, you can set <code>submit = FALSE</code> within <code>slurm_apply</code>. The function will create the <em>_rslurm_[jobname]</em> folder and generate the scripts and .RData files, without submitting the job. You may then copy those files to the cluster and submit the job manually by calling <code>sbatch submit.sh</code> from the command line.</p> +</div> +<div id="how-it-works-advanced-customization" class="section level2"> +<h2>How it works / advanced customization</h2> +<p>As mentioned above, the <code>slurm_apply</code> function creates a job-specific folder. This folder contains the parameters as a <em>.RDS</em> file and (if applicable) the objects specified as <code>add_objects</code> saved together in a <em>.RData</em> file. The function also generates a R script (<code>slurm_run.R</code>) to be run on each cluster node, as well as a Bash script (<code>submit.sh</code>) to submit the job to SLURM.</p> +<p>More specifically, the Bash script creates a SLURM job array, with each cluster node receiving a different value of the <em>SLURM_ARRAY_TASK_ID</em> environment variable. This variable is read by <code>slurm_run.R</code>, which allows each instance of the script to operate on a different parameter subset and write its output to a different results file. The R script calls <code>parallel::mcMap</code> to parallelize calculations on each node.</p> +<p>Both <code>slurm_run.R</code> and <code>submit.sh</code> are generated from templates, using the <strong>whisker</strong> package; these templates can be found in the <code>rslurm/templates</code> subfolder in your R package library. There are two templates for each script, one for <code>slurm_apply</code> and the other (with the word <em>single</em> in its title) for <code>slurm_call</code>.</p> +<p>While you should avoid changing any existing lines in the template scripts, you may want to add <code>#SBATCH</code> lines to the <code>submit.sh</code> templates in order to permanently set certain SLURM command line options and thus customize the package to your particular cluster setup.</p> +</div> + + + +<!-- dynamically load mathjax for compatibility with self-contained --> +<script> + (function () { + var script = document.createElement("script"); + script.type = "text/javascript"; + script.src = "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; + document.getElementsByTagName("head")[0].appendChild(script); + })(); +</script> + +</body> +</html> diff --git a/release.R b/release.R index 0330527..2aca0d4 100644 --- a/release.R +++ b/release.R @@ -2,6 +2,7 @@ library(tools) library(rmarkdown) library(devtools) library(xml2) +library(R.rsp) # Document from R/*.R document() @@ -22,6 +23,16 @@ unlink('README.html') # Remove duplicate documentation for rslurm and rslurm-package from index and search system('sed -i "/alias{rslurm-package}/d" man/rslurm-package.Rd') +# Build vignettes +build_vignettes() +template_name <- 'index.html.rsp' +template <- file.path(system.file("doc/templates", package="R.rsp"), template_name) +file.copy(template, to='inst/doc') +setwd('inst/doc') +rfile(template_name) +unlink(template_name) +setwd('../../') + # Build -pkg <- build() +pkg <- build(path='~/tmp/') check_built(pkg) From a5fec36e9cd19554fcf677071b838c913b1e3a92 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Fri, 7 Apr 2017 22:23:44 -0400 Subject: [PATCH 23/25] include pre-build vignette index --- cran-comments.md | 4 ++-- inst/NEWS.Rd | 1 - inst/doc/index.html | 2 +- inst/doc/rslurm.html | 26 +++++++++++++------------- release.R | 5 +++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cran-comments.md b/cran-comments.md index 7db3606..d46c2b7 100755 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,6 +1,6 @@ -This is a minor update to the package to update the README. +This fixes the version number and provides a pre-built vignette, its source, and a vignette index. -The 'rslurm.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. To prevent build errors, 'BuildVignette: no' is in the description and vignettes are matched in .Buildignore. The source for the vignette is copied into 'inst/doc' for FOSS compliance. This gives the NOTE, but I see no way around it. +The 'rslurm.Rmd' vignette (formerly 'rslurm-vignette.Rmd') now only builds on a SLURM head node. The built vignette in inst/doc can be used as is. To prevent build errors, the vignette is matched in .Buildignore. The source for the vignette is copied into 'inst/doc' for FOSS compliance. This explains the NOTE. ## Tested on diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd index ac76066..dad563a 100644 --- a/inst/NEWS.Rd +++ b/inst/NEWS.Rd @@ -6,7 +6,6 @@ \section{Changes in version 0.3.3 from 2017-04-07}{ Minor update to repair README. \itemize{ - \item Modify DESCRIPTION with "BuildVignette: no". \item Create README from R/slurm.R. } } diff --git a/inst/doc/index.html b/inst/doc/index.html index 9afa307..83a51f8 100644 --- a/inst/doc/index.html +++ b/inst/doc/index.html @@ -23,7 +23,7 @@ <h2>User Guides and Package Vignettes</h2> <hr> <div align="center"><a href="../html/00Index.html">[Package Contents]</a></div> <br> -<div align="center"><small>This page was generated by the <a href="http://cran.r-project.org/web/packages/R.rsp/">R.rsp</a> package.</small></div> +<div align="center"><small>This page was generated by the <a href="http://cran.r-project.org/package=R.rsp">R.rsp</a> package.</small></div> </body> </html> diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html index ced4cf5..09d9702 100644 --- a/inst/doc/rslurm.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551564</code></pre> +<pre><code>## Submitted batch job 551630</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9999407 0.1001478 -## 2 2.0001493 0.2000370 -## 3 3.0005155 0.2999715</code></pre> +<pre><code>## s_mu s_sd +## 1 0.9999289 0.09992801 +## 2 1.9997553 0.19995928 +## 3 3.0000114 0.29994332</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] -## s_mu s_sd -## 0.9999407 0.1001478 +## s_mu s_sd +## 0.99992894 0.09992801 ## ## [[2]] -## s_mu s_sd -## 2.000149 0.200037 +## s_mu s_sd +## 1.9997553 0.1999593 ## ## [[3]] ## s_mu s_sd -## 3.0005155 0.2999715</code></pre> +## 3.0000114 0.2999433</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551568</code></pre> +<pre><code>## Submitted batch job 551634</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551570</code></pre> +<pre><code>## Submitted batch job 551636</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551572</code></pre> +<pre><code>## Submitted batch job 551638</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/release.R b/release.R index 2aca0d4..9db3f98 100644 --- a/release.R +++ b/release.R @@ -25,10 +25,11 @@ system('sed -i "/alias{rslurm-package}/d" man/rslurm-package.Rd') # Build vignettes build_vignettes() +setwd('inst/doc') template_name <- 'index.html.rsp' template <- file.path(system.file("doc/templates", package="R.rsp"), template_name) -file.copy(template, to='inst/doc') -setwd('inst/doc') +file.copy(template, to='.') +system(paste('sed -i "s|web/packages/R\\.rsp/|package=R\\.rsp|"', template_name)) rfile(template_name) unlink(template_name) setwd('../../') From 74f86a90331f807c75a2747304579715177798fe Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Sat, 8 Apr 2017 11:43:52 -0400 Subject: [PATCH 24/25] https for canonical url --- inst/doc/index.html | 2 +- inst/doc/rslurm.html | 24 ++++++++++++------------ release.R | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/inst/doc/index.html b/inst/doc/index.html index 83a51f8..8d9a671 100644 --- a/inst/doc/index.html +++ b/inst/doc/index.html @@ -23,7 +23,7 @@ <h2>User Guides and Package Vignettes</h2> <hr> <div align="center"><a href="../html/00Index.html">[Package Contents]</a></div> <br> -<div align="center"><small>This page was generated by the <a href="http://cran.r-project.org/package=R.rsp">R.rsp</a> package.</small></div> +<div align="center"><small>This page was generated by the <a href="https://cran.r-project.org/package=R.rsp">R.rsp</a> package.</small></div> </body> </html> diff --git a/inst/doc/rslurm.html b/inst/doc/rslurm.html index 09d9702..ea42450 100644 --- a/inst/doc/rslurm.html +++ b/inst/doc/rslurm.html @@ -102,7 +102,7 @@ <h2>Basic example</h2> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(rslurm) sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">jobname =</span> <span class="st">"test_job"</span>, <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551630</code></pre> +<pre><code>## Submitted batch job 551685</code></pre> <p>The output of <code>slurm_apply</code> is a <em>slurm_job</em> object that stores a few pieces of information (job name and number of nodes) needed to retrieve the job’s output.</p> <p>Assuming the function is run on a machine with access to the cluster, it also prints a message confirming the job has been submitted to SLURM.</p> <p>Particular clusters may require the specification of additional SLURM options, such as time and memory limits for the job. Also, when running R on a local machine without direct cluster access, you may want to generate scripts to be copied to the cluster and run at a later time. These topics are covered in additional sections below this basic example.</p> @@ -110,24 +110,24 @@ <h2>Basic example</h2> <p>Once the job completes, <code>get_slurm_out</code> reads and combines the output from all nodes.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"table"</span>) <span class="kw">head</span>(res, <span class="dv">3</span>)</code></pre></div> -<pre><code>## s_mu s_sd -## 1 0.9999289 0.09992801 -## 2 1.9997553 0.19995928 -## 3 3.0000114 0.29994332</code></pre> +<pre><code>## s_mu s_sd +## 1 0.9999848 0.1000190 +## 2 1.9997968 0.2000883 +## 3 2.9998983 0.2998295</code></pre> <p>When <code>outtype = "table"</code>, the outputs from each function evaluation are row-bound into a single data frame; this is an appropriate format when the function returns a simple vector. The default <code>outtype = "raw"</code> combines the outputs into a list and can thus handle arbitrarily complex return objects.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">res_raw <-<span class="st"> </span><span class="kw">get_slurm_out</span>(sjob, <span class="dt">outtype =</span> <span class="st">"raw"</span>, <span class="dt">wait =</span> <span class="ot">FALSE</span>) res_raw[<span class="dv">1</span>:<span class="dv">3</span>]</code></pre></div> <pre><code>## [[1]] -## s_mu s_sd -## 0.99992894 0.09992801 +## s_mu s_sd +## 0.9999848 0.1000190 ## ## [[2]] ## s_mu s_sd -## 1.9997553 0.1999593 +## 1.9997968 0.2000883 ## ## [[3]] ## s_mu s_sd -## 3.0000114 0.2999433</code></pre> +## 2.9998983 0.2998295</code></pre> <p>The files generated by <code>slurm_apply</code> are saved in a folder named <em>_rslurm_[jobname]</em> under the current working directory.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir</span>(<span class="st">"_rslurm_test_job"</span>)</code></pre></div> <pre><code>## [1] "params.RDS" "results_0.RDS" "results_1.RDS" "slurm_0.out" @@ -138,7 +138,7 @@ <h2>Basic example</h2> <h2>Single function evaluation</h2> <p>In addition to <code>slurm_apply</code>, rslurm also defines a <code>slurm_call</code> function, which sends a single function call to the cluster. It is analogous in syntax to the base R function <code>do.call</code>, accepting a function and a named list of parameters as arguments.</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_call</span>(test_func, <span class="kw">list</span>(<span class="dt">par_mu =</span> <span class="dv">5</span>, <span class="dt">par_sd =</span> <span class="dv">1</span>))</code></pre></div> -<pre><code>## Submitted batch job 551634</code></pre> +<pre><code>## Submitted batch job 551689</code></pre> <p>Because <code>slurm_call</code> involves a single process on a single node, it does not recognize the <code>nodes</code> and <code>cpus_per_node</code> arguments; otherwise, it accepts the same additional arguments (detailed in the sections below) as <code>slurm_apply</code>.</p> </div> <div id="adding-auxiliary-data-and-functions" class="section level2"> @@ -148,7 +148,7 @@ <h2>Adding auxiliary data and functions</h2> <span class="kw">data.frame</span>(<span class="dt">i =</span> <span class="kw">seq_along</span>(obj_list)), <span class="dt">add_objects =</span> <span class="kw">c</span>(<span class="st">"func"</span>, <span class="st">"obj_list"</span>), <span class="dt">nodes =</span> <span class="dv">2</span>, <span class="dt">cpus_per_node =</span> <span class="dv">2</span>)</code></pre></div> -<pre><code>## Submitted batch job 551636</code></pre> +<pre><code>## Submitted batch job 551691</code></pre> <p>The <code>add_objects</code> argument specifies the names of any R objects (besides the parameters data frame) that must be accessed by the function passed to <code>slurm_apply</code>. These objects are saved to a <code>.RData</code> file that is loaded on each cluster node prior to evaluating the function in parallel.</p> <p>By default, all R packages attached to the current R session will also be attached (with <code>library</code>) on each cluster node, though this can be modified with the optional <code>pkgs</code> argument.</p> </div> @@ -157,7 +157,7 @@ <h2>Configuring SLURM options</h2> <p>The <code>slurm_options</code> argument allows you to set any of the command line options (<a href="http://slurm.schedmd.com/sbatch.html">view list</a>) recognized by the SLURM <code>sbatch</code> command. It should be formatted as a named list, using the long names of each option (e.g. “time” rather than “t”). Flags, i.e. command line options that are toggled rather than set to a particular value, should be set to <code>TRUE</code> in <code>slurm_options</code>. For example, the following code:</p> <div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">sjob <-<span class="st"> </span><span class="kw">slurm_apply</span>(test_func, pars, <span class="dt">slurm_options =</span> <span class="kw">list</span>(<span class="dt">time =</span> <span class="st">"1:00:00"</span>, <span class="dt">share =</span> <span class="ot">TRUE</span>))</code></pre></div> -<pre><code>## Submitted batch job 551638</code></pre> +<pre><code>## Submitted batch job 551693</code></pre> <p>sets the command line options <code>--time=1:00:00 --share</code>.</p> </div> <div id="generating-scripts-for-later-submission" class="section level2"> diff --git a/release.R b/release.R index 9db3f98..d7ebd83 100644 --- a/release.R +++ b/release.R @@ -29,7 +29,7 @@ setwd('inst/doc') template_name <- 'index.html.rsp' template <- file.path(system.file("doc/templates", package="R.rsp"), template_name) file.copy(template, to='.') -system(paste('sed -i "s|web/packages/R\\.rsp/|package=R\\.rsp|"', template_name)) +system(paste('sed -i "s|http://\\([^/]*\\)/web/packages/R\\.rsp/|https://\\1/package=R\\.rsp|"', template_name)) rfile(template_name) unlink(template_name) setwd('../../') From ac1a3a1d4408f49ffa4e771db6314644e86caff1 Mon Sep 17 00:00:00 2001 From: Ian Carroll <icarroll@sesync.org> Date: Sat, 8 Apr 2017 17:31:53 -0400 Subject: [PATCH 25/25] restore rslurm.Rmd source to vignettes in package --- .Rbuildignore | 1 - .travis.yml | 1 + inst/doc/index.html | 29 ----------------------------- release.R | 9 --------- 4 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 inst/doc/index.html diff --git a/.Rbuildignore b/.Rbuildignore index 0362ee0..c80b62d 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,4 +3,3 @@ ^\.travis\.yml$ ^cran-comments\.md ^release\.R -^vignettes/rslurm\.Rmd diff --git a/.travis.yml b/.travis.yml index 9699bf9..fc438d5 100755 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: r warnings_are_errors: true sudo: required r_build_args: "--no-build-vignettes" +r_check_args: "--no-vignettes --as-cran" env: global: diff --git a/inst/doc/index.html b/inst/doc/index.html deleted file mode 100644 index 8d9a671..0000000 --- a/inst/doc/index.html +++ /dev/null @@ -1,29 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> -<html> -<head> - <title>rslurm User Guides</title> - <link rel="stylesheet" type="text/css" href="../../R.css"> -</head> -<body> - -<h1>Submit R Calculations to a 'SLURM' Cluster <img class="toplogo" src="../../../doc/html/logo.jpg" alt="[R logo]" width="101" height="77"></h1> -<hr> - -<h2>User Guides and Package Vignettes</h2> -<ul> - <li> - <strong>Parallelize R code on a SLURM cluster</strong> - [<a href="./rslurm.html">HTML</a>] - [<a href="./rslurm.Rmd">source</a>] - [<a href="./rslurm.R">R code</a>] - </li> -</ul> -<br> - -<hr> -<div align="center"><a href="../html/00Index.html">[Package Contents]</a></div> -<br> -<div align="center"><small>This page was generated by the <a href="https://cran.r-project.org/package=R.rsp">R.rsp</a> package.</small></div> - -</body> -</html> diff --git a/release.R b/release.R index d7ebd83..d85cf96 100644 --- a/release.R +++ b/release.R @@ -2,7 +2,6 @@ library(tools) library(rmarkdown) library(devtools) library(xml2) -library(R.rsp) # Document from R/*.R document() @@ -25,14 +24,6 @@ system('sed -i "/alias{rslurm-package}/d" man/rslurm-package.Rd') # Build vignettes build_vignettes() -setwd('inst/doc') -template_name <- 'index.html.rsp' -template <- file.path(system.file("doc/templates", package="R.rsp"), template_name) -file.copy(template, to='.') -system(paste('sed -i "s|http://\\([^/]*\\)/web/packages/R\\.rsp/|https://\\1/package=R\\.rsp|"', template_name)) -rfile(template_name) -unlink(template_name) -setwd('../../') # Build pkg <- build(path='~/tmp/')