From 8a2710947dfebe69b47e2ea4a184d9b261ed09d6 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Thu, 5 Sep 2024 16:54:36 +0000 Subject: [PATCH 01/38] updated gh actions yaml to download suggested packages too. --- .github/workflows/RunChecks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index e10e006..aa7d39d 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -33,6 +33,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.config.r }} + suggested-packages: true use-public-rspm: true - name: Download mocked binaries From 753c97341d5737e4b10e2d940fe8f60dc0fa2e09 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:21:27 +0000 Subject: [PATCH 02/38] trying to get bbr installed for vignettes needs for gh actions. --- .github/workflows/RunChecks.yaml | 3 ++- vignettes/slack-alerts.Rmd | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index aa7d39d..614d33c 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -33,7 +33,6 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.config.r }} - suggested-packages: true use-public-rspm: true - name: Download mocked binaries @@ -67,6 +66,8 @@ jobs: with: extra-packages: any::rcmdcheck needs: check + - name: Install bbr from GitHub + run: Rscript -e 'remotes::install_github("metrumresearchgroup/bbr", ref = "main")' - uses: r-lib/actions/check-r-package@v2 with: diff --git a/vignettes/slack-alerts.Rmd b/vignettes/slack-alerts.Rmd index 88f0a53..15731f8 100644 --- a/vignettes/slack-alerts.Rmd +++ b/vignettes/slack-alerts.Rmd @@ -62,7 +62,7 @@ Sys.which("slack_notifier") slack_notifier requires an additional `slack_notifier/config.yaml` file that contains the slack bot OAuth token which is found from -[[[https://api.slack.com/apps/\\\\](https://api.slack.com/apps/\\){.uri}]([https://api.slack.com/apps/\\](https://api.slack.com/apps/\){.uri}){.uri}\ /oauth?]. ``` slack_notifier/config.yaml From 8ad9c09e19722982070349ec0bdd2082e8bec4a2 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:24:16 +0000 Subject: [PATCH 03/38] test for workflows --- .github/workflows/RunChecks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 614d33c..a356241 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -2,9 +2,9 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, workflow_test] pull_request: - branches: [main, master] + branches: [main, master, workflow_test] name: R-CMD-check From b7eebc1d2ea9261134eb1260f883250084a5c17e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:26:37 +0000 Subject: [PATCH 04/38] installing bbr before setup-r-dependencies --- .github/workflows/RunChecks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index a356241..ae46ea4 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -61,13 +61,13 @@ jobs: rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install bbr from GitHub + run: Rscript -e 'remotes::install_github("metrumresearchgroup/bbr", ref = "main")' - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::rcmdcheck needs: check - - name: Install bbr from GitHub - run: Rscript -e 'remotes::install_github("metrumresearchgroup/bbr", ref = "main")' - uses: r-lib/actions/check-r-package@v2 with: From ddf1e944d642be5263a7af5ac8b8f3edfa0133df Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:29:37 +0000 Subject: [PATCH 05/38] added remotes --- .github/workflows/RunChecks.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index ae46ea4..b2dfa93 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -61,6 +61,9 @@ jobs: rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install remotes package + run: Rscript -e 'install.packages("remotes")' + - name: Install bbr from GitHub run: Rscript -e 'remotes::install_github("metrumresearchgroup/bbr", ref = "main")' From d907146650f39483753dcb6164ea4c3bf1810ef0 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:38:48 +0000 Subject: [PATCH 06/38] more bbr dependencies --- .github/workflows/RunChecks.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index b2dfa93..230e511 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -61,11 +61,13 @@ jobs: rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install remotes package - run: Rscript -e 'install.packages("remotes")' - name: Install bbr from GitHub - run: Rscript -e 'remotes::install_github("metrumresearchgroup/bbr", ref = "main")' + run: | + Rscript -e 'install.packages("remotes")' + Rscript -e 'remotes::install_github("metrumresearchgroup/mrgmisc")' + Rscript -e 'remotes::install_github("metrumresearchgroup/nmrec")' + Rscript -e 'remotes::install_github("metrumresearchgroup/bbr")' - uses: r-lib/actions/setup-r-dependencies@v2 with: From 78149c753a4b46fec1b1768d68fec783ca588fde Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:48:26 +0000 Subject: [PATCH 07/38] trying to add bbr to extra-packages --- .github/workflows/RunChecks.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 230e511..3f5e831 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -71,7 +71,9 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: - extra-packages: any::rcmdcheck + extra-packages: | + metrumresearchgorup/bbr + any::rcmdcheck needs: check - uses: r-lib/actions/check-r-package@v2 From 05a0cc12444ff000553644dd32bf8544aac25d29 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:52:36 +0000 Subject: [PATCH 08/38] fixed spelling --- .github/workflows/RunChecks.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 3f5e831..4824c82 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -62,17 +62,10 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install bbr from GitHub - run: | - Rscript -e 'install.packages("remotes")' - Rscript -e 'remotes::install_github("metrumresearchgroup/mrgmisc")' - Rscript -e 'remotes::install_github("metrumresearchgroup/nmrec")' - Rscript -e 'remotes::install_github("metrumresearchgroup/bbr")' - - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: | - metrumresearchgorup/bbr + metrumresearchgroup/bbr any::rcmdcheck needs: check From d71ed222484b66d9190d7be25e8ea22843494834 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 18:55:01 +0000 Subject: [PATCH 09/38] added bbr dependencies --- .github/workflows/RunChecks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 4824c82..befc751 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -65,6 +65,8 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: | + metrumresearchgroup/mrgmisc + metrumresearchgroup/nmrec metrumresearchgroup/bbr any::rcmdcheck needs: check From 930a989080bdf289ba4fac108a6607b36033beb4 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:09:18 +0000 Subject: [PATCH 10/38] added error handling around processx::run("squeue"). Will throw warning and likely errors because option never gets set. --- R/zzz.R | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/R/zzz.R b/R/zzz.R index fc8db21..aaafa6b 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -4,14 +4,23 @@ } .onLoad <- function(libname, pkgname) { - processx_output <- processx::run("squeue", args = "--version") - if (processx_output$status != 0) { - warning("squeue not installed correctly") - } else { - # stdout format is "slurm major.minor.patch/n" - version <- strsplit(processx_output$stdout," ")[[1]][[2]] %>% - trimws() %>% - package_version() - options("squeue.version" = version) + result <- tryCatch( + {processx::run("squeue", args = "--version")}, + error = function(e) { + message("Warning: 'squeue' command not found or failed to run.") + return(NULL) + } + ) + + if (!is.null(result)) { + if (result$status != 0) { + warning("squeue not installed correctly") + } else { + # stdout format is "slurm major.minor.patch/n" + version <- strsplit(result$stdout," ")[[1]][[2]] %>% + trimws() %>% + package_version() + options("squeue.version" = version) + } } } From f32351f9edbb9cd58c4001a820fc944e5fef7ab3 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:21:36 +0000 Subject: [PATCH 11/38] updated error handling messages. set vignettes to fales. --- .github/workflows/RunChecks.yaml | 3 ++- R/zzz.R | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index befc751..deb9774 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -65,7 +65,7 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: | - metrumresearchgroup/mrgmisc + metrumresearchgroup/mrgmisec metrumresearchgroup/nmrec metrumresearchgroup/bbr any::rcmdcheck @@ -74,3 +74,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true + vignettes: 'FALSE' diff --git a/R/zzz.R b/R/zzz.R index aaafa6b..4cb2d32 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -7,7 +7,7 @@ result <- tryCatch( {processx::run("squeue", args = "--version")}, error = function(e) { - message("Warning: 'squeue' command not found or failed to run.") + warning("'squeue' command not found or failed to run.") return(NULL) } ) From dc8665fc125d946365f524248d1036f333c0d7ea Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:24:28 +0000 Subject: [PATCH 12/38] typo fix --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index deb9774..9c73307 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -65,7 +65,7 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: | - metrumresearchgroup/mrgmisec + metrumresearchgroup/mrgmisc metrumresearchgroup/nmrec metrumresearchgroup/bbr any::rcmdcheck From b75aede95b5ab42283f7c3fabf802e058b1fff90 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:33:54 +0000 Subject: [PATCH 13/38] stopping vignettes --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 9c73307..b7ddccb 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -74,4 +74,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - vignettes: 'FALSE' + args: '"FALSE"--no-build-vignettes' From b6e8bc726c7ba435fb1f546e649b9840d2f018c5 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:38:57 +0000 Subject: [PATCH 14/38] another typo --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index b7ddccb..6818a20 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -74,4 +74,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - args: '"FALSE"--no-build-vignettes' + args: '"--no-build-vignettes"' From b4b6588c4d0789aa69409ed89164f3d561c8bd2e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 19:45:21 +0000 Subject: [PATCH 15/38] different args --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 6818a20..9735294 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -74,4 +74,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - args: '"--no-build-vignettes"' + build_args: '"--no-build-vignettes"' From dff4ee2bc55ea87b37b960139981c309cdf23be2 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 20:05:21 +0000 Subject: [PATCH 16/38] prebuilding vignettes to prevent gh actions from rebuilding them. --- .gitignore | 1 - inst/doc/Running-nonmem.R | 81 +++++ inst/doc/Running-nonmem.Rmd | 269 +++++++++++++++ inst/doc/Running-nonmem.html | 638 +++++++++++++++++++++++++++++++++++ inst/doc/custom-alerts.R | 97 ++++++ inst/doc/custom-alerts.Rmd | 267 +++++++++++++++ inst/doc/custom-alerts.html | 590 ++++++++++++++++++++++++++++++++ inst/doc/slack-alerts.R | 78 +++++ inst/doc/slack-alerts.Rmd | 145 ++++++++ inst/doc/slack-alerts.html | 461 +++++++++++++++++++++++++ 10 files changed, 2626 insertions(+), 1 deletion(-) create mode 100644 inst/doc/Running-nonmem.R create mode 100644 inst/doc/Running-nonmem.Rmd create mode 100644 inst/doc/Running-nonmem.html create mode 100644 inst/doc/custom-alerts.R create mode 100644 inst/doc/custom-alerts.Rmd create mode 100644 inst/doc/custom-alerts.html create mode 100644 inst/doc/slack-alerts.R create mode 100644 inst/doc/slack-alerts.Rmd create mode 100644 inst/doc/slack-alerts.html diff --git a/.gitignore b/.gitignore index 3c3a34f..7e6348b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .Rproj.user docs -inst/doc /doc/ /Meta/ diff --git a/inst/doc/Running-nonmem.R b/inst/doc/Running-nonmem.R new file mode 100644 index 0000000..0f22cd3 --- /dev/null +++ b/inst/doc/Running-nonmem.R @@ -0,0 +1,81 @@ +## ----include = FALSE---------------------------------------------------------- +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + +## ----include = FALSE---------------------------------------------------------- +knitr::opts_chunk$set( + collapse = TRUE, + comment = "" +) + +## ----setup-------------------------------------------------------------------- +library(slurmtools) + +## ----------------------------------------------------------------------------- +Sys.which("bbi") + +## ----------------------------------------------------------------------------- +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") + +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml")) + +## ----------------------------------------------------------------------------- +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} + +## ----------------------------------------------------------------------------- +submission <- slurmtools::submit_nonmem_model( + mod, + slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"), +) + +submission + +## ----------------------------------------------------------------------------- +slurmtools::get_slurm_jobs(user = 'matthews') + +## ----------------------------------------------------------------------------- +submission_ntfy <- slurmtools::submit_nonmem_model( + mod, + slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"), + overwrite = TRUE, + slurm_template_opts = list( + ntfy = "ntfy_demo") +) + +submission_ntfy + +## ----include = FALSE---------------------------------------------------------- +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + diff --git a/inst/doc/Running-nonmem.Rmd b/inst/doc/Running-nonmem.Rmd new file mode 100644 index 0000000..ce7a020 --- /dev/null +++ b/inst/doc/Running-nonmem.Rmd @@ -0,0 +1,269 @@ +--- +title: "Running Nonmem with slurmtools" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Running Nonmem with slurmtools} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "" +) +``` + +# Slurmtools for submitting NONMEM runs + +`slurmtools` is an R package for interacting with slurm (fka **S**imple **L**inux **U**tility for **R**esource **M**anagement) and submitting NONMEM jobs. You can submit a NONMEM job with `submit_nonmem_model`, you can view current jobs with `get_slurm_jobs`, and you can see the available partitions with `get_slurm_partitions`. + +## Installing `slurmtools` + +To install `slurmtools` use the following commands: + +``` r +options(repos = c( + "slurmtools" = "https://a2-ai.github.io/gh-pkg-mirror/slurmtools", + getOption("repos"))) +install.packages("slurmtools") +``` + +```{r setup} +library(slurmtools) +``` + +We are given a message when loading slurmtools that some options are not set and that default job submission will not work without them. These options are used for default arguments in the `submit_nonmem_model` function. Running `?submit_nonmem_model` we can see the documentation + +![Help view for `submit_nonmem_model` function](data/images/submit_nonmem_model_help.png) + +This function uses the inputs to populate a template Bash shell script that submits the NONMEM job to slurm. A default template file is supplied with the Project Starter and it can be modified to do additional tasks as long as they are possible within Bash. + +By default these values are provided to the slurm template file: + +``` r +default_template_list = list( + partition = partition, + parallel = parallel, + ncpu = ncpu, + job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)), + project_path = project_path, + project_name = project_name, + bbi_exe_path = Sys.which("bbi"), + bbi_config_path = bbi_config_path, + model_path = .mod$absolute_model_path, + config_toml_path = config_toml_path, + nmm_exe_path = Sys.which("nmm") +) +``` + +- `partition` is an argument to `submit_nonmem_model` + +- `parallel` is `TRUE` if `ncpu > 1`, else `FALSE` + +- `ncpu` is an argument to `submit_nonmem_model` + +- `job_name` is created from the `.mod` argument supplied to `submit_nonmem_model` + +- `bbi_exe_path` is determined via \`Sys.which("bbi") + +- `bbi_config_path` is determined via getOption("slurmtools.bbi_config_path") + +- `model_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` + +- `config_toml_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` and is requried to use `nmm` (NONMEM monitor) + +- `nmm_exe_path` is determined via `Sys.which("nmm")` + +If you need to feed more arguments to the template you simply supply them in the `slurm_template_opts` argument as a list. More on that later. + +## Submitting a NONMEM job with `bbi` + +To submit a NONMEM job, we need to supply either the path to a mod file or create a model object from `bbr`, and supply a `slurm-template.tmpl` file. To use `bbi` we also need a `bbi.yaml` file, which I've also supplied in `/model/nonmem/bbi.yaml` (and is also supplied with the R project starter). + +Here is an example of a template file that will call `bbi`: + +``` slurm-job-bbi.tmpl +#!/bin/bash +#SBATCH --job-name="{{job_name}}" +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task={{ncpu}} +#SBATCH --partition={{partition}} +#SBATCH --account={{project_name}} + +# submit_nonmem_model uses the whisker package to populate template files +# https://github.com/edwindj/whisker + +{{#parallel}} +{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}} +{{/parallel}} + + +{{^parallel}} +{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}} +{{/parallel}} +``` + +This file will call `bbi` to run our supplied model (`{{model_path}}.mod`) if `ncpu > 1` then parallel will be true and the code between `{{#parallel}}` and `{{/parallel}}` will be populated. if `ncpu = 1` then parallel will be false and the code between `{{^parallel}}` and `{{/parallel}}` will be populated. By default, `submit_nonmem_model` will inject `Sys.which("bbi")` into the template, so if `bbi` is not on your path we'll have to supply the `bbi_exe_path` for it to start the NONMEM run. + +```{r} +Sys.which("bbi") +``` + +We will use a few different template files with different functionality so we'll inject those template file paths to `submit_nonmem_model`. However, we'll use the `submission-log` directory for the output, so we'll set that option as well as `bbi_config_path` so `submit_nonmem_model` defaults can be used. The slurm template files are saved in `~/model/nonmem/` Additionally, there is a simple NONMEM control stream in `1001.mod` in the same directory that we can use for testing. + +```{r} +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") + +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml")) +``` + +To create the `bbr` model object, we need to have both `1001.mod` and `1001.yaml` which contains metadata about the model in the supplied directory (`./model/nonmem/`). We'll check for mod_number.yaml and if it exists, read in the model otherwise create it and then read it. + +```{r} +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} +``` + +We can now submit the job and point to the template file in `model/nonmem/slurm-job-bbi.tmpl`. + +```{r} +submission <- slurmtools::submit_nonmem_model( + mod, + slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"), +) + +submission +``` + +We see a `status` with an exit code of 0 suggesting a successful command, and the `stdout` gives us the batch job number. We can use `slurmtools::get_slurm_jobs()` to monitor the status of the job. Here, we can supply the user = "matthews" argument to filter to just the jobs I've submitted. + +```{r} +slurmtools::get_slurm_jobs(user = 'matthews') +``` + +If we look in the `slurmtools.submisstion_root` directory we can see the shell script that was generated with `submit_nonmem_model`. Here is the whisker replaced call to bbi: + +``` 1001.sh +/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml +``` + +## Extending templates + +Because the templates create a bash shell script there is an almost infinite number of things we can do with our template. **Anything you can do in bash you can do by appropriately updating the template file and injecting the needed information!** + +Let's add a notification feature that will send a notification when the job has started and finished. We can use [ntfy.sh](ntfy.sh) and add the necessary info to our template to achieve this. + +Here is a modified template file that adds a `JOBID=$SLURM_JOBID` and some ntfy calls. To get a notification we can supply `submit_nonmem_model` with `ntfy` variable to send notifications. I'll use `ntfy = ntfy_demo` for this. + +``` slurm-job-bbi-ntfy.tmpl +#!/bin/bash +#SBATCH --job-name="{{job_name}}" +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task={{ncpu}} +#SBATCH --partition={{partition}} +#SBATCH --account={{project_name}} + +JOBID=$SLURM_JOBID + +# submit_nonmem_model uses the whisker package to populate template files +# https://github.com/edwindj/whisker + +{{#ntfy}} +curl -d "Starting model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}} +{{/ntfy}} + +{{#parallel}} +{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}} +{{/parallel}} + +{{^parallel}} +{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}} +{{/parallel}} + +{{#ntfy}} +curl -d "Finished model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}} +{{/ntfy}} +``` + +Since we've already run this model we will provide the `overwrite = TRUE` argument to force a new nonmem run. + +```{r} +submission_ntfy <- slurmtools::submit_nonmem_model( + mod, + slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"), + overwrite = TRUE, + slurm_template_opts = list( + ntfy = "ntfy_demo") +) + +submission_ntfy +``` + +We again get a 0 exit code status and now instead of using `slurmtools::get_slurm_jobs()` to monitor the job, we can rely on the new notifications we just set up. ![NONMEM job starting ntfy alert](data/images/ntfy_starting.png) + +and when the run finished we get another notification: ![NONMEM Job finished ntfy alert](data/images/ntfy_finished.png) + +Note that the run number will match the run specified in `submission$stdout`. We can see the new shell script this updated template file generated + +``` 1001.sh +#!/bin/bash +#SBATCH --job-name="1001-nonmem-run" +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --partition=cpu2mem4gb +#SBATCH --account=slurmtools + +JOBID=$SLURM_JOBID + +curl -d "Starting model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo + +/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml + +curl -d "Finished model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo +``` + +To reiterate, this template file is run as a bash shell script so anything you can do in bash you can put into the template and pass the needed arguments and customize the behavior to your liking. + +```{r, include = FALSE} +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` diff --git a/inst/doc/Running-nonmem.html b/inst/doc/Running-nonmem.html new file mode 100644 index 0000000..94827a2 --- /dev/null +++ b/inst/doc/Running-nonmem.html @@ -0,0 +1,638 @@ + + + + + + + + + + + + + + +Running Nonmem with slurmtools + + + + + + + + + + + + + + + + + + + + + + + + + + +

Running Nonmem with slurmtools

+ + + +
+

Slurmtools for submitting NONMEM runs

+

slurmtools is an R package for interacting with slurm +(fka Simple Linux +Utility for Resource +Management) and submitting NONMEM jobs. You can submit +a NONMEM job with submit_nonmem_model, you can view current +jobs with get_slurm_jobs, and you can see the available +partitions with get_slurm_partitions.

+
+

Installing slurmtools

+

To install slurmtools use the following commands:

+
options(repos = c(
+  "slurmtools" = "https://a2-ai.github.io/gh-pkg-mirror/slurmtools", 
+  getOption("repos")))
+install.packages("slurmtools")
+
library(slurmtools)
+

We are given a message when loading slurmtools that some options are +not set and that default job submission will not work without them. +These options are used for default arguments in the +submit_nonmem_model function. Running +?submit_nonmem_model we can see the documentation

+
+Help view for submit_nonmem_model function +
Help view for submit_nonmem_model +function
+
+

This function uses the inputs to populate a template Bash shell +script that submits the NONMEM job to slurm. A default template file is +supplied with the Project Starter and it can be modified to do +additional tasks as long as they are possible within Bash.

+

By default these values are provided to the slurm template file:

+
default_template_list = list(
+  partition = partition,
+  parallel = parallel,
+  ncpu = ncpu,
+  job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)),
+  project_path = project_path,
+  project_name = project_name,
+  bbi_exe_path = Sys.which("bbi"),
+  bbi_config_path = bbi_config_path,
+  model_path = .mod$absolute_model_path,
+  config_toml_path = config_toml_path,
+  nmm_exe_path = Sys.which("nmm")
+)
+
    +

  • partition is an argument to +submit_nonmem_model

    • +

  • parallel is TRUE if +ncpu > 1, else FALSE

    • +

  • ncpu is an argument to +submit_nonmem_model

    • +

  • job_name is created from the .mod +argument supplied to submit_nonmem_model

    • +

  • bbi_exe_path is determined via +`Sys.which(“bbi”)

    • +

  • bbi_config_path is determined via +getOption(“slurmtools.bbi_config_path”)

    • +

  • model_path is determined from the .mod +argument supplied to submit_nonmem_model

    • +

  • config_toml_path is determined from the +.mod argument supplied to submit_nonmem_model +and is requried to use nmm (NONMEM monitor)

    • +

  • nmm_exe_path is determined via +Sys.which("nmm")

    • +
+

If you need to feed more arguments to the template you simply supply +them in the slurm_template_opts argument as a list. More on +that later.

+
+
+

Submitting a NONMEM job with bbi

+

To submit a NONMEM job, we need to supply either the path to a mod +file or create a model object from bbr, and supply a +slurm-template.tmpl file. To use bbi we also +need a bbi.yaml file, which I’ve also supplied in +/model/nonmem/bbi.yaml (and is also supplied with the R +project starter).

+

Here is an example of a template file that will call +bbi:

+
#!/bin/bash
+#SBATCH --job-name="{{job_name}}"
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task={{ncpu}}
+#SBATCH --partition={{partition}}
+#SBATCH --account={{project_name}}
+
+# submit_nonmem_model uses the whisker package to populate template files
+# https://github.com/edwindj/whisker
+
+{{#parallel}}
+{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}}
+{{/parallel}}
+
+
+{{^parallel}}
+{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}}
+{{/parallel}}
+

This file will call bbi to run our supplied model +({{model_path}}.mod) if ncpu > 1 then +parallel will be true and the code between {{#parallel}} +and {{/parallel}} will be populated. if +ncpu = 1 then parallel will be false and the code between +{{^parallel}} and {{/parallel}} will be +populated. By default, submit_nonmem_model will inject +Sys.which("bbi") into the template, so if bbi +is not on your path we’ll have to supply the bbi_exe_path +for it to start the NONMEM run.

+
Sys.which("bbi")
+                 bbi 
+"/usr/local/bin/bbi"
+

We will use a few different template files with different +functionality so we’ll inject those template file paths to +submit_nonmem_model. However, we’ll use the +submission-log directory for the output, so we’ll set that +option as well as bbi_config_path so +submit_nonmem_model defaults can be used. The slurm +template files are saved in ~/model/nonmem/ Additionally, +there is a simple NONMEM control stream in 1001.mod in the +same directory that we can use for testing.

+
library(bbr)
+library(here)
+
+nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
+
+options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
+options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml"))
+

To create the bbr model object, we need to have both +1001.mod and 1001.yaml which contains metadata +about the model in the supplied directory +(./model/nonmem/). We’ll check for mod_number.yaml and if +it exists, read in the model otherwise create it and then read it.

+
mod_number <- "1001"
+
+if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
+  mod <- bbr::read_model(file.path(nonmem, mod_number))
+} else {
+  mod <- bbr::new_model(file.path(nonmem, mod_number))
+}
+

We can now submit the job and point to the template file in +model/nonmem/slurm-job-bbi.tmpl.

+
submission <- slurmtools::submit_nonmem_model(
+  mod,
+  slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"),
+)
+
+submission
+$status
+[1] 0
+
+$stdout
+[1] "Submitted batch job 878\n"
+
+$stderr
+[1] ""
+
+$timeout
+[1] FALSE
+

We see a status with an exit code of 0 suggesting a +successful command, and the stdout gives us the batch job +number. We can use slurmtools::get_slurm_jobs() to monitor +the status of the job. Here, we can supply the user = “matthews” +argument to filter to just the jobs I’ve submitted.

+
slurmtools::get_slurm_jobs(user = 'matthews')
+# A tibble: 11 × 10
+   job_id job_state  cpus partition  standard_input standard_output             
+    <int> <chr>     <int> <chr>      <chr>          <chr>                       
+ 1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+ 2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+ 3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+ 4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+ 5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+ 6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+ 7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+ 8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+ 9    876 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+10    877 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+11    878 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+# ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
+#   current_working_directory <chr>
+

If we look in the slurmtools.submisstion_root directory +we can see the shell script that was generated with +submit_nonmem_model. Here is the whisker replaced call to +bbi:

+
/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml
+
+
+

Extending templates

+

Because the templates create a bash shell script there is an almost +infinite number of things we can do with our template. Anything +you can do in bash you can do by appropriately updating the template +file and injecting the needed information!

+

Let’s add a notification feature that will send a notification when +the job has started and finished. We can use ntfy.sh and add the necessary info to our template to +achieve this.

+

Here is a modified template file that adds a +JOBID=$SLURM_JOBID and some ntfy calls. To get a +notification we can supply submit_nonmem_model with +ntfy variable to send notifications. I’ll use +ntfy = ntfy_demo for this.

+
#!/bin/bash
+#SBATCH --job-name="{{job_name}}"
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task={{ncpu}}
+#SBATCH --partition={{partition}}
+#SBATCH --account={{project_name}}
+
+JOBID=$SLURM_JOBID
+
+# submit_nonmem_model uses the whisker package to populate template files
+# https://github.com/edwindj/whisker
+
+{{#ntfy}}
+curl -d "Starting model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}}
+{{/ntfy}}
+
+{{#parallel}}
+{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}}
+{{/parallel}}
+
+{{^parallel}}
+{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}}
+{{/parallel}}
+
+{{#ntfy}}
+curl -d "Finished model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}}
+{{/ntfy}}
+

Since we’ve already run this model we will provide the +overwrite = TRUE argument to force a new nonmem run.

+
submission_ntfy <- slurmtools::submit_nonmem_model(
+  mod, 
+  slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"),
+  overwrite = TRUE,
+  slurm_template_opts = list(
+    ntfy = "ntfy_demo")
+)
+
+submission_ntfy
+$status
+[1] 0
+
+$stdout
+[1] "Submitted batch job 879\n"
+
+$stderr
+[1] ""
+
+$timeout
+[1] FALSE
+

We again get a 0 exit code status and now instead of using +slurmtools::get_slurm_jobs() to monitor the job, we can +rely on the new notifications we just set up. NONMEM job starting ntfy alert

+

and when the run finished we get another notification: NONMEM Job finished ntfy alert

+

Note that the run number will match the run specified in +submission$stdout. We can see the new shell script this +updated template file generated

+
#!/bin/bash
+#SBATCH --job-name="1001-nonmem-run"
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --partition=cpu2mem4gb
+#SBATCH --account=slurmtools
+
+JOBID=$SLURM_JOBID
+
+curl -d "Starting model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo
+
+/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml
+
+curl -d "Finished model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo
+

To reiterate, this template file is run as a bash shell script so +anything you can do in bash you can put into the template and pass the +needed arguments and customize the behavior to your liking.

+
+
+ + + + + + + + + + + diff --git a/inst/doc/custom-alerts.R b/inst/doc/custom-alerts.R new file mode 100644 index 0000000..100bd0e --- /dev/null +++ b/inst/doc/custom-alerts.R @@ -0,0 +1,97 @@ +## ----include = FALSE---------------------------------------------------------- +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + +## ----include = FALSE---------------------------------------------------------- +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) + +## ----setup-------------------------------------------------------------------- +library(slurmtools) +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) + +## ----------------------------------------------------------------------------- +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} + +## ----------------------------------------------------------------------------- +slurmtools::generate_nmm_config(mod) + +## ----------------------------------------------------------------------------- +slurmtools::generate_nmm_config( + mod, + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") + +## ----------------------------------------------------------------------------- +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm")) +) + +submission_nmm + +## ----------------------------------------------------------------------------- +slurmtools::get_slurm_jobs() + +## ----------------------------------------------------------------------------- +slurmtools::generate_nmm_config( + mod, + alert = "Ntfy", + topic = "NONMEMmonitor", + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") + +## ----------------------------------------------------------------------------- +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm")) +) + +submission_nmm + +## ----------------------------------------------------------------------------- +slurmtools::get_slurm_jobs(user = "matthews") + +## ----include = FALSE---------------------------------------------------------- +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + diff --git a/inst/doc/custom-alerts.Rmd b/inst/doc/custom-alerts.Rmd new file mode 100644 index 0000000..37d6dfe --- /dev/null +++ b/inst/doc/custom-alerts.Rmd @@ -0,0 +1,267 @@ +--- +title: "custom-alerts" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{custom-alerts} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + markdown: + wrap: 72 +--- + +```{r, include = FALSE} +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(slurmtools) +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +``` + +## Submitting a NONMEM job with nmm + +Instead of using bbi we can use `nmm` ([NONMEM +Monitor](https://github.com/A2-ai/nonmem-monitor)) which currently has +some additional functionality of sending notifications about zero +gradients, missing -1E9 lines in ext file, and some very basic control +stream errors. Currently, only slack or [ntfy.sh](ntfy.sh) is supported +for receiving notifications. To use `nmm` you can install the latest +release from the github repository linked above. + +We can update the template file accordingly: + +``` slurm-job-nmm.tmpl +#!/bin/bash +#SBATCH --job-name="{{job_name}}" +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task={{ncpu}} +#SBATCH --partition={{partition}} + +{{nmm_exe_path}} -c {{config_toml_path}} run +``` + +default, `submit_nonmem_model` will provide `nmm_exe_path` and +`config_toml_path` to the template. Just like with `bbi_exe_path`, +`nmm_exe_path` is determined with `Sys.which("nmm")` which may or may +not give you the path to the nmm binary if it is on your path or not. We +can inject the `nmm_exe_path` like we did with `bbi_exe_path` and assume +it's not on our path. + +The `config.toml` file controls what `nmm` will monitor and where to +look for files and how to alert you. We'll use `generate_nmm_config()` +to create this file. First we can look at the documentation to see what +type of information we should pass to this function.![documentation for +generate_nmm_config()](data/images/generate_nmm_config_help.png) + +```{r} +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} +``` + +```{r} +slurmtools::generate_nmm_config(mod) +``` + +This generates the following toml file. Notice that alert is set to +'None', and both email and topic are empty. Since we're in vignettes +we'll need to update the `watched_dir` and `output_dir`. + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress' +poll_duration = 1 +alert = 'None' +level = 'Debug' +email = '' +threads = 1 +topic = '' +``` + +```{r} +slurmtools::generate_nmm_config( + mod, + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") +``` + +This updates the `1001.toml` config file to: + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' +poll_duration = 1 +alert = 'None' +level = 'Debug' +email = '' +threads = 1 +topic = '' +``` + +We can now run `submit_nonmem_model` and get essentially the same +behavior as running with `bbi`. On linux `~/.local/bin/` will be on your path so saving binaries there is a good approach. + +```{r} +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm")) +) + +submission_nmm +``` + +```{r} +slurmtools::get_slurm_jobs() +``` + +The one difference between using `nmm` compared to `bbi` is that a new +directory is created that contains a log file that caught some issues +with our run. This file is updated as nonmem is running and monitors +gradient values, parameters that hit zero, as well as other errors from +bbi. Looking at the first few lines we can see that `bbi` was +successfully able to call nonmem. We also see an info level log that +OMEGA(2,1) has 0 value – in our mod file we don't specify any omega +values off the diagonal so these are fixed at 0. Finally we see that +GRD(6) hit 0 relatively early in the run. + +``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Successfully loaded default configuration from /cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/bbi.yaml" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Beginning Local Path" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="A total of 1 models have completed the initial preparation phase" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="[1001] Beginning local work phase" +20:12:58 [INFO] OMEGA(2,1) has 0 value +20:12:58 [INFO] SIGMA(2,1) has 0 value +20:13:00 [INFO] SIGMA(2,1) has 0 value +20:13:00 [INFO] OMEGA(2,1) has 0 value +20:13:04 [INFO] SIGMA(2,1) has 0 value +20:13:04 [INFO] OMEGA(2,1) has 0 value +20:13:04 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd" has 0 gradient for parameter: GRD(6) +``` + +After a run has finished several messages are sent to the log after a +final check of the files listed in the `files_to_track` field of the +`1001.toml` file. + +``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log +20:13:16 [INFO] Received Exit code: exit status: 0 +20:13:16 [WARN] 1001.ext: Missing ext final output lines. Observed lines were: [-1000000000.0, -1000000004.0, -1000000006.0, -1000000007.0] +20:13:16 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd": The following parameters hit zero gradient through the run: ["GRD(6)"] +``` + +We see that GRD(6) hit zero during the run and that only a subset of the +-1E9 lines were present in the .ext file. + +## Getting alerted during a run + +Like we did with `bbi` and altering the slurm template file to get +notifications from [ntfy.sh](ntfy.sh) `nmm` has this feature built in! +The messages in the log file that relate to zero gradients, missing -1E9 +lines, and 0 parameter values can also be sent to ntfy by altering the +`1001.toml` file. We can get these alerts in real time without having to +dig through a noisy log file. + +Let's update our call to `generate_nmm_config` to have `nmm` send +notifications to the `NONMEMmonitor` topic on [ntfy.sh](ntfy.sh). + +```{r} +slurmtools::generate_nmm_config( + mod, + alert = "Ntfy", + topic = "NONMEMmonitor", + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") +``` + +This updates the `1001.toml` file to this: + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' +poll_duration = 1 +alert = 'Ntfy' +level = 'Debug' +email = '' +threads = 1 +topic = 'NONMEMmonitor' +``` + +When we re-run the `submit_nonmem_model` call we will now get ntfy +notifications. One thing to note is that `nmm` will print full paths in +the log, but will only send notifications with the `model_number` (or +`model_number.file_extension`). + +```{r} +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm")) +) + +submission_nmm +``` + +```{r} +slurmtools::get_slurm_jobs(user = "matthews") +``` + +This gives us the notifications in a much more digestible format + +![nmm ntfy.sh alerts](data/images/nmm_ntfy_alerts.png) + +```{r, include = FALSE} +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` diff --git a/inst/doc/custom-alerts.html b/inst/doc/custom-alerts.html new file mode 100644 index 0000000..2916582 --- /dev/null +++ b/inst/doc/custom-alerts.html @@ -0,0 +1,590 @@ + + + + + + + + + + + + + + +custom-alerts + + + + + + + + + + + + + + + + + + + + + + + + + + +

custom-alerts

+ + + +
library(slurmtools)
+#> ── Needed slurmtools options ───────────────────────────────────────────────────
+#> ✖ option('slurmtools.slurm_job_template_path') is not set.
+#> ✖ option('slurmtools.submission_root') is not set.
+#> ✖ option('slurmtools.bbi_config_path') is not set.
+#> ℹ Please set all options for job submission defaults to work.
+library(bbr)
+library(here)
+#> here() starts at /cluster-data/user-homes/matthews/Packages/slurmtools
+
+nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
+options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
+
+

Submitting a NONMEM job with nmm

+

Instead of using bbi we can use nmm (NONMEM Monitor) which +currently has some additional functionality of sending notifications +about zero gradients, missing -1E9 lines in ext file, and some very +basic control stream errors. Currently, only slack or ntfy.sh is supported for receiving notifications. To +use nmm you can install the latest release from the github +repository linked above.

+

We can update the template file accordingly:

+
#!/bin/bash 
+#SBATCH --job-name="{{job_name}}" 
+#SBATCH --nodes=1 
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task={{ncpu}} 
+#SBATCH --partition={{partition}}
+
+{{nmm_exe_path}} -c {{config_toml_path}} run
+

default, submit_nonmem_model will provide +nmm_exe_path and config_toml_path to the +template. Just like with bbi_exe_path, +nmm_exe_path is determined with +Sys.which("nmm") which may or may not give you the path to +the nmm binary if it is on your path or not. We can inject the +nmm_exe_path like we did with bbi_exe_path and +assume it’s not on our path.

+

The config.toml file controls what nmm will +monitor and where to look for files and how to alert you. We’ll use +generate_nmm_config() to create this file. First we can +look at the documentation to see what type of information we should pass +to this function.documentation for generate_nmm_config()

+
mod_number <- "1001"
+
+if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
+  mod <- bbr::read_model(file.path(nonmem, mod_number))
+} else {
+  mod <- bbr::new_model(file.path(nonmem, mod_number))
+}
+
slurmtools::generate_nmm_config(mod)
+

This generates the following toml file. Notice that alert is set to +‘None’, and both email and topic are empty. Since we’re in vignettes +we’ll need to update the watched_dir and +output_dir.

+
model_number = '1001'
+files_to_track = [ 'lst', 'ext', 'grd' ]
+tmp_dir = '/tmp'
+watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem'
+output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress'
+poll_duration = 1
+alert = 'None'
+level = 'Debug'
+email = ''
+threads = 1
+topic = ''
+
slurmtools::generate_nmm_config( 
+  mod, 
+  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
+  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
+

This updates the 1001.toml config file to:

+
model_number = '1001'
+files_to_track = [ 'lst', 'ext', 'grd' ]
+tmp_dir = '/tmp'
+watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
+output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
+poll_duration = 1
+alert = 'None'
+level = 'Debug'
+email = ''
+threads = 1
+topic = ''
+

We can now run submit_nonmem_model and get essentially +the same behavior as running with bbi. On linux +~/.local/bin/ will be on your path so saving binaries there +is a good approach.

+
submission_nmm <- slurmtools::submit_nonmem_model( 
+  mod, 
+  overwrite = TRUE,
+  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
+  slurm_template_opts = list(
+    nmm_exe_path = normalizePath("~/.local/bin/nmm"))
+)
+
+submission_nmm
+#> $status
+#> [1] 0
+#> 
+#> $stdout
+#> [1] "Submitted batch job 876\n"
+#> 
+#> $stderr
+#> [1] ""
+#> 
+#> $timeout
+#> [1] FALSE
+
slurmtools::get_slurm_jobs()
+#> # A tibble: 9 × 10
+#>   job_id job_state  cpus partition  standard_input standard_output              
+#>    <int> <chr>     <int> <chr>      <chr>          <chr>                        
+#> 1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
+#> 2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
+#> 3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
+#> 4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
+#> 5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
+#> 6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
+#> 7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
+#> 8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
+#> 9    876 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
+#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
+#> #   current_working_directory <chr>
+

The one difference between using nmm compared to +bbi is that a new directory is created that contains a log +file that caught some issues with our run. This file is updated as +nonmem is running and monitors gradient values, parameters that hit +zero, as well as other errors from bbi. Looking at the first few lines +we can see that bbi was successfully able to call nonmem. +We also see an info level log that OMEGA(2,1) has 0 value – in our mod +file we don’t specify any omega values off the diagonal so these are +fixed at 0. Finally we see that GRD(6) hit 0 relatively early in the +run.

+
20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Successfully loaded default configuration from /cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/bbi.yaml"
+20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Beginning Local Path"
+20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="A total of 1 models have completed the initial preparation phase"
+20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="[1001] Beginning local work phase"
+20:12:58 [INFO] OMEGA(2,1) has 0 value
+20:12:58 [INFO] SIGMA(2,1) has 0 value
+20:13:00 [INFO] SIGMA(2,1) has 0 value
+20:13:00 [INFO] OMEGA(2,1) has 0 value
+20:13:04 [INFO] SIGMA(2,1) has 0 value
+20:13:04 [INFO] OMEGA(2,1) has 0 value
+20:13:04 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd" has 0 gradient for parameter: GRD(6)
+

After a run has finished several messages are sent to the log after a +final check of the files listed in the files_to_track field +of the 1001.toml file.

+
20:13:16 [INFO] Received Exit code: exit status: 0
+20:13:16 [WARN] 1001.ext: Missing ext final output lines. Observed lines were: [-1000000000.0, -1000000004.0, -1000000006.0, -1000000007.0]
+20:13:16 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd": The following parameters hit zero gradient through the run: ["GRD(6)"]
+

We see that GRD(6) hit zero during the run and that only a subset of +the -1E9 lines were present in the .ext file.

+
+
+

Getting alerted during a run

+

Like we did with bbi and altering the slurm template +file to get notifications from ntfy.sh +nmm has this feature built in! The messages in the log file +that relate to zero gradients, missing -1E9 lines, and 0 parameter +values can also be sent to ntfy by altering the 1001.toml +file. We can get these alerts in real time without having to dig through +a noisy log file.

+

Let’s update our call to generate_nmm_config to have +nmm send notifications to the NONMEMmonitor +topic on ntfy.sh.

+
slurmtools::generate_nmm_config( 
+  mod, 
+  alert = "Ntfy",
+  topic = "NONMEMmonitor",
+  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
+  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
+

This updates the 1001.toml file to this:

+
model_number = '1001'
+files_to_track = [ 'lst', 'ext', 'grd' ]
+tmp_dir = '/tmp'
+watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
+output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
+poll_duration = 1
+alert = 'Ntfy'
+level = 'Debug'
+email = ''
+threads = 1
+topic = 'NONMEMmonitor'
+

When we re-run the submit_nonmem_model call we will now +get ntfy notifications. One thing to note is that nmm will +print full paths in the log, but will only send notifications with the +model_number (or +model_number.file_extension).

+
submission_nmm <- slurmtools::submit_nonmem_model( 
+  mod, 
+  overwrite = TRUE,
+  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
+  slurm_template_opts = list(
+    nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm"))
+)
+#> Warning in normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm"):
+#> path[1]="/cluster-data/user-homes/matthews/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm":
+#> No such file or directory
+
+submission_nmm
+#> $status
+#> [1] 0
+#> 
+#> $stdout
+#> [1] "Submitted batch job 877\n"
+#> 
+#> $stderr
+#> [1] ""
+#> 
+#> $timeout
+#> [1] FALSE
+
slurmtools::get_slurm_jobs(user = "matthews")
+#> # A tibble: 10 × 10
+#>    job_id job_state  cpus partition  standard_input standard_output             
+#>     <int> <chr>     <int> <chr>      <chr>          <chr>                       
+#>  1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  9    876 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> 10    877 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
+#> #   current_working_directory <chr>
+

This gives us the notifications in a much more digestible format

+
+nmm ntfy.sh alerts +
nmm ntfy.sh alerts
+
+
+ + + + + + + + + + + diff --git a/inst/doc/slack-alerts.R b/inst/doc/slack-alerts.R new file mode 100644 index 0000000..1e97821 --- /dev/null +++ b/inst/doc/slack-alerts.R @@ -0,0 +1,78 @@ +## ----include = FALSE---------------------------------------------------------- +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + +## ----include = FALSE---------------------------------------------------------- +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) + +## ----setup-------------------------------------------------------------------- +library(slurmtools) +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) + +## ----------------------------------------------------------------------------- +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} + +## ----------------------------------------------------------------------------- +Sys.which("slack_notifier") + +## ----------------------------------------------------------------------------- +slurmtools::generate_nmm_config( + mod, + alert = "slack", + email = "matthews@a2-ai.com", + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") + +## ----------------------------------------------------------------------------- +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm") + ) +) + +submission_nmm + +## ----------------------------------------------------------------------------- +slurmtools::get_slurm_jobs() + +## ----include = FALSE---------------------------------------------------------- +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) + diff --git a/inst/doc/slack-alerts.Rmd b/inst/doc/slack-alerts.Rmd new file mode 100644 index 0000000..15731f8 --- /dev/null +++ b/inst/doc/slack-alerts.Rmd @@ -0,0 +1,145 @@ +--- +title: "slack-alerts" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{slack-alerts} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + markdown: + wrap: 72 +--- + +```{r, include = FALSE} +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(slurmtools) +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +``` + +```{r} +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} +``` + +# Cut me some Slack + +There is also functionality to pair `nmm` with +[slack_notifier](https://github.com/A2-ai/slack_notifier/releases/) and +get messages sent directly to you via a slack bot. This requires you to +download the slack_notifier binaries and added them to your path so +`nmm` can find it. You can download the latest release and extract the +binary and again save it to `~/.local/bin`. + +```{r} +Sys.which("slack_notifier") +``` + +slack_notifier requires an additional `slack_notifier/config.yaml` file +that contains the slack bot OAuth token which is found from +[[[https://api.slack.com/apps/\\\\](https://api.slack.com/apps/){.uri}](%5Bhttps://api.slack.com/apps/%5D(https://api.slack.com/apps/)%7B.uri%7D){.uri}\ /oauth?]. + +``` slack_notifier/config.yaml +SLACK_OAUTH_TOKEN: "encrypted(Bot User OAuth Token)" +``` + +Again, we need to update the `1001.toml` file to get slack +notifications. We need to set `alert = "slack"` and provide the `email` +associated with the slack account in `generate_nmm_config`. + +```{r} +slurmtools::generate_nmm_config( + mod, + alert = "slack", + email = "matthews@a2-ai.com", + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") +``` + +This generates the following toml file: + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' +poll_duration = 1 +alert = 'Slack' +level = 'Debug' +email = 'matthews@a2-ai.com' +threads = 1 +topic = '' +``` + +With `alert = 'Slack'` and `email` set in the `1001.toml` file `nmm` +will send slack notifications directly to you when a NONMEM run starts +and it will reply to that message with notifications if any gradients +hit 0 and when the run finishes it checks if all -1E9 lines are present +in the .ext file and gives another message about any parameters that hit +0 gradient. + +```{r} +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm") + ) +) + +submission_nmm +``` + +```{r} +slurmtools::get_slurm_jobs() +``` + +![nmm slack alerts](data/images/nmm_slack_notifications.png) + +```{r, include = FALSE} +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` diff --git a/inst/doc/slack-alerts.html b/inst/doc/slack-alerts.html new file mode 100644 index 0000000..8d96425 --- /dev/null +++ b/inst/doc/slack-alerts.html @@ -0,0 +1,461 @@ + + + + + + + + + + + + + + +slack-alerts + + + + + + + + + + + + + + + + + + + + + + + + + + +

slack-alerts

+ + + +
library(slurmtools)
+library(bbr)
+library(here)
+
+nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
+options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
+
mod_number <- "1001"
+
+if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
+  mod <- bbr::read_model(file.path(nonmem, mod_number))
+} else {
+  mod <- bbr::new_model(file.path(nonmem, mod_number))
+}
+
+

Cut me some Slack

+

There is also functionality to pair nmm with slack_notifier +and get messages sent directly to you via a slack bot. This requires you +to download the slack_notifier binaries and added them to your path so +nmm can find it. You can download the latest release and +extract the binary and again save it to ~/.local/bin.

+
Sys.which("slack_notifier")
+#>                                                slack_notifier 
+#> "/cluster-data/user-homes/matthews/.local/bin/slack_notifier"
+

slack_notifier requires an additional +slack_notifier/config.yaml file that contains the slack bot +OAuth token which is found from [[https://api.slack.com/apps/\\](https://api.slack.com/apps/){.uri}<YOUR +APP ID> /oauth?].

+
SLACK_OAUTH_TOKEN: "encrypted(Bot User OAuth Token)"
+

Again, we need to update the 1001.toml file to get slack +notifications. We need to set alert = "slack" and provide +the email associated with the slack account in +generate_nmm_config.

+
slurmtools::generate_nmm_config( 
+  mod, 
+  alert = "slack",
+  email = "matthews@a2-ai.com",
+  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
+  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
+

This generates the following toml file:

+
model_number = '1001'
+files_to_track = [ 'lst', 'ext', 'grd' ]
+tmp_dir = '/tmp'
+watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
+output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
+poll_duration = 1
+alert = 'Slack'
+level = 'Debug'
+email = 'matthews@a2-ai.com'
+threads = 1
+topic = ''
+

With alert = 'Slack' and email set in the +1001.toml file nmm will send slack +notifications directly to you when a NONMEM run starts and it will reply +to that message with notifications if any gradients hit 0 and when the +run finishes it checks if all -1E9 lines are present in the .ext file +and gives another message about any parameters that hit 0 gradient.

+
submission_nmm <- slurmtools::submit_nonmem_model( 
+  mod, 
+  overwrite = TRUE,
+  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
+  slurm_template_opts = list(
+    nmm_exe_path = normalizePath("~/.local/bin/nmm")
+  )
+)
+
+submission_nmm
+#> $status
+#> [1] 0
+#> 
+#> $stdout
+#> [1] "Submitted batch job 880\n"
+#> 
+#> $stderr
+#> [1] ""
+#> 
+#> $timeout
+#> [1] FALSE
+
slurmtools::get_slurm_jobs()
+#> # A tibble: 13 × 10
+#>    job_id job_state  cpus partition  standard_input standard_output             
+#>     <int> <chr>     <int> <chr>      <chr>          <chr>                       
+#>  1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#>  4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
+#>  9    876 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> 10    877 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> 11    878 RUNNING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> 12    879 COMPLETED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> 13    880 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
+#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
+#> #   current_working_directory <chr>
+
+nmm slack alerts +
nmm slack alerts
+
+
+ + + + + + + + + + + From 6efb77699605d7c0e63d9b3e5704b77f6b149e26 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 20:19:12 +0000 Subject: [PATCH 17/38] why not just install slurm? --- .github/workflows/RunChecks.yaml | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 9735294..b208eaa 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -35,32 +35,10 @@ jobs: r-version: ${{ matrix.config.r }} use-public-rspm: true - - name: Download mocked binaries + - name: Install Slurm tools run: | - mkdir -p /usr/local/bin - cd /tmp - gh release download "sinfo/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ - tar -xvf /usr/local/bin/sinfo_Linux_x86_64.tar.gz - mv ./sinfo /usr/local/bin/sinfo - chmod +x /usr/local/bin/sinfo - - gh release download "sacct/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ - tar -xvf /usr/local/bin/sacct_Linux_x86_64.tar.gz - mv ./sacct /usr/local/bin/sacct - chmod +x /usr/local/bin/sacct - - echo "/usr/local/bin" >> $GITHUB_PATH - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Grab outputs directory from mocked repo - run: | - cd /tmp - gh repo clone a2-ai-tech-training/slurm-mocks - mv /tmp/slurm-mocks/sinfo/outputs $GITHUB_WORKSPACE/ - rm -rf /tmp/slurm-mocks - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + sudo apt-get update + sudo apt-get install -y slurm-client - uses: r-lib/actions/setup-r-dependencies@v2 with: @@ -74,4 +52,5 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - build_args: '"--no-build-vignettes"' + + From 44a6b52dd9094c81e27d5270521a0e5619e6cdb1 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 6 Sep 2024 20:41:40 +0000 Subject: [PATCH 18/38] back to allowing building of vignettes because the error still revolves around slurm. --- .github/workflows/RunChecks.yaml | 30 +- inst/doc/Running-nonmem.R | 81 ---- inst/doc/Running-nonmem.Rmd | 269 ------------- inst/doc/Running-nonmem.html | 638 ------------------------------- inst/doc/custom-alerts.R | 97 ----- inst/doc/custom-alerts.Rmd | 267 ------------- inst/doc/custom-alerts.html | 590 ---------------------------- inst/doc/slack-alerts.R | 78 ---- inst/doc/slack-alerts.Rmd | 145 ------- inst/doc/slack-alerts.html | 461 ---------------------- 10 files changed, 25 insertions(+), 2631 deletions(-) delete mode 100644 inst/doc/Running-nonmem.R delete mode 100644 inst/doc/Running-nonmem.Rmd delete mode 100644 inst/doc/Running-nonmem.html delete mode 100644 inst/doc/custom-alerts.R delete mode 100644 inst/doc/custom-alerts.Rmd delete mode 100644 inst/doc/custom-alerts.html delete mode 100644 inst/doc/slack-alerts.R delete mode 100644 inst/doc/slack-alerts.Rmd delete mode 100644 inst/doc/slack-alerts.html diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index b208eaa..befc751 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -35,10 +35,32 @@ jobs: r-version: ${{ matrix.config.r }} use-public-rspm: true - - name: Install Slurm tools + - name: Download mocked binaries run: | - sudo apt-get update - sudo apt-get install -y slurm-client + mkdir -p /usr/local/bin + cd /tmp + gh release download "sinfo/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ + tar -xvf /usr/local/bin/sinfo_Linux_x86_64.tar.gz + mv ./sinfo /usr/local/bin/sinfo + chmod +x /usr/local/bin/sinfo + + gh release download "sacct/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ + tar -xvf /usr/local/bin/sacct_Linux_x86_64.tar.gz + mv ./sacct /usr/local/bin/sacct + chmod +x /usr/local/bin/sacct + + echo "/usr/local/bin" >> $GITHUB_PATH + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Grab outputs directory from mocked repo + run: | + cd /tmp + gh repo clone a2-ai-tech-training/slurm-mocks + mv /tmp/slurm-mocks/sinfo/outputs $GITHUB_WORKSPACE/ + rm -rf /tmp/slurm-mocks + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: r-lib/actions/setup-r-dependencies@v2 with: @@ -52,5 +74,3 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - - diff --git a/inst/doc/Running-nonmem.R b/inst/doc/Running-nonmem.R deleted file mode 100644 index 0f22cd3..0000000 --- a/inst/doc/Running-nonmem.R +++ /dev/null @@ -1,81 +0,0 @@ -## ----include = FALSE---------------------------------------------------------- -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - -## ----include = FALSE---------------------------------------------------------- -knitr::opts_chunk$set( - collapse = TRUE, - comment = "" -) - -## ----setup-------------------------------------------------------------------- -library(slurmtools) - -## ----------------------------------------------------------------------------- -Sys.which("bbi") - -## ----------------------------------------------------------------------------- -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") - -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) -options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml")) - -## ----------------------------------------------------------------------------- -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} - -## ----------------------------------------------------------------------------- -submission <- slurmtools::submit_nonmem_model( - mod, - slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"), -) - -submission - -## ----------------------------------------------------------------------------- -slurmtools::get_slurm_jobs(user = 'matthews') - -## ----------------------------------------------------------------------------- -submission_ntfy <- slurmtools::submit_nonmem_model( - mod, - slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"), - overwrite = TRUE, - slurm_template_opts = list( - ntfy = "ntfy_demo") -) - -submission_ntfy - -## ----include = FALSE---------------------------------------------------------- -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - diff --git a/inst/doc/Running-nonmem.Rmd b/inst/doc/Running-nonmem.Rmd deleted file mode 100644 index ce7a020..0000000 --- a/inst/doc/Running-nonmem.Rmd +++ /dev/null @@ -1,269 +0,0 @@ ---- -title: "Running Nonmem with slurmtools" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{Running Nonmem with slurmtools} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -```{r, include = FALSE} -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "" -) -``` - -# Slurmtools for submitting NONMEM runs - -`slurmtools` is an R package for interacting with slurm (fka **S**imple **L**inux **U**tility for **R**esource **M**anagement) and submitting NONMEM jobs. You can submit a NONMEM job with `submit_nonmem_model`, you can view current jobs with `get_slurm_jobs`, and you can see the available partitions with `get_slurm_partitions`. - -## Installing `slurmtools` - -To install `slurmtools` use the following commands: - -``` r -options(repos = c( - "slurmtools" = "https://a2-ai.github.io/gh-pkg-mirror/slurmtools", - getOption("repos"))) -install.packages("slurmtools") -``` - -```{r setup} -library(slurmtools) -``` - -We are given a message when loading slurmtools that some options are not set and that default job submission will not work without them. These options are used for default arguments in the `submit_nonmem_model` function. Running `?submit_nonmem_model` we can see the documentation - -![Help view for `submit_nonmem_model` function](data/images/submit_nonmem_model_help.png) - -This function uses the inputs to populate a template Bash shell script that submits the NONMEM job to slurm. A default template file is supplied with the Project Starter and it can be modified to do additional tasks as long as they are possible within Bash. - -By default these values are provided to the slurm template file: - -``` r -default_template_list = list( - partition = partition, - parallel = parallel, - ncpu = ncpu, - job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)), - project_path = project_path, - project_name = project_name, - bbi_exe_path = Sys.which("bbi"), - bbi_config_path = bbi_config_path, - model_path = .mod$absolute_model_path, - config_toml_path = config_toml_path, - nmm_exe_path = Sys.which("nmm") -) -``` - -- `partition` is an argument to `submit_nonmem_model` - -- `parallel` is `TRUE` if `ncpu > 1`, else `FALSE` - -- `ncpu` is an argument to `submit_nonmem_model` - -- `job_name` is created from the `.mod` argument supplied to `submit_nonmem_model` - -- `bbi_exe_path` is determined via \`Sys.which("bbi") - -- `bbi_config_path` is determined via getOption("slurmtools.bbi_config_path") - -- `model_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` - -- `config_toml_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` and is requried to use `nmm` (NONMEM monitor) - -- `nmm_exe_path` is determined via `Sys.which("nmm")` - -If you need to feed more arguments to the template you simply supply them in the `slurm_template_opts` argument as a list. More on that later. - -## Submitting a NONMEM job with `bbi` - -To submit a NONMEM job, we need to supply either the path to a mod file or create a model object from `bbr`, and supply a `slurm-template.tmpl` file. To use `bbi` we also need a `bbi.yaml` file, which I've also supplied in `/model/nonmem/bbi.yaml` (and is also supplied with the R project starter). - -Here is an example of a template file that will call `bbi`: - -``` slurm-job-bbi.tmpl -#!/bin/bash -#SBATCH --job-name="{{job_name}}" -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task={{ncpu}} -#SBATCH --partition={{partition}} -#SBATCH --account={{project_name}} - -# submit_nonmem_model uses the whisker package to populate template files -# https://github.com/edwindj/whisker - -{{#parallel}} -{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}} -{{/parallel}} - - -{{^parallel}} -{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}} -{{/parallel}} -``` - -This file will call `bbi` to run our supplied model (`{{model_path}}.mod`) if `ncpu > 1` then parallel will be true and the code between `{{#parallel}}` and `{{/parallel}}` will be populated. if `ncpu = 1` then parallel will be false and the code between `{{^parallel}}` and `{{/parallel}}` will be populated. By default, `submit_nonmem_model` will inject `Sys.which("bbi")` into the template, so if `bbi` is not on your path we'll have to supply the `bbi_exe_path` for it to start the NONMEM run. - -```{r} -Sys.which("bbi") -``` - -We will use a few different template files with different functionality so we'll inject those template file paths to `submit_nonmem_model`. However, we'll use the `submission-log` directory for the output, so we'll set that option as well as `bbi_config_path` so `submit_nonmem_model` defaults can be used. The slurm template files are saved in `~/model/nonmem/` Additionally, there is a simple NONMEM control stream in `1001.mod` in the same directory that we can use for testing. - -```{r} -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") - -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) -options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml")) -``` - -To create the `bbr` model object, we need to have both `1001.mod` and `1001.yaml` which contains metadata about the model in the supplied directory (`./model/nonmem/`). We'll check for mod_number.yaml and if it exists, read in the model otherwise create it and then read it. - -```{r} -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} -``` - -We can now submit the job and point to the template file in `model/nonmem/slurm-job-bbi.tmpl`. - -```{r} -submission <- slurmtools::submit_nonmem_model( - mod, - slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"), -) - -submission -``` - -We see a `status` with an exit code of 0 suggesting a successful command, and the `stdout` gives us the batch job number. We can use `slurmtools::get_slurm_jobs()` to monitor the status of the job. Here, we can supply the user = "matthews" argument to filter to just the jobs I've submitted. - -```{r} -slurmtools::get_slurm_jobs(user = 'matthews') -``` - -If we look in the `slurmtools.submisstion_root` directory we can see the shell script that was generated with `submit_nonmem_model`. Here is the whisker replaced call to bbi: - -``` 1001.sh -/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml -``` - -## Extending templates - -Because the templates create a bash shell script there is an almost infinite number of things we can do with our template. **Anything you can do in bash you can do by appropriately updating the template file and injecting the needed information!** - -Let's add a notification feature that will send a notification when the job has started and finished. We can use [ntfy.sh](ntfy.sh) and add the necessary info to our template to achieve this. - -Here is a modified template file that adds a `JOBID=$SLURM_JOBID` and some ntfy calls. To get a notification we can supply `submit_nonmem_model` with `ntfy` variable to send notifications. I'll use `ntfy = ntfy_demo` for this. - -``` slurm-job-bbi-ntfy.tmpl -#!/bin/bash -#SBATCH --job-name="{{job_name}}" -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task={{ncpu}} -#SBATCH --partition={{partition}} -#SBATCH --account={{project_name}} - -JOBID=$SLURM_JOBID - -# submit_nonmem_model uses the whisker package to populate template files -# https://github.com/edwindj/whisker - -{{#ntfy}} -curl -d "Starting model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}} -{{/ntfy}} - -{{#parallel}} -{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}} -{{/parallel}} - -{{^parallel}} -{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}} -{{/parallel}} - -{{#ntfy}} -curl -d "Finished model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}} -{{/ntfy}} -``` - -Since we've already run this model we will provide the `overwrite = TRUE` argument to force a new nonmem run. - -```{r} -submission_ntfy <- slurmtools::submit_nonmem_model( - mod, - slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"), - overwrite = TRUE, - slurm_template_opts = list( - ntfy = "ntfy_demo") -) - -submission_ntfy -``` - -We again get a 0 exit code status and now instead of using `slurmtools::get_slurm_jobs()` to monitor the job, we can rely on the new notifications we just set up. ![NONMEM job starting ntfy alert](data/images/ntfy_starting.png) - -and when the run finished we get another notification: ![NONMEM Job finished ntfy alert](data/images/ntfy_finished.png) - -Note that the run number will match the run specified in `submission$stdout`. We can see the new shell script this updated template file generated - -``` 1001.sh -#!/bin/bash -#SBATCH --job-name="1001-nonmem-run" -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=1 -#SBATCH --partition=cpu2mem4gb -#SBATCH --account=slurmtools - -JOBID=$SLURM_JOBID - -curl -d "Starting model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo - -/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml - -curl -d "Finished model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo -``` - -To reiterate, this template file is run as a bash shell script so anything you can do in bash you can put into the template and pass the needed arguments and customize the behavior to your liking. - -```{r, include = FALSE} -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` diff --git a/inst/doc/Running-nonmem.html b/inst/doc/Running-nonmem.html deleted file mode 100644 index 94827a2..0000000 --- a/inst/doc/Running-nonmem.html +++ /dev/null @@ -1,638 +0,0 @@ - - - - - - - - - - - - - - -Running Nonmem with slurmtools - - - - - - - - - - - - - - - - - - - - - - - - - - -

Running Nonmem with slurmtools

- - - -
-

Slurmtools for submitting NONMEM runs

-

slurmtools is an R package for interacting with slurm -(fka Simple Linux -Utility for Resource -Management) and submitting NONMEM jobs. You can submit -a NONMEM job with submit_nonmem_model, you can view current -jobs with get_slurm_jobs, and you can see the available -partitions with get_slurm_partitions.

-
-

Installing slurmtools

-

To install slurmtools use the following commands:

-
options(repos = c(
-  "slurmtools" = "https://a2-ai.github.io/gh-pkg-mirror/slurmtools", 
-  getOption("repos")))
-install.packages("slurmtools")
-
library(slurmtools)
-

We are given a message when loading slurmtools that some options are -not set and that default job submission will not work without them. -These options are used for default arguments in the -submit_nonmem_model function. Running -?submit_nonmem_model we can see the documentation

-
-Help view for submit_nonmem_model function -
Help view for submit_nonmem_model -function
-
-

This function uses the inputs to populate a template Bash shell -script that submits the NONMEM job to slurm. A default template file is -supplied with the Project Starter and it can be modified to do -additional tasks as long as they are possible within Bash.

-

By default these values are provided to the slurm template file:

-
default_template_list = list(
-  partition = partition,
-  parallel = parallel,
-  ncpu = ncpu,
-  job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)),
-  project_path = project_path,
-  project_name = project_name,
-  bbi_exe_path = Sys.which("bbi"),
-  bbi_config_path = bbi_config_path,
-  model_path = .mod$absolute_model_path,
-  config_toml_path = config_toml_path,
-  nmm_exe_path = Sys.which("nmm")
-)
-
    -

  • partition is an argument to -submit_nonmem_model

    • -

  • parallel is TRUE if -ncpu > 1, else FALSE

    • -

  • ncpu is an argument to -submit_nonmem_model

    • -

  • job_name is created from the .mod -argument supplied to submit_nonmem_model

    • -

  • bbi_exe_path is determined via -`Sys.which(“bbi”)

    • -

  • bbi_config_path is determined via -getOption(“slurmtools.bbi_config_path”)

    • -

  • model_path is determined from the .mod -argument supplied to submit_nonmem_model

    • -

  • config_toml_path is determined from the -.mod argument supplied to submit_nonmem_model -and is requried to use nmm (NONMEM monitor)

    • -

  • nmm_exe_path is determined via -Sys.which("nmm")

    • -
-

If you need to feed more arguments to the template you simply supply -them in the slurm_template_opts argument as a list. More on -that later.

-
-
-

Submitting a NONMEM job with bbi

-

To submit a NONMEM job, we need to supply either the path to a mod -file or create a model object from bbr, and supply a -slurm-template.tmpl file. To use bbi we also -need a bbi.yaml file, which I’ve also supplied in -/model/nonmem/bbi.yaml (and is also supplied with the R -project starter).

-

Here is an example of a template file that will call -bbi:

-
#!/bin/bash
-#SBATCH --job-name="{{job_name}}"
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task={{ncpu}}
-#SBATCH --partition={{partition}}
-#SBATCH --account={{project_name}}
-
-# submit_nonmem_model uses the whisker package to populate template files
-# https://github.com/edwindj/whisker
-
-{{#parallel}}
-{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}}
-{{/parallel}}
-
-
-{{^parallel}}
-{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}}
-{{/parallel}}
-

This file will call bbi to run our supplied model -({{model_path}}.mod) if ncpu > 1 then -parallel will be true and the code between {{#parallel}} -and {{/parallel}} will be populated. if -ncpu = 1 then parallel will be false and the code between -{{^parallel}} and {{/parallel}} will be -populated. By default, submit_nonmem_model will inject -Sys.which("bbi") into the template, so if bbi -is not on your path we’ll have to supply the bbi_exe_path -for it to start the NONMEM run.

-
Sys.which("bbi")
-                 bbi 
-"/usr/local/bin/bbi"
-

We will use a few different template files with different -functionality so we’ll inject those template file paths to -submit_nonmem_model. However, we’ll use the -submission-log directory for the output, so we’ll set that -option as well as bbi_config_path so -submit_nonmem_model defaults can be used. The slurm -template files are saved in ~/model/nonmem/ Additionally, -there is a simple NONMEM control stream in 1001.mod in the -same directory that we can use for testing.

-
library(bbr)
-library(here)
-
-nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
-
-options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
-options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml"))
-

To create the bbr model object, we need to have both -1001.mod and 1001.yaml which contains metadata -about the model in the supplied directory -(./model/nonmem/). We’ll check for mod_number.yaml and if -it exists, read in the model otherwise create it and then read it.

-
mod_number <- "1001"
-
-if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
-  mod <- bbr::read_model(file.path(nonmem, mod_number))
-} else {
-  mod <- bbr::new_model(file.path(nonmem, mod_number))
-}
-

We can now submit the job and point to the template file in -model/nonmem/slurm-job-bbi.tmpl.

-
submission <- slurmtools::submit_nonmem_model(
-  mod,
-  slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"),
-)
-
-submission
-$status
-[1] 0
-
-$stdout
-[1] "Submitted batch job 878\n"
-
-$stderr
-[1] ""
-
-$timeout
-[1] FALSE
-

We see a status with an exit code of 0 suggesting a -successful command, and the stdout gives us the batch job -number. We can use slurmtools::get_slurm_jobs() to monitor -the status of the job. Here, we can supply the user = “matthews” -argument to filter to just the jobs I’ve submitted.

-
slurmtools::get_slurm_jobs(user = 'matthews')
-# A tibble: 11 × 10
-   job_id job_state  cpus partition  standard_input standard_output             
-    <int> <chr>     <int> <chr>      <chr>          <chr>                       
- 1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
- 2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
- 3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
- 4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
- 5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
- 6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
- 7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
- 8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
- 9    876 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-10    877 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-11    878 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-# ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
-#   current_working_directory <chr>
-

If we look in the slurmtools.submisstion_root directory -we can see the shell script that was generated with -submit_nonmem_model. Here is the whisker replaced call to -bbi:

-
/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml
-
-
-

Extending templates

-

Because the templates create a bash shell script there is an almost -infinite number of things we can do with our template. Anything -you can do in bash you can do by appropriately updating the template -file and injecting the needed information!

-

Let’s add a notification feature that will send a notification when -the job has started and finished. We can use ntfy.sh and add the necessary info to our template to -achieve this.

-

Here is a modified template file that adds a -JOBID=$SLURM_JOBID and some ntfy calls. To get a -notification we can supply submit_nonmem_model with -ntfy variable to send notifications. I’ll use -ntfy = ntfy_demo for this.

-
#!/bin/bash
-#SBATCH --job-name="{{job_name}}"
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task={{ncpu}}
-#SBATCH --partition={{partition}}
-#SBATCH --account={{project_name}}
-
-JOBID=$SLURM_JOBID
-
-# submit_nonmem_model uses the whisker package to populate template files
-# https://github.com/edwindj/whisker
-
-{{#ntfy}}
-curl -d "Starting model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}}
-{{/ntfy}}
-
-{{#parallel}}
-{{bbi_exe_path}} nonmem run local {{model_path}}.mod --parallel --threads={{ncpu}} --config {{bbi_config_path}}
-{{/parallel}}
-
-{{^parallel}}
-{{bbi_exe_path}} nonmem run local {{model_path}}.mod --config {{bbi_config_path}}
-{{/parallel}}
-
-{{#ntfy}}
-curl -d "Finished model run: {{job_name}} $JOBID" ntfy.sh/{{ntfy}}
-{{/ntfy}}
-

Since we’ve already run this model we will provide the -overwrite = TRUE argument to force a new nonmem run.

-
submission_ntfy <- slurmtools::submit_nonmem_model(
-  mod, 
-  slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"),
-  overwrite = TRUE,
-  slurm_template_opts = list(
-    ntfy = "ntfy_demo")
-)
-
-submission_ntfy
-$status
-[1] 0
-
-$stdout
-[1] "Submitted batch job 879\n"
-
-$stderr
-[1] ""
-
-$timeout
-[1] FALSE
-

We again get a 0 exit code status and now instead of using -slurmtools::get_slurm_jobs() to monitor the job, we can -rely on the new notifications we just set up. NONMEM job starting ntfy alert

-

and when the run finished we get another notification: NONMEM Job finished ntfy alert

-

Note that the run number will match the run specified in -submission$stdout. We can see the new shell script this -updated template file generated

-
#!/bin/bash
-#SBATCH --job-name="1001-nonmem-run"
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=1
-#SBATCH --partition=cpu2mem4gb
-#SBATCH --account=slurmtools
-
-JOBID=$SLURM_JOBID
-
-curl -d "Starting model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo
-
-/usr/local/bin/bbi nonmem run local /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/1001.mod --config /cluster-data/user-homes/matthews/Projects/slurmtools_vignette/model/nonmem/bbi.yaml
-
-curl -d "Finished model run: 1001-nonmem-run $JOBID" ntfy.sh/ntfy_demo
-

To reiterate, this template file is run as a bash shell script so -anything you can do in bash you can put into the template and pass the -needed arguments and customize the behavior to your liking.

-
-
- - - - - - - - - - - diff --git a/inst/doc/custom-alerts.R b/inst/doc/custom-alerts.R deleted file mode 100644 index 100bd0e..0000000 --- a/inst/doc/custom-alerts.R +++ /dev/null @@ -1,97 +0,0 @@ -## ----include = FALSE---------------------------------------------------------- -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - -## ----include = FALSE---------------------------------------------------------- -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) - -## ----setup-------------------------------------------------------------------- -library(slurmtools) -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) - -## ----------------------------------------------------------------------------- -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} - -## ----------------------------------------------------------------------------- -slurmtools::generate_nmm_config(mod) - -## ----------------------------------------------------------------------------- -slurmtools::generate_nmm_config( - mod, - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") - -## ----------------------------------------------------------------------------- -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm")) -) - -submission_nmm - -## ----------------------------------------------------------------------------- -slurmtools::get_slurm_jobs() - -## ----------------------------------------------------------------------------- -slurmtools::generate_nmm_config( - mod, - alert = "Ntfy", - topic = "NONMEMmonitor", - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") - -## ----------------------------------------------------------------------------- -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm")) -) - -submission_nmm - -## ----------------------------------------------------------------------------- -slurmtools::get_slurm_jobs(user = "matthews") - -## ----include = FALSE---------------------------------------------------------- -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - diff --git a/inst/doc/custom-alerts.Rmd b/inst/doc/custom-alerts.Rmd deleted file mode 100644 index 37d6dfe..0000000 --- a/inst/doc/custom-alerts.Rmd +++ /dev/null @@ -1,267 +0,0 @@ ---- -title: "custom-alerts" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{custom-alerts} - %\VignetteEncoding{UTF-8} - %\VignetteEngine{knitr::rmarkdown} -editor_options: - markdown: - wrap: 72 ---- - -```{r, include = FALSE} -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(slurmtools) -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) -``` - -## Submitting a NONMEM job with nmm - -Instead of using bbi we can use `nmm` ([NONMEM -Monitor](https://github.com/A2-ai/nonmem-monitor)) which currently has -some additional functionality of sending notifications about zero -gradients, missing -1E9 lines in ext file, and some very basic control -stream errors. Currently, only slack or [ntfy.sh](ntfy.sh) is supported -for receiving notifications. To use `nmm` you can install the latest -release from the github repository linked above. - -We can update the template file accordingly: - -``` slurm-job-nmm.tmpl -#!/bin/bash -#SBATCH --job-name="{{job_name}}" -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task={{ncpu}} -#SBATCH --partition={{partition}} - -{{nmm_exe_path}} -c {{config_toml_path}} run -``` - -default, `submit_nonmem_model` will provide `nmm_exe_path` and -`config_toml_path` to the template. Just like with `bbi_exe_path`, -`nmm_exe_path` is determined with `Sys.which("nmm")` which may or may -not give you the path to the nmm binary if it is on your path or not. We -can inject the `nmm_exe_path` like we did with `bbi_exe_path` and assume -it's not on our path. - -The `config.toml` file controls what `nmm` will monitor and where to -look for files and how to alert you. We'll use `generate_nmm_config()` -to create this file. First we can look at the documentation to see what -type of information we should pass to this function.![documentation for -generate_nmm_config()](data/images/generate_nmm_config_help.png) - -```{r} -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} -``` - -```{r} -slurmtools::generate_nmm_config(mod) -``` - -This generates the following toml file. Notice that alert is set to -'None', and both email and topic are empty. Since we're in vignettes -we'll need to update the `watched_dir` and `output_dir`. - -``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir = '/tmp' -watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem' -output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress' -poll_duration = 1 -alert = 'None' -level = 'Debug' -email = '' -threads = 1 -topic = '' -``` - -```{r} -slurmtools::generate_nmm_config( - mod, - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") -``` - -This updates the `1001.toml` config file to: - -``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir = '/tmp' -watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' -output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' -poll_duration = 1 -alert = 'None' -level = 'Debug' -email = '' -threads = 1 -topic = '' -``` - -We can now run `submit_nonmem_model` and get essentially the same -behavior as running with `bbi`. On linux `~/.local/bin/` will be on your path so saving binaries there is a good approach. - -```{r} -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm")) -) - -submission_nmm -``` - -```{r} -slurmtools::get_slurm_jobs() -``` - -The one difference between using `nmm` compared to `bbi` is that a new -directory is created that contains a log file that caught some issues -with our run. This file is updated as nonmem is running and monitors -gradient values, parameters that hit zero, as well as other errors from -bbi. Looking at the first few lines we can see that `bbi` was -successfully able to call nonmem. We also see an info level log that -OMEGA(2,1) has 0 value – in our mod file we don't specify any omega -values off the diagonal so these are fixed at 0. Finally we see that -GRD(6) hit 0 relatively early in the run. - -``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log -20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Successfully loaded default configuration from /cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/bbi.yaml" -20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Beginning Local Path" -20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="A total of 1 models have completed the initial preparation phase" -20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="[1001] Beginning local work phase" -20:12:58 [INFO] OMEGA(2,1) has 0 value -20:12:58 [INFO] SIGMA(2,1) has 0 value -20:13:00 [INFO] SIGMA(2,1) has 0 value -20:13:00 [INFO] OMEGA(2,1) has 0 value -20:13:04 [INFO] SIGMA(2,1) has 0 value -20:13:04 [INFO] OMEGA(2,1) has 0 value -20:13:04 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd" has 0 gradient for parameter: GRD(6) -``` - -After a run has finished several messages are sent to the log after a -final check of the files listed in the `files_to_track` field of the -`1001.toml` file. - -``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log -20:13:16 [INFO] Received Exit code: exit status: 0 -20:13:16 [WARN] 1001.ext: Missing ext final output lines. Observed lines were: [-1000000000.0, -1000000004.0, -1000000006.0, -1000000007.0] -20:13:16 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd": The following parameters hit zero gradient through the run: ["GRD(6)"] -``` - -We see that GRD(6) hit zero during the run and that only a subset of the --1E9 lines were present in the .ext file. - -## Getting alerted during a run - -Like we did with `bbi` and altering the slurm template file to get -notifications from [ntfy.sh](ntfy.sh) `nmm` has this feature built in! -The messages in the log file that relate to zero gradients, missing -1E9 -lines, and 0 parameter values can also be sent to ntfy by altering the -`1001.toml` file. We can get these alerts in real time without having to -dig through a noisy log file. - -Let's update our call to `generate_nmm_config` to have `nmm` send -notifications to the `NONMEMmonitor` topic on [ntfy.sh](ntfy.sh). - -```{r} -slurmtools::generate_nmm_config( - mod, - alert = "Ntfy", - topic = "NONMEMmonitor", - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") -``` - -This updates the `1001.toml` file to this: - -``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir = '/tmp' -watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' -output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' -poll_duration = 1 -alert = 'Ntfy' -level = 'Debug' -email = '' -threads = 1 -topic = 'NONMEMmonitor' -``` - -When we re-run the `submit_nonmem_model` call we will now get ntfy -notifications. One thing to note is that `nmm` will print full paths in -the log, but will only send notifications with the `model_number` (or -`model_number.file_extension`). - -```{r} -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm")) -) - -submission_nmm -``` - -```{r} -slurmtools::get_slurm_jobs(user = "matthews") -``` - -This gives us the notifications in a much more digestible format - -![nmm ntfy.sh alerts](data/images/nmm_ntfy_alerts.png) - -```{r, include = FALSE} -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` diff --git a/inst/doc/custom-alerts.html b/inst/doc/custom-alerts.html deleted file mode 100644 index 2916582..0000000 --- a/inst/doc/custom-alerts.html +++ /dev/null @@ -1,590 +0,0 @@ - - - - - - - - - - - - - - -custom-alerts - - - - - - - - - - - - - - - - - - - - - - - - - - -

custom-alerts

- - - -
library(slurmtools)
-#> ── Needed slurmtools options ───────────────────────────────────────────────────
-#> ✖ option('slurmtools.slurm_job_template_path') is not set.
-#> ✖ option('slurmtools.submission_root') is not set.
-#> ✖ option('slurmtools.bbi_config_path') is not set.
-#> ℹ Please set all options for job submission defaults to work.
-library(bbr)
-library(here)
-#> here() starts at /cluster-data/user-homes/matthews/Packages/slurmtools
-
-nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
-options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
-
-

Submitting a NONMEM job with nmm

-

Instead of using bbi we can use nmm (NONMEM Monitor) which -currently has some additional functionality of sending notifications -about zero gradients, missing -1E9 lines in ext file, and some very -basic control stream errors. Currently, only slack or ntfy.sh is supported for receiving notifications. To -use nmm you can install the latest release from the github -repository linked above.

-

We can update the template file accordingly:

-
#!/bin/bash 
-#SBATCH --job-name="{{job_name}}" 
-#SBATCH --nodes=1 
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task={{ncpu}} 
-#SBATCH --partition={{partition}}
-
-{{nmm_exe_path}} -c {{config_toml_path}} run
-

default, submit_nonmem_model will provide -nmm_exe_path and config_toml_path to the -template. Just like with bbi_exe_path, -nmm_exe_path is determined with -Sys.which("nmm") which may or may not give you the path to -the nmm binary if it is on your path or not. We can inject the -nmm_exe_path like we did with bbi_exe_path and -assume it’s not on our path.

-

The config.toml file controls what nmm will -monitor and where to look for files and how to alert you. We’ll use -generate_nmm_config() to create this file. First we can -look at the documentation to see what type of information we should pass -to this function.documentation for generate_nmm_config()

-
mod_number <- "1001"
-
-if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
-  mod <- bbr::read_model(file.path(nonmem, mod_number))
-} else {
-  mod <- bbr::new_model(file.path(nonmem, mod_number))
-}
-
slurmtools::generate_nmm_config(mod)
-

This generates the following toml file. Notice that alert is set to -‘None’, and both email and topic are empty. Since we’re in vignettes -we’ll need to update the watched_dir and -output_dir.

-
model_number = '1001'
-files_to_track = [ 'lst', 'ext', 'grd' ]
-tmp_dir = '/tmp'
-watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem'
-output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress'
-poll_duration = 1
-alert = 'None'
-level = 'Debug'
-email = ''
-threads = 1
-topic = ''
-
slurmtools::generate_nmm_config( 
-  mod, 
-  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
-  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
-

This updates the 1001.toml config file to:

-
model_number = '1001'
-files_to_track = [ 'lst', 'ext', 'grd' ]
-tmp_dir = '/tmp'
-watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
-output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
-poll_duration = 1
-alert = 'None'
-level = 'Debug'
-email = ''
-threads = 1
-topic = ''
-

We can now run submit_nonmem_model and get essentially -the same behavior as running with bbi. On linux -~/.local/bin/ will be on your path so saving binaries there -is a good approach.

-
submission_nmm <- slurmtools::submit_nonmem_model( 
-  mod, 
-  overwrite = TRUE,
-  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
-  slurm_template_opts = list(
-    nmm_exe_path = normalizePath("~/.local/bin/nmm"))
-)
-
-submission_nmm
-#> $status
-#> [1] 0
-#> 
-#> $stdout
-#> [1] "Submitted batch job 876\n"
-#> 
-#> $stderr
-#> [1] ""
-#> 
-#> $timeout
-#> [1] FALSE
-
slurmtools::get_slurm_jobs()
-#> # A tibble: 9 × 10
-#>   job_id job_state  cpus partition  standard_input standard_output              
-#>    <int> <chr>     <int> <chr>      <chr>          <chr>                        
-#> 1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
-#> 2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
-#> 3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
-#> 4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
-#> 5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
-#> 6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
-#> 7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
-#> 8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d0…
-#> 9    876 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/mat…
-#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
-#> #   current_working_directory <chr>
-

The one difference between using nmm compared to -bbi is that a new directory is created that contains a log -file that caught some issues with our run. This file is updated as -nonmem is running and monitors gradient values, parameters that hit -zero, as well as other errors from bbi. Looking at the first few lines -we can see that bbi was successfully able to call nonmem. -We also see an info level log that OMEGA(2,1) has 0 value – in our mod -file we don’t specify any omega values off the diagonal so these are -fixed at 0. Finally we see that GRD(6) hit 0 relatively early in the -run.

-
20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Successfully loaded default configuration from /cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/bbi.yaml"
-20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Beginning Local Path"
-20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="A total of 1 models have completed the initial preparation phase"
-20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="[1001] Beginning local work phase"
-20:12:58 [INFO] OMEGA(2,1) has 0 value
-20:12:58 [INFO] SIGMA(2,1) has 0 value
-20:13:00 [INFO] SIGMA(2,1) has 0 value
-20:13:00 [INFO] OMEGA(2,1) has 0 value
-20:13:04 [INFO] SIGMA(2,1) has 0 value
-20:13:04 [INFO] OMEGA(2,1) has 0 value
-20:13:04 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd" has 0 gradient for parameter: GRD(6)
-

After a run has finished several messages are sent to the log after a -final check of the files listed in the files_to_track field -of the 1001.toml file.

-
20:13:16 [INFO] Received Exit code: exit status: 0
-20:13:16 [WARN] 1001.ext: Missing ext final output lines. Observed lines were: [-1000000000.0, -1000000004.0, -1000000006.0, -1000000007.0]
-20:13:16 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd": The following parameters hit zero gradient through the run: ["GRD(6)"]
-

We see that GRD(6) hit zero during the run and that only a subset of -the -1E9 lines were present in the .ext file.

-
-
-

Getting alerted during a run

-

Like we did with bbi and altering the slurm template -file to get notifications from ntfy.sh -nmm has this feature built in! The messages in the log file -that relate to zero gradients, missing -1E9 lines, and 0 parameter -values can also be sent to ntfy by altering the 1001.toml -file. We can get these alerts in real time without having to dig through -a noisy log file.

-

Let’s update our call to generate_nmm_config to have -nmm send notifications to the NONMEMmonitor -topic on ntfy.sh.

-
slurmtools::generate_nmm_config( 
-  mod, 
-  alert = "Ntfy",
-  topic = "NONMEMmonitor",
-  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
-  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
-

This updates the 1001.toml file to this:

-
model_number = '1001'
-files_to_track = [ 'lst', 'ext', 'grd' ]
-tmp_dir = '/tmp'
-watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
-output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
-poll_duration = 1
-alert = 'Ntfy'
-level = 'Debug'
-email = ''
-threads = 1
-topic = 'NONMEMmonitor'
-

When we re-run the submit_nonmem_model call we will now -get ntfy notifications. One thing to note is that nmm will -print full paths in the log, but will only send notifications with the -model_number (or -model_number.file_extension).

-
submission_nmm <- slurmtools::submit_nonmem_model( 
-  mod, 
-  overwrite = TRUE,
-  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
-  slurm_template_opts = list(
-    nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm"))
-)
-#> Warning in normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm"):
-#> path[1]="/cluster-data/user-homes/matthews/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm":
-#> No such file or directory
-
-submission_nmm
-#> $status
-#> [1] 0
-#> 
-#> $stdout
-#> [1] "Submitted batch job 877\n"
-#> 
-#> $stderr
-#> [1] ""
-#> 
-#> $timeout
-#> [1] FALSE
-
slurmtools::get_slurm_jobs(user = "matthews")
-#> # A tibble: 10 × 10
-#>    job_id job_state  cpus partition  standard_input standard_output             
-#>     <int> <chr>     <int> <chr>      <chr>          <chr>                       
-#>  1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  9    876 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> 10    877 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
-#> #   current_working_directory <chr>
-

This gives us the notifications in a much more digestible format

-
-nmm ntfy.sh alerts -
nmm ntfy.sh alerts
-
-
- - - - - - - - - - - diff --git a/inst/doc/slack-alerts.R b/inst/doc/slack-alerts.R deleted file mode 100644 index 1e97821..0000000 --- a/inst/doc/slack-alerts.R +++ /dev/null @@ -1,78 +0,0 @@ -## ----include = FALSE---------------------------------------------------------- -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - -## ----include = FALSE---------------------------------------------------------- -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) - -## ----setup-------------------------------------------------------------------- -library(slurmtools) -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) - -## ----------------------------------------------------------------------------- -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} - -## ----------------------------------------------------------------------------- -Sys.which("slack_notifier") - -## ----------------------------------------------------------------------------- -slurmtools::generate_nmm_config( - mod, - alert = "slack", - email = "matthews@a2-ai.com", - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") - -## ----------------------------------------------------------------------------- -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm") - ) -) - -submission_nmm - -## ----------------------------------------------------------------------------- -slurmtools::get_slurm_jobs() - -## ----include = FALSE---------------------------------------------------------- -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) - diff --git a/inst/doc/slack-alerts.Rmd b/inst/doc/slack-alerts.Rmd deleted file mode 100644 index 15731f8..0000000 --- a/inst/doc/slack-alerts.Rmd +++ /dev/null @@ -1,145 +0,0 @@ ---- -title: "slack-alerts" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{slack-alerts} - %\VignetteEncoding{UTF-8} - %\VignetteEngine{knitr::rmarkdown} -editor_options: - markdown: - wrap: 72 ---- - -```{r, include = FALSE} -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(slurmtools) -library(bbr) -library(here) - -nonmem = file.path(here::here(), "vignettes", "model", "nonmem") -options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) -``` - -```{r} -mod_number <- "1001" - -if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { - mod <- bbr::read_model(file.path(nonmem, mod_number)) -} else { - mod <- bbr::new_model(file.path(nonmem, mod_number)) -} -``` - -# Cut me some Slack - -There is also functionality to pair `nmm` with -[slack_notifier](https://github.com/A2-ai/slack_notifier/releases/) and -get messages sent directly to you via a slack bot. This requires you to -download the slack_notifier binaries and added them to your path so -`nmm` can find it. You can download the latest release and extract the -binary and again save it to `~/.local/bin`. - -```{r} -Sys.which("slack_notifier") -``` - -slack_notifier requires an additional `slack_notifier/config.yaml` file -that contains the slack bot OAuth token which is found from -[[[https://api.slack.com/apps/\\\\](https://api.slack.com/apps/){.uri}](%5Bhttps://api.slack.com/apps/%5D(https://api.slack.com/apps/)%7B.uri%7D){.uri}\ /oauth?]. - -``` slack_notifier/config.yaml -SLACK_OAUTH_TOKEN: "encrypted(Bot User OAuth Token)" -``` - -Again, we need to update the `1001.toml` file to get slack -notifications. We need to set `alert = "slack"` and provide the `email` -associated with the slack account in `generate_nmm_config`. - -```{r} -slurmtools::generate_nmm_config( - mod, - alert = "slack", - email = "matthews@a2-ai.com", - watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", - output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") -``` - -This generates the following toml file: - -``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir = '/tmp' -watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' -output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' -poll_duration = 1 -alert = 'Slack' -level = 'Debug' -email = 'matthews@a2-ai.com' -threads = 1 -topic = '' -``` - -With `alert = 'Slack'` and `email` set in the `1001.toml` file `nmm` -will send slack notifications directly to you when a NONMEM run starts -and it will reply to that message with notifications if any gradients -hit 0 and when the run finishes it checks if all -1E9 lines are present -in the .ext file and gives another message about any parameters that hit -0 gradient. - -```{r} -submission_nmm <- slurmtools::submit_nonmem_model( - mod, - overwrite = TRUE, - slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), - slurm_template_opts = list( - nmm_exe_path = normalizePath("~/.local/bin/nmm") - ) -) - -submission_nmm -``` - -```{r} -slurmtools::get_slurm_jobs() -``` - -![nmm slack alerts](data/images/nmm_slack_notifications.png) - -```{r, include = FALSE} -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) -``` diff --git a/inst/doc/slack-alerts.html b/inst/doc/slack-alerts.html deleted file mode 100644 index 8d96425..0000000 --- a/inst/doc/slack-alerts.html +++ /dev/null @@ -1,461 +0,0 @@ - - - - - - - - - - - - - - -slack-alerts - - - - - - - - - - - - - - - - - - - - - - - - - - -

slack-alerts

- - - -
library(slurmtools)
-library(bbr)
-library(here)
-
-nonmem = file.path(here::here(), "vignettes", "model", "nonmem")
-options('slurmtools.submission_root' = file.path(nonmem, "submission-log"))
-
mod_number <- "1001"
-
-if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) {
-  mod <- bbr::read_model(file.path(nonmem, mod_number))
-} else {
-  mod <- bbr::new_model(file.path(nonmem, mod_number))
-}
-
-

Cut me some Slack

-

There is also functionality to pair nmm with slack_notifier -and get messages sent directly to you via a slack bot. This requires you -to download the slack_notifier binaries and added them to your path so -nmm can find it. You can download the latest release and -extract the binary and again save it to ~/.local/bin.

-
Sys.which("slack_notifier")
-#>                                                slack_notifier 
-#> "/cluster-data/user-homes/matthews/.local/bin/slack_notifier"
-

slack_notifier requires an additional -slack_notifier/config.yaml file that contains the slack bot -OAuth token which is found from [[https://api.slack.com/apps/\\](https://api.slack.com/apps/){.uri}<YOUR -APP ID> /oauth?].

-
SLACK_OAUTH_TOKEN: "encrypted(Bot User OAuth Token)"
-

Again, we need to update the 1001.toml file to get slack -notifications. We need to set alert = "slack" and provide -the email associated with the slack account in -generate_nmm_config.

-
slurmtools::generate_nmm_config( 
-  mod, 
-  alert = "slack",
-  email = "matthews@a2-ai.com",
-  watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem",
-  output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress")
-

This generates the following toml file:

-
model_number = '1001'
-files_to_track = [ 'lst', 'ext', 'grd' ]
-tmp_dir = '/tmp'
-watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem'
-output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress'
-poll_duration = 1
-alert = 'Slack'
-level = 'Debug'
-email = 'matthews@a2-ai.com'
-threads = 1
-topic = ''
-

With alert = 'Slack' and email set in the -1001.toml file nmm will send slack -notifications directly to you when a NONMEM run starts and it will reply -to that message with notifications if any gradients hit 0 and when the -run finishes it checks if all -1E9 lines are present in the .ext file -and gives another message about any parameters that hit 0 gradient.

-
submission_nmm <- slurmtools::submit_nonmem_model( 
-  mod, 
-  overwrite = TRUE,
-  slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"),
-  slurm_template_opts = list(
-    nmm_exe_path = normalizePath("~/.local/bin/nmm")
-  )
-)
-
-submission_nmm
-#> $status
-#> [1] 0
-#> 
-#> $stdout
-#> [1] "Submitted batch job 880\n"
-#> 
-#> $stderr
-#> [1] ""
-#> 
-#> $timeout
-#> [1] FALSE
-
slurmtools::get_slurm_jobs()
-#> # A tibble: 13 × 10
-#>    job_id job_state  cpus partition  standard_input standard_output             
-#>     <int> <chr>     <int> <chr>      <chr>          <chr>                       
-#>  1    868 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  2    869 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  3    870 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#>  4    871 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  5    872 COMPLETED     1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  6    873 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  7    874 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  8    875 FAILED        1 cpu2mem4gb /dev/null      /tmp/RtmpIBC04D/Rbuild1af3d…
-#>  9    876 CANCELLED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> 10    877 FAILED        1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> 11    878 RUNNING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> 12    879 COMPLETED     1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> 13    880 PENDING       1 cpu2mem4gb /dev/null      /cluster-data/user-homes/ma…
-#> # ℹ 4 more variables: submit_time <dttm>, start_time <dttm>, user_name <chr>,
-#> #   current_working_directory <chr>
-
-nmm slack alerts -
nmm slack alerts
-
-
- - - - - - - - - - - From 4b2cff3b0999948d8f60b492c1d43acc8001b705 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:36:41 +0000 Subject: [PATCH 19/38] seeing if sinfo/sacct run after download adding env to r cmd check --- .github/workflows/RunChecks.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index befc751..102c359 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,6 +50,8 @@ jobs: chmod +x /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH + sinfo + sacct env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -74,3 +76,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true + env: ${{ GITHUB_PATH }} From 75bbc1c98199d8e6a471484625e5cc4d0205b71a Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:38:40 +0000 Subject: [PATCH 20/38] removed env for now --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 102c359..7167f72 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -76,4 +76,4 @@ jobs: - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true - env: ${{ GITHUB_PATH }} + From 15c06ebaec8ece79aef2677128bde3f3c82be32e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:40:32 +0000 Subject: [PATCH 21/38] sacct showed reasonable output --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 7167f72..c042604 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -51,7 +51,7 @@ jobs: echo "/usr/local/bin" >> $GITHUB_PATH sinfo - sacct + env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 38fb9cff7a9eeb169c3450b87a9fd8e95b239450 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:45:48 +0000 Subject: [PATCH 22/38] added sacct outputs to github_workspace --- .github/workflows/RunChecks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index c042604..7b65dcc 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -60,6 +60,7 @@ jobs: cd /tmp gh repo clone a2-ai-tech-training/slurm-mocks mv /tmp/slurm-mocks/sinfo/outputs $GITHUB_WORKSPACE/ + mv /tmp/slurm-mocks/sacct/outputs $GITHUB_WORKSPACE/ rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6c2edf535b3339a2fb4d0d133c9b3c16f888e9ec Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:48:55 +0000 Subject: [PATCH 23/38] sacct is still running in gh actions even though only sinfo was called? I think outputs directories need to be rename? --- .github/workflows/RunChecks.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 7b65dcc..4502ec4 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,7 +50,7 @@ jobs: chmod +x /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH - sinfo + sacct env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -60,7 +60,6 @@ jobs: cd /tmp gh repo clone a2-ai-tech-training/slurm-mocks mv /tmp/slurm-mocks/sinfo/outputs $GITHUB_WORKSPACE/ - mv /tmp/slurm-mocks/sacct/outputs $GITHUB_WORKSPACE/ rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c650f6063340cee4067f1fe19df4e0afd5c3483d Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:53:21 +0000 Subject: [PATCH 24/38] sacct and sinfo were both printed due to echo to GITHUB_PATH. Trying to call sinfo with processx --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 4502ec4..fcb8e48 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,7 +50,7 @@ jobs: chmod +x /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH - sacct + Rscript -e 'processx::run("sinfo")' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4c0473217e013477af10dc38491cf8762eed03ec Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sat, 7 Sep 2024 23:57:42 +0000 Subject: [PATCH 25/38] didn't know processx wasn't included --- .github/workflows/RunChecks.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index fcb8e48..735e1b9 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,7 +50,8 @@ jobs: chmod +x /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH - Rscript -e 'processx::run("sinfo")' + Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo")' + env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 58dd857635b327b08434d673cd99be06064e591e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 13:09:51 +0000 Subject: [PATCH 26/38] removed outputs directory moves beacuse sinfo seemed to work without it? if both sinfo and sacct work without the move going to add --json args --- .github/workflows/RunChecks.yaml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 735e1b9..39fe477 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -51,17 +51,8 @@ jobs: echo "/usr/local/bin" >> $GITHUB_PATH Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo")' + Rscript -e 'processx::run("sacct")' - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Grab outputs directory from mocked repo - run: | - cd /tmp - gh repo clone a2-ai-tech-training/slurm-mocks - mv /tmp/slurm-mocks/sinfo/outputs $GITHUB_WORKSPACE/ - rm -rf /tmp/slurm-mocks env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c8cd3350c45bdc9875a1aa1411d4f037fde31764 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 13:15:28 +0000 Subject: [PATCH 27/38] sacct is showing no output, but sinfo is. --- .github/workflows/RunChecks.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 39fe477..c28eb8b 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,9 +50,10 @@ jobs: chmod +x /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH - Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo")' - Rscript -e 'processx::run("sacct")' - + Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo", args = "--json")' + Rscript -e 'Sys.which("sinfo")' + Rscript -e 'processx::run("sacct", args = "--json")' + Rscript -e 'Sys.which("sacct")' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 13117be6f19e02bb94570ad78ec36160effcdeb8 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 13:24:27 +0000 Subject: [PATCH 28/38] args = --json works for both. moving the call to sinfo and sacct to different step --- .github/workflows/RunChecks.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index c28eb8b..d34810a 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -48,14 +48,16 @@ jobs: tar -xvf /usr/local/bin/sacct_Linux_x86_64.tar.gz mv ./sacct /usr/local/bin/sacct chmod +x /usr/local/bin/sacct - echo "/usr/local/bin" >> $GITHUB_PATH + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check mocked binaries work in different step + run: | Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo", args = "--json")' Rscript -e 'Sys.which("sinfo")' Rscript -e 'processx::run("sacct", args = "--json")' Rscript -e 'Sys.which("sacct")' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: r-lib/actions/setup-r-dependencies@v2 with: From ffa8c34a337535718e5bc06528e88a3d351591a0 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 13:38:17 +0000 Subject: [PATCH 29/38] added sys.which("sinfo")/squeue --- R/get-jobs.R | 3 ++- R/submit-model.R | 2 +- R/zzz.R | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/R/get-jobs.R b/R/get-jobs.R index 8575083..35d5857 100644 --- a/R/get-jobs.R +++ b/R/get-jobs.R @@ -45,7 +45,8 @@ parse_jobs_json <- function(.json) { #' @param user user filter #' @export get_slurm_jobs <- function(user = NULL){ - cmd <- list(cmd = "squeue", args = "--json") + + cmd <- list(cmd = Sys.which("squeue"), args = "--json") res <- processx::run(cmd$cmd, args = cmd$args) if (res$status != 0) { # todo: better handle returning why diff --git a/R/submit-model.R b/R/submit-model.R index ccfdcd0..596aad7 100644 --- a/R/submit-model.R +++ b/R/submit-model.R @@ -118,7 +118,7 @@ submit_nonmem_model <- brio::write_file(template_script, script_file_path) fs::file_chmod(script_file_path, "0755") } - cmd <- list(cmd = "sbatch", args = script_file_path, template_script = template_script, partition = partition) + cmd <- list(cmd = Sys.which("sbatch"), args = script_file_path, template_script = template_script, partition = partition) if (dry_run) { return(cmd) } diff --git a/R/zzz.R b/R/zzz.R index 4cb2d32..701c8aa 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -5,7 +5,7 @@ .onLoad <- function(libname, pkgname) { result <- tryCatch( - {processx::run("squeue", args = "--version")}, + {processx::run(Sys.which("squeue"), args = "--version")}, error = function(e) { warning("'squeue' command not found or failed to run.") return(NULL) From f159377d5ec53cee0542c70fa47c6d8a8f7dd838 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 13:40:53 +0000 Subject: [PATCH 30/38] changed chmod to 755 --- .github/workflows/RunChecks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index d34810a..2174908 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -42,12 +42,12 @@ jobs: gh release download "sinfo/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ tar -xvf /usr/local/bin/sinfo_Linux_x86_64.tar.gz mv ./sinfo /usr/local/bin/sinfo - chmod +x /usr/local/bin/sinfo + chmod 755 /usr/local/bin/sinfo gh release download "sacct/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ tar -xvf /usr/local/bin/sacct_Linux_x86_64.tar.gz mv ./sacct /usr/local/bin/sacct - chmod +x /usr/local/bin/sacct + chmod 755 /usr/local/bin/sacct echo "/usr/local/bin" >> $GITHUB_PATH env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e3823f906a8a8f67c666a4470526605f29b73621 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 16:26:14 +0000 Subject: [PATCH 31/38] updated sinfo mock call correctly and built new sinfo version --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 2174908..29169e8 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -54,7 +54,7 @@ jobs: - name: Check mocked binaries work in different step run: | - Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo", args = "--json")' + Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo", args = c("--format", "%P,%c,%m"),)' Rscript -e 'Sys.which("sinfo")' Rscript -e 'processx::run("sacct", args = "--json")' Rscript -e 'Sys.which("sacct")' From abe1f351c00a2af78a454944e5e9b7f2c119b1cd Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 16:29:46 +0000 Subject: [PATCH 32/38] updated sinfo version in gh actions. --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 29169e8..ced3976 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -39,7 +39,7 @@ jobs: run: | mkdir -p /usr/local/bin cd /tmp - gh release download "sinfo/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ + gh release download "sinfo/v0.2.1" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ tar -xvf /usr/local/bin/sinfo_Linux_x86_64.tar.gz mv ./sinfo /usr/local/bin/sinfo chmod 755 /usr/local/bin/sinfo From 047cc4eb9f0d4fdf24cf076ceff29dc5e7b51c83 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 17:16:50 +0000 Subject: [PATCH 33/38] added squeue to slurm mocks --- .github/workflows/RunChecks.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index ced3976..d1a53fb 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -48,6 +48,12 @@ jobs: tar -xvf /usr/local/bin/sacct_Linux_x86_64.tar.gz mv ./sacct /usr/local/bin/sacct chmod 755 /usr/local/bin/sacct + + gh release download "squeue/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ + tar -xvf /usr/local/bin/squeue_Linxu_x86_64.tar.gz + mv ./squeue /usr/local/bin/squeue + chmod 755 /usr/local/bin/squeue + echo "/usr/local/bin" >> $GITHUB_PATH env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 08d729316e595ca23f0ea3e34ec988db500d9a3e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 17:18:42 +0000 Subject: [PATCH 34/38] fixed typo --- .github/workflows/RunChecks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index d1a53fb..05d6cc0 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -50,7 +50,7 @@ jobs: chmod 755 /usr/local/bin/sacct gh release download "squeue/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ - tar -xvf /usr/local/bin/squeue_Linxu_x86_64.tar.gz + tar -xvf /usr/local/bin/squeue_Linux_x86_64.tar.gz mv ./squeue /usr/local/bin/squeue chmod 755 /usr/local/bin/squeue From e4fb1d133954fa0c901b48233584266df6a303cb Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 17:26:53 +0000 Subject: [PATCH 35/38] changed Sys.which("sbatch") to sbatch... Testing that this is currently causing the error. --- R/submit-model.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/submit-model.R b/R/submit-model.R index 596aad7..ccfdcd0 100644 --- a/R/submit-model.R +++ b/R/submit-model.R @@ -118,7 +118,7 @@ submit_nonmem_model <- brio::write_file(template_script, script_file_path) fs::file_chmod(script_file_path, "0755") } - cmd <- list(cmd = Sys.which("sbatch"), args = script_file_path, template_script = template_script, partition = partition) + cmd <- list(cmd = "sbatch", args = script_file_path, template_script = template_script, partition = partition) if (dry_run) { return(cmd) } From e8eeca3dd8d11656e4c891a99550c7c86095d7c8 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 18:14:53 +0000 Subject: [PATCH 36/38] added sbatch slurm-mock --- .github/workflows/RunChecks.yaml | 5 +++++ R/submit-model.R | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 05d6cc0..04da01a 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -54,6 +54,11 @@ jobs: mv ./squeue /usr/local/bin/squeue chmod 755 /usr/local/bin/squeue + gh release download "sbatch/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/" + tar -xvf /usr/local/bin/sbatch_Linux_x86_64.tar.gz + mv ./sbatch /usr/local/bin/sbatch + chmod 755 /usr/local/bin/sbatch + echo "/usr/local/bin" >> $GITHUB_PATH env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/R/submit-model.R b/R/submit-model.R index ccfdcd0..80f75d7 100644 --- a/R/submit-model.R +++ b/R/submit-model.R @@ -109,6 +109,7 @@ submit_nonmem_model <- template_list ) }) + script_file_path <- file.path(submission_root, sprintf("%s.sh", basename(.mod$absolute_model_path))) if (!dry_run) { @@ -118,7 +119,7 @@ submit_nonmem_model <- brio::write_file(template_script, script_file_path) fs::file_chmod(script_file_path, "0755") } - cmd <- list(cmd = "sbatch", args = script_file_path, template_script = template_script, partition = partition) + cmd <- list(cmd = Sys.which("sbatch"), args = script_file_path, template_script = template_script, partition = partition) if (dry_run) { return(cmd) } From 458ca1381a04bef639413e6a657b9c75e5b60953 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 18:28:05 +0000 Subject: [PATCH 37/38] fixed typo --- .github/workflows/RunChecks.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/RunChecks.yaml b/.github/workflows/RunChecks.yaml index 04da01a..2ab2d41 100644 --- a/.github/workflows/RunChecks.yaml +++ b/.github/workflows/RunChecks.yaml @@ -54,7 +54,7 @@ jobs: mv ./squeue /usr/local/bin/squeue chmod 755 /usr/local/bin/squeue - gh release download "sbatch/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/" + gh release download "sbatch/v0.2.0" -R a2-ai-tech-training/slurm-mocks -p "*Linux_x86_64.tar.gz" -D /usr/local/bin/ tar -xvf /usr/local/bin/sbatch_Linux_x86_64.tar.gz mv ./sbatch /usr/local/bin/sbatch chmod 755 /usr/local/bin/sbatch @@ -66,9 +66,10 @@ jobs: - name: Check mocked binaries work in different step run: | Rscript -e 'if (!requireNamespace("processx", quietly = TRUE)) install.packages("processx"); processx::run("sinfo", args = c("--format", "%P,%c,%m"),)' - Rscript -e 'Sys.which("sinfo")' Rscript -e 'processx::run("sacct", args = "--json")' - Rscript -e 'Sys.which("sacct")' + Rscript -e 'processx::run("squeue", args = "--version")' + Rscript -e 'processx::run("squeue", args = "--json")' + Rscript -e 'processx::run("sbatch", args = "path_to_file")' - uses: r-lib/actions/setup-r-dependencies@v2 with: From 11591341c9a344f2d0d3df5caba446803941dbb4 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 9 Sep 2024 18:42:49 +0000 Subject: [PATCH 38/38] updated get_slurm_jobs to prevent error filtering when no jobs are in squeue. --- R/get-jobs.R | 13 +++++++++++-- vignettes/Running-nonmem.Rmd | 32 ++++++++++++++++---------------- vignettes/custom-alerts.Rmd | 34 +++++++++++++++++----------------- vignettes/slack-alerts.Rmd | 34 +++++++++++++++++----------------- 4 files changed, 61 insertions(+), 52 deletions(-) diff --git a/R/get-jobs.R b/R/get-jobs.R index 35d5857..a511a60 100644 --- a/R/get-jobs.R +++ b/R/get-jobs.R @@ -56,7 +56,16 @@ get_slurm_jobs <- function(user = NULL){ res_df$submit_time <- as.POSIXct(res_df$submit_time, origin = "1970-01-01") res_df$start_time <- as.POSIXct(res_df$start_time, origin = "1970-01-01") if (!is.null(user)) { - return(dplyr::filter(res_df, user_name == user)) + df <- tryCatch( + { + dplyr::filter(res_df, user_name == user) + }, + error = function(e) { + res_df + } + ) + } else { + df <- res_df } - res_df + return(df) } diff --git a/vignettes/Running-nonmem.Rmd b/vignettes/Running-nonmem.Rmd index ce7a020..fd043ff 100644 --- a/vignettes/Running-nonmem.Rmd +++ b/vignettes/Running-nonmem.Rmd @@ -250,20 +250,20 @@ To reiterate, this template file is run as a bash shell script so anything you c ```{r, include = FALSE} #cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +# state <- slurmtools::get_slurm_jobs(user = "matthews") +# +# if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { +# for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { +# processx::run("scancel", args = paste0(job_id)) +# } +# } +# +# #removing generated files from running this vignette +# nonmem <- file.path("model", "nonmem") +# +# unlink(file.path(nonmem, "1001"), recursive = TRUE) +# unlink(file.path(nonmem, "1001.yaml")) +# unlink(file.path(nonmem, "1001.toml")) +# unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +# unlink(file.path(nonmem, "in_progress"), recursive = TRUE) ``` diff --git a/vignettes/custom-alerts.Rmd b/vignettes/custom-alerts.Rmd index 37d6dfe..a48be3c 100644 --- a/vignettes/custom-alerts.Rmd +++ b/vignettes/custom-alerts.Rmd @@ -247,21 +247,21 @@ This gives us the notifications in a much more digestible format ![nmm ntfy.sh alerts](data/images/nmm_ntfy_alerts.png) ```{r, include = FALSE} -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +# #cancelling any running nonmem jobs +# state <- slurmtools::get_slurm_jobs(user = "matthews") +# +# if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { +# for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { +# processx::run("scancel", args = paste0(job_id)) +# } +# } +# +# #removing generated files from running this vignette +# nonmem <- file.path("model", "nonmem") +# +# unlink(file.path(nonmem, "1001"), recursive = TRUE) +# unlink(file.path(nonmem, "1001.yaml")) +# unlink(file.path(nonmem, "1001.toml")) +# unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +# unlink(file.path(nonmem, "in_progress"), recursive = TRUE) ``` diff --git a/vignettes/slack-alerts.Rmd b/vignettes/slack-alerts.Rmd index 15731f8..3b62f92 100644 --- a/vignettes/slack-alerts.Rmd +++ b/vignettes/slack-alerts.Rmd @@ -125,21 +125,21 @@ slurmtools::get_slurm_jobs() ![nmm slack alerts](data/images/nmm_slack_notifications.png) ```{r, include = FALSE} -#cancelling any running nonmem jobs -state <- slurmtools::get_slurm_jobs(user = "matthews") - -if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { - for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { - processx::run("scancel", args = paste0(job_id)) - } -} - -#removing generated files from running this vignette -nonmem <- file.path("model", "nonmem") - -unlink(file.path(nonmem, "1001"), recursive = TRUE) -unlink(file.path(nonmem, "1001.yaml")) -unlink(file.path(nonmem, "1001.toml")) -unlink(file.path(nonmem, "submission-log"), recursive = TRUE) -unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +# #cancelling any running nonmem jobs +# state <- slurmtools::get_slurm_jobs(user = "matthews") +# +# if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { +# for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { +# processx::run("scancel", args = paste0(job_id)) +# } +# } +# +# #removing generated files from running this vignette +# nonmem <- file.path("model", "nonmem") +# +# unlink(file.path(nonmem, "1001"), recursive = TRUE) +# unlink(file.path(nonmem, "1001.yaml")) +# unlink(file.path(nonmem, "1001.toml")) +# unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +# unlink(file.path(nonmem, "in_progress"), recursive = TRUE) ```