From 446018790b14bf9f4e85df2addeb476452b5d3c8 Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 11 Mar 2024 14:29:01 +0100 Subject: [PATCH 1/3] Remove iamCheck --- R/modeltests.R | 26 ++------------------------ vignettes/testingSuite.Rmd | 12 +----------- 2 files changed, 3 insertions(+), 35 deletions(-) diff --git a/R/modeltests.R b/R/modeltests.R index fb01fb0..da42be5 100644 --- a/R/modeltests.R +++ b/R/modeltests.R @@ -12,7 +12,6 @@ #' @param model Model name #' @param test Use this option to run a test of the workflow (no runs will be submitted) #' The test parameter needs to be of the form "YY-MM-DD" -#' @param iamccheck Use this option to turn iamc-style checks on and off #' @param email whether an email notification will be send or not #' @param mattermostToken token used for mattermost notifications #' @param compScen whether compScen has to run or not @@ -22,7 +21,6 @@ #' @seealso \code{\link{package2readme}} #' @importFrom utils read.csv2 #' @importFrom dplyr filter %>% -#' @importFrom piamModelTests iamCheck #' @importFrom quitte read.quitte #' @importFrom lucode2 sendmail #' @importFrom remind2 compareScenarios2 @@ -35,7 +33,6 @@ modeltests <- function( model = NULL, user = NULL, test = NULL, - iamccheck = TRUE, email = TRUE, compScen = TRUE, mattermostToken = NULL, @@ -57,7 +54,7 @@ modeltests <- function( message("Found 'end' in ", normalizePath("../.testsstatus"), "\nCalling 'evaluateRuns'") withr::with_dir("output", { evaluateRuns(model = model, mydir = mydir, gitPath = gitPath, compScen = compScen, email = email, - mattermostToken = mattermostToken, gitdir = gitdir, iamccheck = iamccheck, user = user) + mattermostToken = mattermostToken, gitdir = gitdir, user = user) }) # make sure next call will start runs message("Writing 'start' to ", normalizePath("../.testsstatus")) @@ -178,7 +175,7 @@ startRuns <- function(test, model, mydir, gitPath, user) { message("Function 'startRuns' finished.") } -evaluateRuns <- function(model, mydir, gitPath, compScen, email, mattermostToken, gitdir, iamccheck, user, test = NULL) { +evaluateRuns <- function(model, mydir, gitPath, compScen, email, mattermostToken, gitdir, user, test = NULL) { message("Current working directory ", normalizePath(".")) if (is.null(test)) test <- readRDS(paste0(mydir, "/test.rds")) if (!test) { @@ -391,25 +388,6 @@ evaluateRuns <- function(model, mydir, gitPath, compScen, email, mattermostToken } } - if (iamccheck) { - a <- NULL - if (length(runsStarted) > 0) { - mifs <- paste0(runsStarted, "/REMIND_generic_", sub("_20[0-9][0-9].*.$", "", runsStarted), ".mif") - mifs <- mifs[file.exists(mifs)] - try(a <- read.quitte(mifs)) - if (!is.null(a)) { - out[["iamCheck"]] <- iamCheck(a, cfg = model) - if (!test) { - saveRDS(out[["iamCheck"]], file = paste0("iamccheck-", commitTested, ".rds")) - } else { - saveRDS(out[["iamCheck"]], file = paste0("iamccheck-", test, ".rds")) - } - } - } - write(paste0("The IAMC check of these runs can be found in /p/projects/remind/modeltests/remind/output/iamccheck-", - commitTested, ".rds", "\n"), myfile, append = TRUE) - } - if(length(runsStarted) < 1) errorList <- c(errorList, "No runs started") if(is.null(errorList)) { diff --git a/vignettes/testingSuite.Rmd b/vignettes/testingSuite.Rmd index ded5188..9a6da1b 100644 --- a/vignettes/testingSuite.Rmd +++ b/vignettes/testingSuite.Rmd @@ -25,15 +25,7 @@ It is fully automated (runs every weekend), documents its findings in a gitlab p ## How it works: -A scheduled and recurring job runs on the cluster every weekend, starting model runs according to a predefined set of scenarios. Once the runs are finished the job creates an overview of the results using [rs2](http://htmlpreview.github.io/?https://github.com/pik-piam/modelstats/blob/master/vignettes/rs2.html), and (optionally) additionally runs [compareScenarios](https://github.com/pik-piam/remind2/blob/master/R/compareScenarios.R) comparing each run with the previous run with the same scenario name (the comparison PDF files are found in each scenario folder, they also contain comparisons with historical data). It also uploads all runs to the [shinyResults::appResults](https://github.com/pik-piam/shinyresults/blob/master/R/appResults.R) app for easy and interactive viewing. Also here a comparison with historical data is found. Further, the results of all runs are gathered, passed through an [IAMC-style](https://github.com/IAMconsortium/iamc) check for consistency, and archived. These checks include: - - * non-standard variable names - * missing variables - * min and max values - * summation groups (using the |+| notation) - * units - -Finally, using a gitlab integration hook an overview of the results is automatically reported to the developer team via email. +A scheduled and recurring job runs on the cluster every weekend, starting model runs according to a predefined set of scenarios. Once the runs are finished the job creates an overview of the results using [rs2](http://htmlpreview.github.io/?https://github.com/pik-piam/modelstats/blob/master/vignettes/rs2.html), and (optionally) additionally runs [compareScenarios](https://github.com/pik-piam/remind2/blob/master/R/compareScenarios.R) comparing each run with the previous run with the same scenario name (the comparison PDF files are found in each scenario folder, they also contain comparisons with historical data). It also uploads all runs to the [shinyResults::appResults](https://github.com/pik-piam/shinyresults/blob/master/R/appResults.R) app for easy and interactive viewing. Also here a comparison with historical data is found. Finally, using a gitlab integration hook an overview of the results is automatically reported to the developer team via email. ## How to use/contribute: @@ -41,6 +33,4 @@ If the report contains runs that failed, or output that looks wrong (or has not If you want your own scenarios to be included in future tests: For REMIND, add your scenarios to the `scenario_config.csv`, add `AMT` in the `start` column and simply commit it to the model's develop branch (similar for MAgPIE). They will be automatically executed the next time the tests are run. -Tailor the IAMC-style checks to your needs or add further IAMC-style checks here (read the vignette to see how this is done): http://htmlpreview.github.io/?https://github.com/pik-piam/piamModelTests/blob/master/vignettes/iamc.html - For wishes, changes etc, please contact RSE or open a new issue here: https://github.com/pik-piam/modelstats/issues From bcbf3f73d5e19143a2de363ac286fbc6bf62619b Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 11 Mar 2024 14:48:20 +0100 Subject: [PATCH 2/3] Remove iamCheck --- DESCRIPTION | 3 +- man/modeltests.Rd | 3 - vignettes/testingSuite.html | 362 ++++++++++++++++++++++++++++-------- 3 files changed, 286 insertions(+), 82 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5f69006..bb6a8f3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: modelstats Type: Package Title: Run Analysis Tools Version: 0.19.0 -Date: 2024-03-08 +Date: 2024-03-11 Authors@R: c( person("Anastasis", "Giannousakis", email = "giannou@pik-potsdam.de", role = c("aut","cre")), person("Oliver", "Richters", role = "aut") @@ -14,7 +14,6 @@ Imports: crayon, gdx, gtools, - piamModelTests, lubridate, lucode2, utils, diff --git a/man/modeltests.Rd b/man/modeltests.Rd index 6f8709d..a361b86 100644 --- a/man/modeltests.Rd +++ b/man/modeltests.Rd @@ -10,7 +10,6 @@ modeltests( model = NULL, user = NULL, test = NULL, - iamccheck = TRUE, email = TRUE, compScen = TRUE, mattermostToken = NULL, @@ -29,8 +28,6 @@ modeltests( \item{test}{Use this option to run a test of the workflow (no runs will be submitted) The test parameter needs to be of the form "YY-MM-DD"} -\item{iamccheck}{Use this option to turn iamc-style checks on and off} - \item{email}{whether an email notification will be send or not} \item{compScen}{whether compScen has to run or not} diff --git a/vignettes/testingSuite.html b/vignettes/testingSuite.html index a285a33..cd9cc75 100644 --- a/vignettes/testingSuite.html +++ b/vignettes/testingSuite.html @@ -14,34 +14,228 @@ Testing Suite - - + + - - - - + + + + - - +h1.title {font-size: 38px;} +h2 {font-size: 30px;} +h3 {font-size: 24px;} +h4 {font-size: 18px;} +h5 {font-size: 16px;} +h6 {font-size: 12px;} +code {color: inherit; background-color: rgba(0, 0, 0, 0.04);} +pre:not([class]) { background-color: white } + + +code{white-space: pre-wrap;} +span.smallcaps{font-variant: small-caps;} +span.underline{text-decoration: underline;} +div.column{display: inline-block; vertical-align: top; width: 50%;} +div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} +ul.task-list{list-style: none;} +