improve unit check

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '4192781'
+ValidationKey: '4212864'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'piamValidation: Validation Tools for PIK-PIAM'
-version: 0.2.11
-date-released: '2024-05-28'
+version: 0.2.12
+date-released: '2024-05-29'
 abstract: The piamValidation package provides validation tools for the Potsdam Integrated
   Assessment Modelling environment.
 authors:

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: piamValidation
 Title: Validation Tools for PIK-PIAM
-Version: 0.2.11
-Date: 2024-05-28
+Version: 0.2.12
+Date: 2024-05-29
 Authors@R:
     c(person("Pascal", "Weigmann",, "[email protected]", role = c("aut", "cre")),
       person("Oliver", "Richters",, role = "aut"))

R/checkUnits.R

@@ -0,0 +1,38 @@
+#' Check variable consistency
+#'
+#' Test whether unit of on row of config and data for this variable match.
+#'
+#' @importFrom piamInterfaces areUnitsIdentical
+#' @importFrom dplyr filter
+#'
+#' @param data scenario or reference data for one variable
+#' @param cfgRow one row of a config file containing the same variable as the
+#'        data object
+#'
+checkUnits <- function(data, cfgRow) {
+  # empty unit field in config means skipping the check
+  if (is.na(cfgRow$unit)) {
+    message("No unit given in config for ", cfgRow$variable,
+            ", skipping consistency check.\n")
+  } else {
+    # identify data type
+    dataType <-
+      if(cfgRow$ref_scenario %in% "historical") "reference" else "scenario"
+
+    units <- as.character(unique(data$unit))
+    for (i in 1:length(units)) {
+      if (!piamInterfaces::areUnitsIdentical(cfgRow$unit, units[i])) {
+        warning(paste0(
+          "Non-matching units in config and ", dataType," data found.\n",
+          "variable: ", cfgRow$variable, "\n",
+          "config unit: ", cfgRow$unit, "\n",
+          dataType, " unit:   ", units[i]), "\n")
+        # in case of the presence of non-matching units:
+        # filter data for correct unit as it might also available, if not this
+        # will result in an empty data object being returned
+        data <- filter(data, data$unit %in% cfgRow$unit)
+      }
+    }
+  }
+  return(data)
+}

R/combineData.R

@@ -57,19 +57,7 @@ combineData <- function(data, cfgRow, histData = NULL) {
            critical = c$critical)
 
   # test whether units of config and scenario data match
-  scen_units <- as.character(unique(d$unit))
-  for (i in 1:length(scen_units)) {
-    if (!piamInterfaces::areUnitsIdentical(c$unit, scen_units[i])) {
-      warning(paste0(
-        "Non-matching units in config and scenario data found.\n",
-        "Variable: ", c$variable, "\n",
-        "Cfg unit: ", c$unit, "\n",
-        "Sce unit: ", scen_units[i]), "\n")
-      # in case of the presence of non-matching units:
-      # filter data for correct unit as it might also be available
-      d <- filter(d, d$unit %in% c$unit)
-    }
-  }
+  d <- checkUnits(d, c)
 
   # historic ####
   # depending on category: filter and attach reference values if they are needed
@@ -84,19 +72,7 @@ combineData <- function(data, cfgRow, histData = NULL) {
 
 
     # test whether units of config and reference data match
-    hist_units <- as.character(unique(h$unit))
-    for (i in 1:length(hist_units)) {
-      if (!piamInterfaces::areUnitsIdentical(c$unit, hist_units[i])) {
-        warning(paste0(
-        "Non-matching units in config and reference data found.\n",
-        "Variable: ", c$variable, "\n",
-        "Cfg unit: ", c$unit, "\n",
-        "Ref unit: ", hist_units[i]), "\n")
-        # in case of the presence of non-matching units:
-        # filter data for correct unit as it might also available
-        h <- filter(h, h$unit %in% c$unit)
-      }
-    }
+    h <- checkUnits(h, c)
 
     # test whether historical ref_model exists and has data to compare to
     if (nrow(h) == 0) {

R/importFunctions.R

@@ -46,6 +46,8 @@ getConfig <- function(configName) {
   # remove empty (all NA) rows
   cfg <- cfg[rowSums(is.na(cfg)) != ncol(cfg), ]
 
+  # remove rows without variables
+  cfg <- cfg[!is.na(cfg$variable), ]
 
   # convert "%" thresholds to decimals
   cfg <- cfg %>%

R/validateScenarios.R

@@ -23,9 +23,6 @@ validateScenarios <- function(dataPath, config, outputFile = NULL) {
     expandPeriods(scen) %>%
     expandVariables(scen)
 
-  # test if units of variables in config and data are consistent
-  # checkUnits(data, cfg)
-
   # filter data for variables from config
   hist <- filter(hist, variable %in% unique(cfg$variable))
   scen <- filter(scen, variable %in% unique(cfg$variable))
@@ -48,6 +45,10 @@ validateScenarios <- function(dataPath, config, outputFile = NULL) {
   # perform actual checks and write results in new columns of data.frame
   df <- evaluateThresholds(df)
 
+  if (nrow(df) == 0) {
+    stop("Something went wrong, returned data.frame is empty.")
+  }
+
   # export df to file in case outputFile is specified
   if (!is.null(outputFile)) {
     output_path <- paste0(path.package("piamValidation"), "/output")

README.md

@@ -1,6 +1,6 @@
 # Validation Tools for PIK-PIAM
 
-R package **piamValidation**, version **0.2.11**
+R package **piamValidation**, version **0.2.12**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/piamValidation)](https://cran.r-project.org/package=piamValidation)  [![R build status](https://github.com/pik-piam/piamValidation/workflows/check/badge.svg)](https://github.com/pik-piam/piamValidation/actions) [![codecov](https://codecov.io/gh/pik-piam/piamValidation/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/piamValidation) [![r-universe](https://pik-piam.r-universe.dev/badges/piamValidation)](https://pik-piam.r-universe.dev/builds)
 
@@ -46,7 +46,7 @@ In case of questions / problems please contact Pascal Weigmann <pascal.weigmann@
 
 To cite package **piamValidation** in publications use:
 
-Weigmann P, Richters O (2024). _piamValidation: Validation Tools for PIK-PIAM_. R package version 0.2.11, <https://github.com/pik-piam/piamValidation>.
+Weigmann P, Richters O (2024). _piamValidation: Validation Tools for PIK-PIAM_. R package version 0.2.12, <https://github.com/pik-piam/piamValidation>.
 
 A BibTeX entry for LaTeX users is
 
@@ -55,7 +55,7 @@ A BibTeX entry for LaTeX users is
   title = {piamValidation: Validation Tools for PIK-PIAM},
   author = {Pascal Weigmann and Oliver Richters},
   year = {2024},
-  note = {R package version 0.2.11},
+  note = {R package version 0.2.12},
   url = {https://github.com/pik-piam/piamValidation},
 }
 ```

vignettes/validateScenarios.Rmd

@@ -53,7 +53,6 @@ performed, different columns can or need to be filled.
 
 ![Rules for writing the config file.](./config_rules.jpg){width=100%}
 
-
 **General Rules**
 
 - historical reference data needs to read "historical" in the column "scenario"
@@ -70,6 +69,21 @@ It is recommended to choose a historical reference source explicitly in the
 historical sources will be averaged and the tooltip will show
 ``ref_model = "multiple"``.
 
+### Units
+
+It is recommended to include the unit of each variable in the config file to
+avoid inconsistency between data sources. The tools performs checks, whether the
+units in the config match those in scenario and reference data and returns a 
+warning in case they don't. This check is performed using 
+``piamInterfaces::areUnitsIdentical()`` to avoid false positives.
+
+In case scenario or reference data contains variables with multiple units, it
+will be filtered for the units matching those from the config file.
+
+If the ``unit`` column is left empty, no consistency check will be performed.
+This is the recommended approach when selecting multiple variables in one go via
+``**``, which don't all share the same unit.
+
 
 ### Use Case 1: relative comparison to historical data
 | metric   | critical | variable     | model | scenario | region | period | min_red | min_yel | max_yel | max_red | ref_model | ref_scenario | ref_period |