Run unit tests on all available DB Servers (#153)

OHDSI · Jun 5, 2024 · d48abb1 · d48abb1
1 parent e3efad6
commit d48abb1
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 39 deletions.
diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml
@@ -50,9 +50,23 @@ jobs:
       CDM5_REDSHIFT_PASSWORD: ${{ secrets.CDM5_REDSHIFT_PASSWORD }}
       CDM5_REDSHIFT_SERVER: ${{ secrets.CDM5_REDSHIFT_SERVER }}
       CDM5_REDSHIFT_USER: ${{ secrets.CDM5_REDSHIFT_USER }}
+      CDM_SNOWFLAKE_CDM53_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_CDM53_SCHEMA }}
+      CDM_SNOWFLAKE_OHDSI_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_OHDSI_SCHEMA }}
+      CDM_SNOWFLAKE_PASSWORD: ${{ secrets.CDM_SNOWFLAKE_PASSWORD }}
+      CDM_SNOWFLAKE_CONNECTION_STRING: ${{ secrets.CDM_SNOWFLAKE_CONNECTION_STRING }}
+      CDM_SNOWFLAKE_USER: ${{ secrets.CDM_SNOWFLAKE_USER }}      
+      CDM5_SPARK_USER: ${{ secrets.CDM5_SPARK_USER }}
+      CDM5_SPARK_PASSWORD: ${{ secrets.CDM5_SPARK_PASSWORD }}
+      CDM5_SPARK_CONNECTION_STRING: ${{ secrets.CDM5_SPARK_CONNECTION_STRING }}
+      CDM5_SPARK_CDM_SCHEMA: ${{ secrets.CDM5_SPARK_CDM_SCHEMA }}
+      CDM5_SPARK_OHDSI_SCHEMA: ${{ secrets.CDM5_SPARK_OHDSI_SCHEMA }}
+      CDM_BIG_QUERY_CONNECTION_STRING: ${{ secrets.CDM_BIG_QUERY_CONNECTION_STRING }}
+      CDM_BIG_QUERY_KEY_FILE: ${{ secrets.CDM_BIG_QUERY_KEY_FILE }}
+      CDM_BIG_QUERY_CDM_SCHEMA: ${{ secrets.CDM_BIG_QUERY_CDM_SCHEMA }}
+      CDM_BIG_QUERY_OHDSI_SCHEMA: ${{ secrets.CDM_BIG_QUERY_OHDSI_SCHEMA }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - uses: r-lib/actions/setup-r@v2
         with:
@@ -91,10 +105,13 @@ jobs:
             eval sudo $cmd
           done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
           
-      - name: Reconfigure Java
+      - name: Setup Java
         if: runner.os == 'macOS'
-        run: R CMD javareconf
-
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '8'
+
       - name: Install libssh
         if: runner.os == 'Linux'
         run: |    

diff --git a/.github/workflows/R_CMD_check_main_weekly.yaml b/.github/workflows/R_CMD_check_main_weekly.yaml
@@ -20,30 +20,41 @@ jobs:
       GITHUB_PAT: ${{ secrets.GH_TOKEN }}
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       RSPM: ${{ matrix.config.rspm }}
-      CDM5_ORACLE_CDM_SCHEMA: ${{ secrets.CDM5_ORACLE_CDM_SCHEMA }}
+      CDM5_ORACLE_CDM_SCHEMA: ${{ secrets.CDM5_ORACLE_CDM54_SCHEMA }}
       CDM5_ORACLE_OHDSI_SCHEMA: ${{ secrets.CDM5_ORACLE_OHDSI_SCHEMA }}
       CDM5_ORACLE_PASSWORD: ${{ secrets.CDM5_ORACLE_PASSWORD }}
       CDM5_ORACLE_SERVER: ${{ secrets.CDM5_ORACLE_SERVER }}
       CDM5_ORACLE_USER: ${{ secrets.CDM5_ORACLE_USER }}
-      CDM5_POSTGRESQL_CDM_SCHEMA: ${{ secrets.CDM5_POSTGRESQL_CDM_SCHEMA }}
+      CDM5_POSTGRESQL_CDM_SCHEMA: ${{ secrets.CDM5_POSTGRESQL_CDM54_SCHEMA }}
       CDM5_POSTGRESQL_OHDSI_SCHEMA: ${{ secrets.CDM5_POSTGRESQL_OHDSI_SCHEMA }}
       CDM5_POSTGRESQL_PASSWORD: ${{ secrets.CDM5_POSTGRESQL_PASSWORD }}
       CDM5_POSTGRESQL_SERVER: ${{ secrets.CDM5_POSTGRESQL_SERVER }}
       CDM5_POSTGRESQL_USER: ${{ secrets.CDM5_POSTGRESQL_USER }}
-      CDM5_SQL_SERVER_CDM_SCHEMA: ${{ secrets.CDM5_SQL_SERVER_CDM_SCHEMA }}
+      CDM5_SQL_SERVER_CDM_SCHEMA: ${{ secrets.CDM5_SQL_SERVER_CDM54_SCHEMA }}
       CDM5_SQL_SERVER_OHDSI_SCHEMA: ${{ secrets.CDM5_SQL_SERVER_OHDSI_SCHEMA }}
       CDM5_SQL_SERVER_PASSWORD: ${{ secrets.CDM5_SQL_SERVER_PASSWORD }}
       CDM5_SQL_SERVER_SERVER: ${{ secrets.CDM5_SQL_SERVER_SERVER }}
       CDM5_SQL_SERVER_USER: ${{ secrets.CDM5_SQL_SERVER_USER }}
-      CDM5_REDSHIFT_CDM_SCHEMA: ${{ secrets.CDM5_REDSHIFT_CDM_SCHEMA }}
+      CDM5_REDSHIFT_CDM_SCHEMA: ${{ secrets.CDM5_REDSHIFT_CDM54_SCHEMA }}
       CDM5_REDSHIFT_OHDSI_SCHEMA: ${{ secrets.CDM5_REDSHIFT_OHDSI_SCHEMA }}
       CDM5_REDSHIFT_PASSWORD: ${{ secrets.CDM5_REDSHIFT_PASSWORD }}
       CDM5_REDSHIFT_SERVER: ${{ secrets.CDM5_REDSHIFT_SERVER }}
       CDM5_REDSHIFT_USER: ${{ secrets.CDM5_REDSHIFT_USER }}
+      CDM_SNOWFLAKE_CDM53_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_CDM53_SCHEMA }}
+      CDM_SNOWFLAKE_OHDSI_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_OHDSI_SCHEMA }}
+      CDM_SNOWFLAKE_PASSWORD: ${{ secrets.CDM_SNOWFLAKE_PASSWORD }}
+      CDM_SNOWFLAKE_CONNECTION_STRING: ${{ secrets.CDM_SNOWFLAKE_CONNECTION_STRING }}
+      CDM_SNOWFLAKE_USER: ${{ secrets.CDM_SNOWFLAKE_USER }}      
       CDM5_SPARK_USER: ${{ secrets.CDM5_SPARK_USER }}
       CDM5_SPARK_PASSWORD: ${{ secrets.CDM5_SPARK_PASSWORD }}
       CDM5_SPARK_CONNECTION_STRING: ${{ secrets.CDM5_SPARK_CONNECTION_STRING }}
-
+      CDM5_SPARK_CDM_SCHEMA: ${{ secrets.CDM5_SPARK_CDM_SCHEMA }}
+      CDM5_SPARK_OHDSI_SCHEMA: ${{ secrets.CDM5_SPARK_OHDSI_SCHEMA }}
+      CDM_BIG_QUERY_CONNECTION_STRING: ${{ secrets.CDM_BIG_QUERY_CONNECTION_STRING }}
+      CDM_BIG_QUERY_KEY_FILE: ${{ secrets.CDM_BIG_QUERY_KEY_FILE }}
+      CDM_BIG_QUERY_CDM_SCHEMA: ${{ secrets.CDM_BIG_QUERY_CDM_SCHEMA }}
+      CDM_BIG_QUERY_OHDSI_SCHEMA: ${{ secrets.CDM_BIG_QUERY_OHDSI_SCHEMA }}
+
     steps:
       - uses: actions/checkout@v2
 

diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R
@@ -84,49 +84,102 @@ getPlatformConnectionDetails <- function(dbmsPlatform) {
     options("sqlRenderTempEmulationSchema" = NULL)
     cohortTable <- "cohort"
   } else {
-    if (dbmsPlatform == "postgresql") {
-      dbUser <- Sys.getenv("CDM5_POSTGRESQL_USER")
-      dbPassword <- Sys.getenv("CDM5_POSTGRESQL_PASSWORD")
-      dbServer <- Sys.getenv("CDM5_POSTGRESQL_SERVER")
-      cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
-      vocabularyDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
-      cohortDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_OHDSI_SCHEMA")
+    if (dbmsPlatform == "bigquery") {
+      # To avoid rate limit on BigQuery, only test on 1 OS:
+      if (.Platform$OS.type == "windows") {
+          bqKeyFile <- tempfile(fileext = ".json")
+          writeLines(Sys.getenv("CDM_BIG_QUERY_KEY_FILE"), bqKeyFile)
+          if (testthat::is_testing()) {
+            withr::defer(unlink(bqKeyFile, force = TRUE), testthat::teardown_env())
+          }
+          bqConnectionString <- gsub("<keyfile path>",
+                                     normalizePath(bqKeyFile, winslash = "/"),
+                                     Sys.getenv("CDM_BIG_QUERY_CONNECTION_STRING"))
+          connectionDetails = DatabaseConnector::createConnectionDetails(
+              dbms = dbmsPlatform,
+              user = "",
+              password = "",
+              connectionString = !!bqConnectionString,
+              pathToDriver = jdbcDriverFolder
+            )
+        cdmDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_CDM_SCHEMA")
+        vocabularyDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_CDM_SCHEMA")
+        cohortDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_OHDSI_SCHEMA")
+        options(sqlRenderTempEmulationSchema = Sys.getenv("CDM_BIG_QUERY_OHDSI_SCHEMA"))
+      } else {
+        return(NULL)
+      }
     } else if (dbmsPlatform == "oracle") {
-      dbUser <- Sys.getenv("CDM5_ORACLE_USER")
-      dbPassword <- Sys.getenv("CDM5_ORACLE_PASSWORD")
-      dbServer <- Sys.getenv("CDM5_ORACLE_SERVER")
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_ORACLE_USER"),
+        password = URLdecode(Sys.getenv("CDM5_ORACLE_PASSWORD")),
+        server = Sys.getenv("CDM5_ORACLE_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )      
       cdmDatabaseSchema <- Sys.getenv("CDM5_ORACLE_CDM_SCHEMA")
       vocabularyDatabaseSchema <- Sys.getenv("CDM5_ORACLE_CDM_SCHEMA")
       cohortDatabaseSchema <- Sys.getenv("CDM5_ORACLE_OHDSI_SCHEMA")
       options(sqlRenderTempEmulationSchema = Sys.getenv("CDM5_ORACLE_OHDSI_SCHEMA"))
+    } else if (dbmsPlatform == "postgresql") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_POSTGRESQL_USER"),
+        password = URLdecode(Sys.getenv("CDM5_POSTGRESQL_PASSWORD")),
+        server = Sys.getenv("CDM5_POSTGRESQL_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_OHDSI_SCHEMA")
     } else if (dbmsPlatform == "redshift") {
-      dbUser <- Sys.getenv("CDM5_REDSHIFT_USER")
-      dbPassword <- Sys.getenv("CDM5_REDSHIFT_PASSWORD")
-      dbServer <- Sys.getenv("CDM5_REDSHIFT_SERVER")
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_REDSHIFT_USER"),
+        password = URLdecode(Sys.getenv("CDM5_REDSHIFT_PASSWORD")),
+        server = Sys.getenv("CDM5_REDSHIFT_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )      
       cdmDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_CDM_SCHEMA")
       vocabularyDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_CDM_SCHEMA")
       cohortDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_OHDSI_SCHEMA")
+    } else if (dbmsPlatform == "snowflake") {
+      connectionDetails = DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM_SNOWFLAKE_USER"),
+        password = URLdecode(Sys.getenv("CDM_SNOWFLAKE_PASSWORD")),
+        connectionString = Sys.getenv("CDM_SNOWFLAKE_CONNECTION_STRING"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_CDM53_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_CDM53_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_OHDSI_SCHEMA")
+      options(sqlRenderTempEmulationSchema = Sys.getenv("CDM_SNOWFLAKE_OHDSI_SCHEMA"))
+    } else if (dbmsPlatform == "spark") {
+      connectionDetails = DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_SPARK_USER"),
+        password = URLdecode(Sys.getenv("CDM5_SPARK_PASSWORD")),
+        connectionString = Sys.getenv("CDM5_SPARK_CONNECTION_STRING"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_SPARK_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_SPARK_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_SPARK_OHDSI_SCHEMA")
+      options(sqlRenderTempEmulationSchema = Sys.getenv("CDM5_SPARK_OHDSI_SCHEMA"))
     } else if (dbmsPlatform == "sql server") {
-      dbUser <- Sys.getenv("CDM5_SQL_SERVER_USER")
-      dbPassword <- Sys.getenv("CDM5_SQL_SERVER_PASSWORD")
-      dbServer <- Sys.getenv("CDM5_SQL_SERVER_SERVER")
+      connectionDetails <- createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_SQL_SERVER_USER"),
+        password = URLdecode(Sys.getenv("CDM5_SQL_SERVER_PASSWORD")),
+        server = Sys.getenv("CDM5_SQL_SERVER_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )      
       cdmDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_CDM_SCHEMA")
       vocabularyDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_CDM_SCHEMA")
       cohortDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_OHDSI_SCHEMA")
     }
 
-    if (dbServer == "") {
-      return(NULL)
-    }
-
-    connectionDetails <- DatabaseConnector::createConnectionDetails(
-      dbms = dbmsPlatform,
-      user = dbUser,
-      password = URLdecode(dbPassword),
-      server = dbServer,
-      pathToDriver = jdbcDriverFolder
-    )
-
     # Add drivers
     DatabaseConnector::downloadJdbcDrivers(dbmsPlatform, pathToDriver = jdbcDriverFolder)
     # Table created to avoid collisions

diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R
@@ -1,4 +1,4 @@
-dbmsPlatforms <- c("redshift", "postgresql", "oracle", "sql server")
+dbmsPlatforms <- c("bigquery", "oracle", "postgresql", "redshift", "snowflake", "spark", "sql server")
 connectionDetails <- Eunomia::getEunomiaConnectionDetails()
 outputFolder <- tempfile()
 dir.create(outputFolder)

diff --git a/tests/testthat/test-dbms-platforms.R b/tests/testthat/test-dbms-platforms.R
@@ -110,8 +110,9 @@ test_that("platform specific create cohorts with stats, Incremental, get results
   for (dbmsPlatform in dbmsPlatforms) {
     dbmsDetails <- getPlatformConnectionDetails(dbmsPlatform)
     if (is.null(dbmsDetails)) {
-      print(paste("No pltatform details available for", dbmsPlatform))
+      print(paste("No platform details available for", dbmsPlatform))
     } else {
+      print(paste("Testing", dbmsPlatform))
       testPlatform(dbmsDetails)
     }
   }