diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 13af97d63..3e1ee9689 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -31,6 +31,8 @@ jobs: dir: "delphi_hhs" - package: "nchs_mortality" dir: "delphi_nchs_mortality" + - package: "nssp" + dir: "delphi_nssp" - package: "nwss_wastewater" dir: "delphi_nwss" - package: "quidel_covidtest" diff --git a/Jenkinsfile b/Jenkinsfile index 0052fd215..3011ebde7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ - TODO: #527 Get this list automatically from python-ci.yml at runtime. */ -def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "nchs_mortality", "quidel_covidtest", "sir_complainsalot", "doctor_visits", "nwss_wastewater"] +def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "nchs_mortality", "quidel_covidtest", "sir_complainsalot", "doctor_visits", "nwss_wastewater", "nssp"] def build_package_main = [:] def build_package_prod = [:] def deploy_staging = [:] diff --git a/_delphi_utils_python/tests/test_weekday.py b/_delphi_utils_python/tests/test_weekday.py index 9b187e114..53da7088c 100644 --- a/_delphi_utils_python/tests/test_weekday.py +++ b/_delphi_utils_python/tests/test_weekday.py @@ -58,4 +58,4 @@ def test_calc_adjustment(self): # The date and "den" column are unchanged by this function assert np.allclose(result["num"].values, expected_nums) assert np.allclose(result["den"].values, self.TEST_DATA["den"].values) - assert np.array_equal(result["date"].values, self.TEST_DATA["date"].values) \ No newline at end of file + assert np.array_equal(result["date"].values, self.TEST_DATA["date"].values) diff --git a/ansible/templates/nssp-params-prod.json.j2 b/ansible/templates/nssp-params-prod.json.j2 new file mode 100644 index 000000000..b131b6130 --- /dev/null +++ b/ansible/templates/nssp-params-prod.json.j2 @@ -0,0 +1,29 @@ +{ + "common": { + "export_dir": "/common/covidcast/receiving/nssp", + "log_filename": "/var/log/indicators/nssp.log", + "log_exceptions": false + }, + "indicator": { + "wip_signal": true, + "static_file_dir": "./static", + "socrata_token": "{{ nssp_token }}" + }, + "validation": { + "common": { + "data_source": "nssp", + "api_credentials": "{{ validation_api_key }}", + "span_length": 15, + "min_expected_lag": {"all": "7"}, + "max_expected_lag": {"all": "13"}, + "dry_run": true, + "suppressed_errors": [] + }, + "static": { + "minimum_sample_size": 0, + "missing_se_allowed": true, + "missing_sample_size_allowed": true + }, + "dynamic": {} + } +} diff --git a/ansible/templates/sir_complainsalot-params-prod.json.j2 b/ansible/templates/sir_complainsalot-params-prod.json.j2 index d6f5da081..a016fc470 100644 --- a/ansible/templates/sir_complainsalot-params-prod.json.j2 +++ b/ansible/templates/sir_complainsalot-params-prod.json.j2 @@ -50,6 +50,10 @@ "hhs": { "max_age":15, "maintainers": [] + }, + "nssp": { + "max_age":13, + "maintainers": [] } } } diff --git a/ansible/vars.yaml b/ansible/vars.yaml index 8e059c873..ff9ba135c 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -56,6 +56,9 @@ nchs_mortality_token: "{{ vault_cdc_socrata_token }}" # NWSS nwss_wastewater_token: "{{ vault_cdc_socrata_token }}" +# nssp +nssp_token: "{{ vault_cdc_socrata_token }}" + # SirCAL sir_complainsalot_api_key: "{{ vault_sir_complainsalot_api_key }}" sir_complainsalot_slack_token: "{{ vault_sir_complainsalot_slack_token }}" diff --git a/notebooks/.Rprofile b/notebooks/.Rprofile new file mode 100644 index 000000000..81b960f5c --- /dev/null +++ b/notebooks/.Rprofile @@ -0,0 +1 @@ +source("renv/activate.R") diff --git a/notebooks/nssp/cor_dashboard.Rmd b/notebooks/nssp/cor_dashboard.Rmd new file mode 100644 index 000000000..58d3ed6a0 --- /dev/null +++ b/notebooks/nssp/cor_dashboard.Rmd @@ -0,0 +1,257 @@ +--- +title: "Correlation Analyses for COVID-19 Indicators" +author: "Delphi Group" +date: "`r format(Sys.time(), '%B %d, %Y')`" +output: + html_document: + code_folding: hide +--- + +```{r, include = FALSE} +knitr::opts_chunk$set(message = FALSE, warning = FALSE, fig.width = 8, + fig.height = 7) +``` + +### Getting data +This requires that you've already run the nssp pipeline. See the `nssp` directory for instructions on doing that. +First loading some libraries and reading the results from the pipeline: +```{r} +library(covidcast) +library(epidatr) +library(dplyr) +library(ggplot2) + +library(purrr) +library(tidyverse) +library(dplyr) +library(readr) +files <- list.files(here::here("nssp/receiving"), pattern="\\.csv$", full.names = TRUE) +read_row <- function(filename) { + split_name <- filename %>% + tools::file_path_sans_ext() %>% + strsplit("/") %>% `[[`(1) %>% tail(n=1) %>% + strsplit("_") %>% `[[`(1) + week_number <- split_name[[2]] + geo_type <- split_name[[3]] + col_name <- split_name[-(1:3)] %>% paste(collapse = "_") + read_csv(filename, show_col_types = FALSE) %>% + as_tibble %>% + mutate(signal = col_name, + geo_type = geo_type, + week_number = week_number) %>% + mutate(across(geo_id, factor)) %>% + rename(geo_value = geo_id, time_value = week_number) %>% + select(-missing_se, -se, -sample_size, -missing_sample_size) %>% + return +} +res <- map(files, read_row) +nssp_data <- bind_rows(res) +nssp_state <- nssp_data %>% + filter(geo_type == "state") %>% + mutate(time_value = epidatr:::parse_api_week(time_value)) %>% + as_epi_df(time_type = "week", geo_type = "state") %>% + select(-missing_val, -geo_type) +unique(nssp_data$time_value) +``` +And epidatr versions of hhs for comparison +```{r} +library(epidatr) +eval_time <- epidatr::epirange(from = "2020-01-01", to = Sys.Date()) +fetch_args <- epidatr::fetch_args_list(return_empty = TRUE, timeout_seconds = 300) + +flu_hhs <- epidatr::pub_covidcast( + source = "hhs", + signals = "confirmed_admissions_influenza_1d_prop_7dav", + geo_type = "state", + time_type = "day", + geo_values = "*", + time_values = eval_time, + fetch_args = fetch_args + ) %>% + select(-signal, -source, - time_type) + +covid_hhs <- epidatr::pub_covidcast( + source = "hhs", + signals = "confirmed_admissions_covid_1d_prop_7dav", + geo_type = "state", + time_type = "day", + geo_values = "*", + time_values = eval_time, + fetch_args = fetch_args + ) %>% + select(-signal, -source, - time_type) + + +nchs <- epidatr::pub_covidcast( + source = "nchs-mortality", + signals = "deaths_allcause_incidence_num", + geo_type = "state", + time_type = "week", + geo_values = "*", + time_values = epidatr::epirange(from = "202001", to = "202418"), + fetch_args = epidatr::fetch_args_list(return_empty = TRUE, timeout_seconds = 300) + ) +``` +# Flu +```{r} +library(epiprocess) +nssp_flu_state <- nssp_state %>% filter(signal == "pct_ed_visits_influenza") %>% select(-signal) %>% drop_na %>% rename(pct_flu_visits = val) %>% as_epi_df(time_type = "week", geo_type = "state") +week_starts <- nssp_flu_state$time_value %>% unique() +flu_hhs_weekly <- flu_hhs %>% select(geo_value, time_value, value) %>% filter(time_value %in% week_starts) %>% rename(conf_admission = value) %>% drop_na %>% as_epi_df(time_type = "week", geo_type = "state") +joined <- nssp_flu_state %>% left_join(flu_hhs_weekly) +``` + +After the necessary joining, lets look at the average correlations +```{r} +cor(joined$pct_flu_visits, joined$conf_admission, method = "spearman") +``` +So the overall correlation is pretty high. + +## Correlations sliced by state +```{r} +correlations_space_flu <- epi_cor(joined, pct_flu_visits, conf_admission, cor_by = "geo_value", use = "complete.obs", method = "spearman") +library(maps) # For map data +states_map <- map_data("state") +mapped <- states_map %>% as_tibble %>% mutate(geo_value = setNames(tolower(state.abb), tolower(state.name))[region]) %>% right_join(correlations_space_flu) %>% arrange(group, order) +library(viridis) +ggplot(mapped, aes(x = long, y = lat, group = group, fill = cor)) + + geom_polygon(colour = "black") + + scale_fill_viridis(discrete=FALSE, option="viridis", limits = c(0,1)) + + coord_map("polyconic") + + labs(title = "Spearman Correlations between Flu ER visits and Flu hospital admissions") +ggsave("flu_ER_admissions_state_correlations.pdf") +``` +Over space, hospital admissions look like they're highly correlated with ER visits (which makes sense, frequently when one is admitted it is via the ER). +The lowest overall correlation is +```{r} +correlations_space_flu %>% summarize(across(where(is.numeric), .fns = list(min = min, median = median, mean = mean, std = sd, q25 = ~quantile(.,0.25), q75 = ~quantile(.,0.75), max = max))) +``` +### Lag evaluation +```{r} +library(purrr) +lags <- 0:35 + +lagged_flu_state <- map_dfr(lags, function(lag) { + epi_cor(joined, pct_flu_visits, conf_admission, cor_by = geo_value, dt1 = lag, use = "complete.obs", method = "spearman") %>% + mutate(lag = .env$lag) +}) + +lagged_flu_state %>% + group_by(lag) %>% + summarize(mean = mean(cor, na.rm = TRUE)) %>% + ggplot(aes(x = lag, y = mean)) + + geom_line() + + geom_point() + + labs(x = "Lag", y = "Mean correlation", title = "Lag comparison for state spearman correlations for flu ER and Hosp admissions") +ggsave("flu_ER_admissions_state_lag_cor.pdf") +``` +Somewhat unsurprisingly, the correlation is highest immediately afterward. +## Correlations sliced by time +```{r} +correlations_time_flu <- epi_cor(joined, pct_flu_visits, conf_admission, cor_by = "time_value", use = "complete.obs", method = "spearman") +correlations_time_flu +ggplot(correlations_time_flu, aes(x = time_value, y = cor)) + geom_line() + lims(y=c(0,1)) + labs(title = "Spearman Correlations between Flu ER visits and Flu hospital admissions") +ggsave("flu_ER_admissions_time_correlations.pdf") +``` +Strangely, sliced by time, we get significantly lower correlations +```{r} +correlations_time_flu %>% summarize(across(where(is.numeric), .fns = list(min = min, median = median, mean = mean, std = sd, q25 = ~quantile(.,0.25), q75 = ~quantile(.,0.75), max = max))) +``` +Seems like we have a Simpson's paradox adjacent result, since for any given location the signals are fairly well correlated when averaged over time, but at a given time, averaging over different locations suggests they're not very well correlated. +If the typical explanation applies, this means that there are large differences in the number of points. + +so, getting the counts: +```{r} +joined %>% group_by(geo_value) %>% count %>% arrange(n) %>% ungroup %>% summarise(across(where(is.numeric), .fns = list(min = min, max = max))) +``` +Each location has 82 + +```{r} +joined %>% group_by(time_value) %>% count %>% arrange(n) %>% ungroup %>% summarise(across(where(is.numeric), .fns = list(min = min, max = max))) +``` +# Covid +```{r} +library(epiprocess) +nssp_data %>% pull(signal) %>% unique +nssp_state <- nssp_data %>% + filter(geo_type == "state") %>% + mutate(time_value = epidatr:::parse_api_week(time_value)) %>% + as_epi_df(time_type = "week", geo_type = "state") %>% + select(-missing_val, -geo_type) +nssp_covid_state <- nssp_state %>% filter(signal == "pct_ed_visits_covid") %>% select(-signal) %>% drop_na %>% rename(pct_covid_visits = val) %>% as_epi_df(time_type = "week", geo_type = "state") +week_starts <- nssp_covid_state$time_value %>% unique() +covid_hhs_weekly <- covid_hhs %>% select(geo_value, time_value, value) %>% filter(time_value %in% week_starts) %>% rename(conf_admission = value) %>% drop_na %>% as_epi_df(time_type = "week", geo_type = "state") +joined_covid <- nssp_covid_state %>% left_join(covid_hhs_weekly) +``` + +After the necessary joining, lets look at the average correlations +```{r} +cor(joined_covid$pct_covid_visits, joined_covid$conf_admission, method = "spearman") +``` +So the overall correlation is pretty high, but lower than flu. + +## Correlations sliced by state +```{r} +correlations_space_covid <- epi_cor(joined_covid, pct_covid_visits, conf_admission, cor_by = "geo_value", use = "complete.obs", method = "spearman") +library(maps) # For map data +states_map <- map_data("state") +mapped <- states_map %>% as_tibble %>% mutate(geo_value = setNames(tolower(state.abb), tolower(state.name))[region]) %>% right_join(correlations_space_covid) %>% arrange(group, order) +library(viridis) +ggplot(mapped, aes(x = long, y = lat, group = group, fill = cor)) + + geom_polygon(colour = "black") + + scale_fill_viridis(discrete=FALSE, option="viridis", limits = c(0,1)) + + coord_map("polyconic") + + labs(title = "Spearman Correlations between covid ER visits and covid hospital admissions") +ggsave("covid_ER_admissions_state_correlations.pdf") +ggsave("covid_ER_admissions_state_correlations.png") +``` +Over space, hospital admissions look like they're highly correlated with ER visits (which makes sense, frequently when one is admitted it is via the ER). +The lowest overall correlation is +```{r} +correlations_space_covid %>% summarize(across(where(is.numeric), .fns = list(min = min, median = median, mean = mean, std = sd, q25 = ~quantile(.,0.25), q75 = ~quantile(.,0.75), max = max))) +``` +### Lag evaluation +```{r} +library(purrr) +lags <- 0:35 + +lagged_covid_state <- map_dfr(lags, function(lag) { + epi_cor(joined_covid, pct_covid_visits, conf_admission, cor_by = geo_value, dt1 = -lag, use = "complete.obs", method = "spearman") %>% + mutate(lag = .env$lag) +}) + +lagged_covid_state %>% + group_by(lag) %>% + summarize(mean = mean(cor, na.rm = TRUE)) %>% + ggplot(aes(x = lag, y = mean)) + + geom_line() + + geom_point() + + labs(x = "Lag", y = "Mean correlation", title = "Lag comparison for state spearman correlations for covid ER and Hosp admissions") +ggsave("covid_ER_admissions_state_lag_cor.pdf") +ggsave("covid_ER_admissions_state_lag_cor.png") +``` +Somewhat unsurprisingly, the correlation is highest immediately afterward, though its significantly lower than in the flu case. +## Correlations sliced by time +```{r} +correlations_time_covid <- epi_cor(joined_covid, pct_covid_visits, conf_admission, cor_by = "time_value", use = "complete.obs", method = "spearman") +correlations_time_covid +ggplot(correlations_time_covid, aes(x = time_value, y = cor)) + geom_line() + lims(y=c(0,1)) + labs(title = "Spearman Correlations between covid ER visits and covid hospital admissions") +ggsave("covid_ER_admissions_time_correlations.pdf") +ggsave("covid_ER_admissions_time_correlations.png") +``` +Strangely, sliced by time, we get significantly lower correlations, some of them are even negative +```{r} +correlations_time_covid %>% summarize(across(where(is.numeric), .fns = list(min = min, median = median, mean = mean, std = sd, q25 = ~quantile(.,0.25), q75 = ~quantile(.,0.75), max = max))) +``` +Seems like we have a Simpson's paradox adjacent result, since for any given location the signals are fairly well correlated when averaged over time, but at a given time, averaging over different locations suggests they're not very well correlated. +If the typical explanation applies, this means that there are large differences in the number of points. + +so, getting the counts: +```{r} +joined_covid %>% group_by(geo_value) %>% count %>% arrange(n) %>% ungroup %>% summarise(across(where(is.numeric), .fns = list(min = min, max = max))) +``` +Each location has 82 + +```{r} +joined_covid %>% group_by(time_value) %>% count %>% arrange(n) %>% ungroup %>% summarise(across(where(is.numeric), .fns = list(min = min, max = max))) +``` diff --git a/notebooks/nssp/covid_ER_admissions_state_correlations.pdf b/notebooks/nssp/covid_ER_admissions_state_correlations.pdf new file mode 100644 index 000000000..35f272b03 Binary files /dev/null and b/notebooks/nssp/covid_ER_admissions_state_correlations.pdf differ diff --git a/notebooks/nssp/covid_ER_admissions_state_lag_cor.pdf b/notebooks/nssp/covid_ER_admissions_state_lag_cor.pdf new file mode 100644 index 000000000..86852e048 Binary files /dev/null and b/notebooks/nssp/covid_ER_admissions_state_lag_cor.pdf differ diff --git a/notebooks/nssp/covid_ER_admissions_time_correlations.pdf b/notebooks/nssp/covid_ER_admissions_time_correlations.pdf new file mode 100644 index 000000000..be131cd92 Binary files /dev/null and b/notebooks/nssp/covid_ER_admissions_time_correlations.pdf differ diff --git a/notebooks/nssp/flu_ER_admissions_state_correlations.pdf b/notebooks/nssp/flu_ER_admissions_state_correlations.pdf new file mode 100644 index 000000000..56e04209d Binary files /dev/null and b/notebooks/nssp/flu_ER_admissions_state_correlations.pdf differ diff --git a/notebooks/nssp/flu_ER_admissions_state_lag_cor.pdf b/notebooks/nssp/flu_ER_admissions_state_lag_cor.pdf new file mode 100644 index 000000000..d08c84c32 Binary files /dev/null and b/notebooks/nssp/flu_ER_admissions_state_lag_cor.pdf differ diff --git a/notebooks/nssp/flu_ER_admissions_time_correlations.pdf b/notebooks/nssp/flu_ER_admissions_time_correlations.pdf new file mode 100644 index 000000000..c00c1479e Binary files /dev/null and b/notebooks/nssp/flu_ER_admissions_time_correlations.pdf differ diff --git a/notebooks/renv.lock b/notebooks/renv.lock new file mode 100644 index 000000000..dd103cd87 --- /dev/null +++ b/notebooks/renv.lock @@ -0,0 +1,1367 @@ +{ + "R": { + "Version": "4.4.0", + "Repositories": [ + { + "Name": "RSPM", + "URL": "https://packagemanager.posit.co/all/latest" + } + ] + }, + "Packages": { + "DBI": { + "Package": "DBI", + "Version": "1.2.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods" + ], + "Hash": "164809cd72e1d5160b4cb3aa57f510fe" + }, + "KernSmooth": { + "Package": "KernSmooth", + "Version": "2.23-22", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "R", + "stats" + ], + "Hash": "2fecebc3047322fa5930f74fae5de70f" + }, + "MASS": { + "Package": "MASS", + "Version": "7.3-60.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "graphics", + "methods", + "stats", + "utils" + ], + "Hash": "2f342c46163b0b54d7b64d1f798e2c78" + }, + "MMWRweek": { + "Package": "MMWRweek", + "Version": "0.1.3", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "4329e57e2536e12afe479e8571416dbc" + }, + "Matrix": { + "Package": "Matrix", + "Version": "1.7-0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "graphics", + "grid", + "lattice", + "methods", + "stats", + "utils" + ], + "Hash": "1920b2f11133b12350024297d8a4ff4a" + }, + "R6": { + "Package": "R6", + "Version": "2.5.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "470851b6d5d0ac559e9d01bb352b4021" + }, + "RColorBrewer": { + "Package": "RColorBrewer", + "Version": "1.1-3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "45f0398006e83a5b10b72a90663d8d8c" + }, + "Rcpp": { + "Package": "Rcpp", + "Version": "1.0.12", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "methods", + "utils" + ], + "Hash": "5ea2700d21e038ace58269ecdbeb9ec0" + }, + "askpass": { + "Package": "askpass", + "Version": "1.2.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "sys" + ], + "Hash": "cad6cf7f1d5f6e906700b9d3e718c796" + }, + "backports": { + "Package": "backports", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "c39fbec8a30d23e721980b8afb31984c" + }, + "base64enc": { + "Package": "base64enc", + "Version": "0.1-3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "543776ae6848fde2f48ff3816d0628bc" + }, + "bit": { + "Package": "bit", + "Version": "4.0.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "d242abec29412ce988848d0294b208fd" + }, + "bit64": { + "Package": "bit64", + "Version": "4.0.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "bit", + "methods", + "stats", + "utils" + ], + "Hash": "9fe98599ca456d6552421db0d6772d8f" + }, + "bslib": { + "Package": "bslib", + "Version": "0.7.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "base64enc", + "cachem", + "fastmap", + "grDevices", + "htmltools", + "jquerylib", + "jsonlite", + "lifecycle", + "memoise", + "mime", + "rlang", + "sass" + ], + "Hash": "8644cc53f43828f19133548195d7e59e" + }, + "cachem": { + "Package": "cachem", + "Version": "1.0.8", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "fastmap", + "rlang" + ], + "Hash": "c35768291560ce302c0a6589f92e837d" + }, + "checkmate": { + "Package": "checkmate", + "Version": "2.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "backports", + "utils" + ], + "Hash": "c01cab1cb0f9125211a6fc99d540e315" + }, + "class": { + "Package": "class", + "Version": "7.3-22", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "MASS", + "R", + "stats", + "utils" + ], + "Hash": "f91f6b29f38b8c280f2b9477787d4bb2" + }, + "classInt": { + "Package": "classInt", + "Version": "0.4-10", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "KernSmooth", + "R", + "class", + "e1071", + "grDevices", + "graphics", + "stats" + ], + "Hash": "f5a40793b1ae463a7ffb3902a95bf864" + }, + "cli": { + "Package": "cli", + "Version": "3.6.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "utils" + ], + "Hash": "1216ac65ac55ec0058a6f75d7ca0fd52" + }, + "clipr": { + "Package": "clipr", + "Version": "0.8.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "utils" + ], + "Hash": "3f038e5ac7f41d4ac41ce658c85e3042" + }, + "colorspace": { + "Package": "colorspace", + "Version": "2.1-0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "graphics", + "methods", + "stats" + ], + "Hash": "f20c47fd52fae58b4e377c37bb8c335b" + }, + "covidcast": { + "Package": "covidcast", + "Version": "0.5.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "MMWRweek", + "R", + "dplyr", + "ggplot2", + "grDevices", + "httr", + "purrr", + "rlang", + "sf", + "tidyr", + "xml2" + ], + "Hash": "ee88255e014ff787bd3db3f4735fb24a" + }, + "cpp11": { + "Package": "cpp11", + "Version": "0.4.7", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "5a295d7d963cc5035284dcdbaf334f4e" + }, + "crayon": { + "Package": "crayon", + "Version": "1.5.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "grDevices", + "methods", + "utils" + ], + "Hash": "e8a1e41acf02548751f45c718d55aa6a" + }, + "credentials": { + "Package": "credentials", + "Version": "2.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "askpass", + "curl", + "jsonlite", + "openssl", + "sys" + ], + "Hash": "c7844b32098dcbd1c59cbd8dddb4ecc6" + }, + "curl": { + "Package": "curl", + "Version": "5.2.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "411ca2c03b1ce5f548345d2fc2685f7a" + }, + "desc": { + "Package": "desc", + "Version": "1.4.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "cli", + "utils" + ], + "Hash": "99b79fcbd6c4d1ce087f5c5c758b384f" + }, + "digest": { + "Package": "digest", + "Version": "0.6.35", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "utils" + ], + "Hash": "698ece7ba5a4fa4559e3d537e7ec3d31" + }, + "dplyr": { + "Package": "dplyr", + "Version": "1.1.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "cli", + "generics", + "glue", + "lifecycle", + "magrittr", + "methods", + "pillar", + "rlang", + "tibble", + "tidyselect", + "utils", + "vctrs" + ], + "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" + }, + "e1071": { + "Package": "e1071", + "Version": "1.7-14", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "class", + "grDevices", + "graphics", + "methods", + "proxy", + "stats", + "utils" + ], + "Hash": "4ef372b716824753719a8a38b258442d" + }, + "epidatr": { + "Package": "epidatr", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "MMWRweek", + "R", + "cachem", + "checkmate", + "cli", + "glue", + "httr", + "jsonlite", + "magrittr", + "openssl", + "purrr", + "rappdirs", + "readr", + "tibble", + "usethis", + "xml2" + ], + "Hash": "cf6f60be321bfd49298e27717be8c2b2" + }, + "evaluate": { + "Package": "evaluate", + "Version": "0.23", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods" + ], + "Hash": "daf4a1246be12c1fa8c7705a0935c1a0" + }, + "fansi": { + "Package": "fansi", + "Version": "1.0.6", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "utils" + ], + "Hash": "962174cf2aeb5b9eea581522286a911f" + }, + "farver": { + "Package": "farver", + "Version": "2.1.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "8106d78941f34855c440ddb946b8f7a5" + }, + "fastmap": { + "Package": "fastmap", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "f7736a18de97dea803bde0a2daaafb27" + }, + "fontawesome": { + "Package": "fontawesome", + "Version": "0.5.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "htmltools", + "rlang" + ], + "Hash": "c2efdd5f0bcd1ea861c2d4e2a883a67d" + }, + "fs": { + "Package": "fs", + "Version": "1.6.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods" + ], + "Hash": "15aeb8c27f5ea5161f9f6a641fafd93a" + }, + "generics": { + "Package": "generics", + "Version": "0.1.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods" + ], + "Hash": "15e9634c0fcd294799e9b2e929ed1b86" + }, + "gert": { + "Package": "gert", + "Version": "2.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "askpass", + "credentials", + "openssl", + "rstudioapi", + "sys", + "zip" + ], + "Hash": "f70d3fe2d9e7654213a946963d1591eb" + }, + "ggplot2": { + "Package": "ggplot2", + "Version": "3.5.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "MASS", + "R", + "cli", + "glue", + "grDevices", + "grid", + "gtable", + "isoband", + "lifecycle", + "mgcv", + "rlang", + "scales", + "stats", + "tibble", + "vctrs", + "withr" + ], + "Hash": "44c6a2f8202d5b7e878ea274b1092426" + }, + "gh": { + "Package": "gh", + "Version": "1.4.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "gitcreds", + "glue", + "httr2", + "ini", + "jsonlite", + "lifecycle", + "rlang" + ], + "Hash": "fbbbc48eba7a6626a08bb365e44b563b" + }, + "gitcreds": { + "Package": "gitcreds", + "Version": "0.1.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "ab08ac61f3e1be454ae21911eb8bc2fe" + }, + "glue": { + "Package": "glue", + "Version": "1.7.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "methods" + ], + "Hash": "e0b3a53876554bd45879e596cdb10a52" + }, + "gtable": { + "Package": "gtable", + "Version": "0.3.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "glue", + "grid", + "lifecycle", + "rlang" + ], + "Hash": "e18861963cbc65a27736e02b3cd3c4a0" + }, + "highr": { + "Package": "highr", + "Version": "0.10", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "xfun" + ], + "Hash": "06230136b2d2b9ba5805e1963fa6e890" + }, + "hms": { + "Package": "hms", + "Version": "1.1.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "lifecycle", + "methods", + "pkgconfig", + "rlang", + "vctrs" + ], + "Hash": "b59377caa7ed00fa41808342002138f9" + }, + "htmltools": { + "Package": "htmltools", + "Version": "0.5.8.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "base64enc", + "digest", + "fastmap", + "grDevices", + "rlang", + "utils" + ], + "Hash": "81d371a9cc60640e74e4ab6ac46dcedc" + }, + "httr": { + "Package": "httr", + "Version": "1.4.7", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "curl", + "jsonlite", + "mime", + "openssl" + ], + "Hash": "ac107251d9d9fd72f0ca8049988f1d7f" + }, + "httr2": { + "Package": "httr2", + "Version": "1.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "cli", + "curl", + "glue", + "lifecycle", + "magrittr", + "openssl", + "rappdirs", + "rlang", + "vctrs", + "withr" + ], + "Hash": "03d741c92fda96d98c3a3f22494e3b4a" + }, + "ini": { + "Package": "ini", + "Version": "0.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "6154ec2223172bce8162d4153cda21f7" + }, + "isoband": { + "Package": "isoband", + "Version": "0.2.7", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "grid", + "utils" + ], + "Hash": "0080607b4a1a7b28979aecef976d8bc2" + }, + "jquerylib": { + "Package": "jquerylib", + "Version": "0.1.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "htmltools" + ], + "Hash": "5aab57a3bd297eee1c1d862735972182" + }, + "jsonlite": { + "Package": "jsonlite", + "Version": "1.8.8", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "methods" + ], + "Hash": "e1b9c55281c5adc4dd113652d9e26768" + }, + "knitr": { + "Package": "knitr", + "Version": "1.46", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "evaluate", + "highr", + "methods", + "tools", + "xfun", + "yaml" + ], + "Hash": "6e008ab1d696a5283c79765fa7b56b47" + }, + "labeling": { + "Package": "labeling", + "Version": "0.4.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "graphics", + "stats" + ], + "Hash": "b64ec208ac5bc1852b285f665d6368b3" + }, + "lattice": { + "Package": "lattice", + "Version": "0.22-6", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "graphics", + "grid", + "stats", + "utils" + ], + "Hash": "cc5ac1ba4c238c7ca9fa6a87ca11a7e2" + }, + "lifecycle": { + "Package": "lifecycle", + "Version": "1.0.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "glue", + "rlang" + ], + "Hash": "b8552d117e1b808b09a832f589b79035" + }, + "magrittr": { + "Package": "magrittr", + "Version": "2.0.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "7ce2733a9826b3aeb1775d56fd305472" + }, + "memoise": { + "Package": "memoise", + "Version": "2.0.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "cachem", + "rlang" + ], + "Hash": "e2817ccf4a065c5d9d7f2cfbe7c1d78c" + }, + "mgcv": { + "Package": "mgcv", + "Version": "1.9-1", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "Matrix", + "R", + "graphics", + "methods", + "nlme", + "splines", + "stats", + "utils" + ], + "Hash": "110ee9d83b496279960e162ac97764ce" + }, + "mime": { + "Package": "mime", + "Version": "0.12", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "tools" + ], + "Hash": "18e9c28c1d3ca1560ce30658b22ce104" + }, + "munsell": { + "Package": "munsell", + "Version": "0.5.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "colorspace", + "methods" + ], + "Hash": "4fd8900853b746af55b81fda99da7695" + }, + "nlme": { + "Package": "nlme", + "Version": "3.1-164", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "graphics", + "lattice", + "stats", + "utils" + ], + "Hash": "a623a2239e642806158bc4dc3f51565d" + }, + "openssl": { + "Package": "openssl", + "Version": "2.1.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "askpass" + ], + "Hash": "ea2475b073243d9d338aa8f086ce973e" + }, + "pillar": { + "Package": "pillar", + "Version": "1.9.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "cli", + "fansi", + "glue", + "lifecycle", + "rlang", + "utf8", + "utils", + "vctrs" + ], + "Hash": "15da5a8412f317beeee6175fbc76f4bb" + }, + "pkgconfig": { + "Package": "pkgconfig", + "Version": "2.0.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "utils" + ], + "Hash": "01f28d4278f15c76cddbea05899c5d6f" + }, + "prettyunits": { + "Package": "prettyunits", + "Version": "1.2.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "6b01fc98b1e86c4f705ce9dcfd2f57c7" + }, + "progress": { + "Package": "progress", + "Version": "1.2.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "crayon", + "hms", + "prettyunits" + ], + "Hash": "f4625e061cb2865f111b47ff163a5ca6" + }, + "proxy": { + "Package": "proxy", + "Version": "0.4-27", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "stats", + "utils" + ], + "Hash": "e0ef355c12942cf7a6b91a6cfaea8b3e" + }, + "purrr": { + "Package": "purrr", + "Version": "1.0.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "lifecycle", + "magrittr", + "rlang", + "vctrs" + ], + "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc" + }, + "rappdirs": { + "Package": "rappdirs", + "Version": "0.3.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "5e3c5dc0b071b21fa128676560dbe94d" + }, + "readr": { + "Package": "readr", + "Version": "2.1.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "cli", + "clipr", + "cpp11", + "crayon", + "hms", + "lifecycle", + "methods", + "rlang", + "tibble", + "tzdb", + "utils", + "vroom" + ], + "Hash": "9de96463d2117f6ac49980577939dfb3" + }, + "renv": { + "Package": "renv", + "Version": "1.0.7", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "utils" + ], + "Hash": "397b7b2a265bc5a7a06852524dabae20" + }, + "rlang": { + "Package": "rlang", + "Version": "1.1.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "utils" + ], + "Hash": "42548638fae05fd9a9b5f3f437fbbbe2" + }, + "rmarkdown": { + "Package": "rmarkdown", + "Version": "2.26", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "bslib", + "evaluate", + "fontawesome", + "htmltools", + "jquerylib", + "jsonlite", + "knitr", + "methods", + "tinytex", + "tools", + "utils", + "xfun", + "yaml" + ], + "Hash": "9b148e7f95d33aac01f31282d49e4f44" + }, + "rprojroot": { + "Package": "rprojroot", + "Version": "2.0.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "4c8415e0ec1e29f3f4f6fc108bef0144" + }, + "rstudioapi": { + "Package": "rstudioapi", + "Version": "0.16.0", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "96710351d642b70e8f02ddeb237c46a7" + }, + "s2": { + "Package": "s2", + "Version": "1.1.6", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "Rcpp", + "wk" + ], + "Hash": "32f7b1a15bb01ae809022960abad5363" + }, + "sass": { + "Package": "sass", + "Version": "0.4.9", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R6", + "fs", + "htmltools", + "rappdirs", + "rlang" + ], + "Hash": "d53dbfddf695303ea4ad66f86e99b95d" + }, + "scales": { + "Package": "scales", + "Version": "1.3.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "R6", + "RColorBrewer", + "cli", + "farver", + "glue", + "labeling", + "lifecycle", + "munsell", + "rlang", + "viridisLite" + ], + "Hash": "c19df082ba346b0ffa6f833e92de34d1" + }, + "sf": { + "Package": "sf", + "Version": "1.0-16", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "DBI", + "R", + "Rcpp", + "classInt", + "grDevices", + "graphics", + "grid", + "magrittr", + "methods", + "s2", + "stats", + "tools", + "units", + "utils" + ], + "Hash": "ad57b543f7c3fca05213ba78ff63df9b" + }, + "stringi": { + "Package": "stringi", + "Version": "1.8.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "stats", + "tools", + "utils" + ], + "Hash": "058aebddea264f4c99401515182e656a" + }, + "stringr": { + "Package": "stringr", + "Version": "1.5.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "glue", + "lifecycle", + "magrittr", + "rlang", + "stringi", + "vctrs" + ], + "Hash": "960e2ae9e09656611e0b8214ad543207" + }, + "sys": { + "Package": "sys", + "Version": "3.4.2", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "3a1be13d68d47a8cd0bfd74739ca1555" + }, + "tibble": { + "Package": "tibble", + "Version": "3.2.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "fansi", + "lifecycle", + "magrittr", + "methods", + "pillar", + "pkgconfig", + "rlang", + "utils", + "vctrs" + ], + "Hash": "a84e2cc86d07289b3b6f5069df7a004c" + }, + "tidyr": { + "Package": "tidyr", + "Version": "1.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "cpp11", + "dplyr", + "glue", + "lifecycle", + "magrittr", + "purrr", + "rlang", + "stringr", + "tibble", + "tidyselect", + "utils", + "vctrs" + ], + "Hash": "915fb7ce036c22a6a33b5a8adb712eb1" + }, + "tidyselect": { + "Package": "tidyselect", + "Version": "1.2.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "glue", + "lifecycle", + "rlang", + "vctrs", + "withr" + ], + "Hash": "829f27b9c4919c16b593794a6344d6c0" + }, + "tinytex": { + "Package": "tinytex", + "Version": "0.50", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "xfun" + ], + "Hash": "be7a76845222ad20adb761f462eed3ea" + }, + "tzdb": { + "Package": "tzdb", + "Version": "0.4.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cpp11" + ], + "Hash": "f561504ec2897f4d46f0c7657e488ae1" + }, + "units": { + "Package": "units", + "Version": "0.8-5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "Rcpp" + ], + "Hash": "119d19da480e873f72241ff6962ffd83" + }, + "usethis": { + "Package": "usethis", + "Version": "2.2.3", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "clipr", + "crayon", + "curl", + "desc", + "fs", + "gert", + "gh", + "glue", + "jsonlite", + "lifecycle", + "purrr", + "rappdirs", + "rlang", + "rprojroot", + "rstudioapi", + "stats", + "utils", + "whisker", + "withr", + "yaml" + ], + "Hash": "d524fd42c517035027f866064417d7e6" + }, + "utf8": { + "Package": "utf8", + "Version": "1.2.4", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "62b65c52671e6665f803ff02954446e9" + }, + "vctrs": { + "Package": "vctrs", + "Version": "0.6.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "glue", + "lifecycle", + "rlang" + ], + "Hash": "c03fa420630029418f7e6da3667aac4a" + }, + "viridisLite": { + "Package": "viridisLite", + "Version": "0.4.2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "c826c7c4241b6fc89ff55aaea3fa7491" + }, + "vroom": { + "Package": "vroom", + "Version": "1.6.5", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "bit64", + "cli", + "cpp11", + "crayon", + "glue", + "hms", + "lifecycle", + "methods", + "progress", + "rlang", + "stats", + "tibble", + "tidyselect", + "tzdb", + "vctrs", + "withr" + ], + "Hash": "390f9315bc0025be03012054103d227c" + }, + "whisker": { + "Package": "whisker", + "Version": "0.4.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "c6abfa47a46d281a7d5159d0a8891e88" + }, + "withr": { + "Package": "withr", + "Version": "3.0.0", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "grDevices", + "graphics" + ], + "Hash": "d31b6c62c10dcf11ec530ca6b0dd5d35" + }, + "wk": { + "Package": "wk", + "Version": "0.9.1", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R" + ], + "Hash": "5d4545e140e36476f35f20d0ca87963e" + }, + "xfun": { + "Package": "xfun", + "Version": "0.43", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "grDevices", + "stats", + "tools" + ], + "Hash": "ab6371d8653ce5f2f9290f4ec7b42a8e" + }, + "xml2": { + "Package": "xml2", + "Version": "1.3.6", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "cli", + "methods", + "rlang" + ], + "Hash": "1d0336142f4cd25d8d23cd3ba7a8fb61" + }, + "yaml": { + "Package": "yaml", + "Version": "2.3.8", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "29240487a071f535f5e5d5a323b7afbd" + }, + "zip": { + "Package": "zip", + "Version": "2.3.1", + "Source": "Repository", + "Repository": "RSPM", + "Hash": "fcc4bd8e6da2d2011eb64a5e5cc685ab" + } + } +} diff --git a/notebooks/renv/.gitignore b/notebooks/renv/.gitignore new file mode 100644 index 000000000..0ec0cbba2 --- /dev/null +++ b/notebooks/renv/.gitignore @@ -0,0 +1,7 @@ +library/ +local/ +cellar/ +lock/ +python/ +sandbox/ +staging/ diff --git a/notebooks/renv/activate.R b/notebooks/renv/activate.R new file mode 100644 index 000000000..d13f9932a --- /dev/null +++ b/notebooks/renv/activate.R @@ -0,0 +1,1220 @@ + +local({ + + # the requested version of renv + version <- "1.0.7" + attr(version, "sha") <- NULL + + # the project directory + project <- Sys.getenv("RENV_PROJECT") + if (!nzchar(project)) + project <- getwd() + + # use start-up diagnostics if enabled + diagnostics <- Sys.getenv("RENV_STARTUP_DIAGNOSTICS", unset = "FALSE") + if (diagnostics) { + start <- Sys.time() + profile <- tempfile("renv-startup-", fileext = ".Rprof") + utils::Rprof(profile) + on.exit({ + utils::Rprof(NULL) + elapsed <- signif(difftime(Sys.time(), start, units = "auto"), digits = 2L) + writeLines(sprintf("- renv took %s to run the autoloader.", format(elapsed))) + writeLines(sprintf("- Profile: %s", profile)) + print(utils::summaryRprof(profile)) + }, add = TRUE) + } + + # figure out whether the autoloader is enabled + enabled <- local({ + + # first, check config option + override <- getOption("renv.config.autoloader.enabled") + if (!is.null(override)) + return(override) + + # if we're being run in a context where R_LIBS is already set, + # don't load -- presumably we're being run as a sub-process and + # the parent process has already set up library paths for us + rcmd <- Sys.getenv("R_CMD", unset = NA) + rlibs <- Sys.getenv("R_LIBS", unset = NA) + if (!is.na(rlibs) && !is.na(rcmd)) + return(FALSE) + + # next, check environment variables + # TODO: prefer using the configuration one in the future + envvars <- c( + "RENV_CONFIG_AUTOLOADER_ENABLED", + "RENV_AUTOLOADER_ENABLED", + "RENV_ACTIVATE_PROJECT" + ) + + for (envvar in envvars) { + envval <- Sys.getenv(envvar, unset = NA) + if (!is.na(envval)) + return(tolower(envval) %in% c("true", "t", "1")) + } + + # enable by default + TRUE + + }) + + # bail if we're not enabled + if (!enabled) { + + # if we're not enabled, we might still need to manually load + # the user profile here + profile <- Sys.getenv("R_PROFILE_USER", unset = "~/.Rprofile") + if (file.exists(profile)) { + cfg <- Sys.getenv("RENV_CONFIG_USER_PROFILE", unset = "TRUE") + if (tolower(cfg) %in% c("true", "t", "1")) + sys.source(profile, envir = globalenv()) + } + + return(FALSE) + + } + + # avoid recursion + if (identical(getOption("renv.autoloader.running"), TRUE)) { + warning("ignoring recursive attempt to run renv autoloader") + return(invisible(TRUE)) + } + + # signal that we're loading renv during R startup + options(renv.autoloader.running = TRUE) + on.exit(options(renv.autoloader.running = NULL), add = TRUE) + + # signal that we've consented to use renv + options(renv.consent = TRUE) + + # load the 'utils' package eagerly -- this ensures that renv shims, which + # mask 'utils' packages, will come first on the search path + library(utils, lib.loc = .Library) + + # unload renv if it's already been loaded + if ("renv" %in% loadedNamespaces()) + unloadNamespace("renv") + + # load bootstrap tools + `%||%` <- function(x, y) { + if (is.null(x)) y else x + } + + catf <- function(fmt, ..., appendLF = TRUE) { + + quiet <- getOption("renv.bootstrap.quiet", default = FALSE) + if (quiet) + return(invisible()) + + msg <- sprintf(fmt, ...) + cat(msg, file = stdout(), sep = if (appendLF) "\n" else "") + + invisible(msg) + + } + + header <- function(label, + ..., + prefix = "#", + suffix = "-", + n = min(getOption("width"), 78)) + { + label <- sprintf(label, ...) + n <- max(n - nchar(label) - nchar(prefix) - 2L, 8L) + if (n <= 0) + return(paste(prefix, label)) + + tail <- paste(rep.int(suffix, n), collapse = "") + paste0(prefix, " ", label, " ", tail) + + } + + heredoc <- function(text, leave = 0) { + + # remove leading, trailing whitespace + trimmed <- gsub("^\\s*\\n|\\n\\s*$", "", text) + + # split into lines + lines <- strsplit(trimmed, "\n", fixed = TRUE)[[1L]] + + # compute common indent + indent <- regexpr("[^[:space:]]", lines) + common <- min(setdiff(indent, -1L)) - leave + paste(substring(lines, common), collapse = "\n") + + } + + startswith <- function(string, prefix) { + substring(string, 1, nchar(prefix)) == prefix + } + + bootstrap <- function(version, library) { + + friendly <- renv_bootstrap_version_friendly(version) + section <- header(sprintf("Bootstrapping renv %s", friendly)) + catf(section) + + # attempt to download renv + catf("- Downloading renv ... ", appendLF = FALSE) + withCallingHandlers( + tarball <- renv_bootstrap_download(version), + error = function(err) { + catf("FAILED") + stop("failed to download:\n", conditionMessage(err)) + } + ) + catf("OK") + on.exit(unlink(tarball), add = TRUE) + + # now attempt to install + catf("- Installing renv ... ", appendLF = FALSE) + withCallingHandlers( + status <- renv_bootstrap_install(version, tarball, library), + error = function(err) { + catf("FAILED") + stop("failed to install:\n", conditionMessage(err)) + } + ) + catf("OK") + + # add empty line to break up bootstrapping from normal output + catf("") + + return(invisible()) + } + + renv_bootstrap_tests_running <- function() { + getOption("renv.tests.running", default = FALSE) + } + + renv_bootstrap_repos <- function() { + + # get CRAN repository + cran <- getOption("renv.repos.cran", "https://cloud.r-project.org") + + # check for repos override + repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA) + if (!is.na(repos)) { + + # check for RSPM; if set, use a fallback repository for renv + rspm <- Sys.getenv("RSPM", unset = NA) + if (identical(rspm, repos)) + repos <- c(RSPM = rspm, CRAN = cran) + + return(repos) + + } + + # check for lockfile repositories + repos <- tryCatch(renv_bootstrap_repos_lockfile(), error = identity) + if (!inherits(repos, "error") && length(repos)) + return(repos) + + # retrieve current repos + repos <- getOption("repos") + + # ensure @CRAN@ entries are resolved + repos[repos == "@CRAN@"] <- cran + + # add in renv.bootstrap.repos if set + default <- c(FALLBACK = "https://cloud.r-project.org") + extra <- getOption("renv.bootstrap.repos", default = default) + repos <- c(repos, extra) + + # remove duplicates that might've snuck in + dupes <- duplicated(repos) | duplicated(names(repos)) + repos[!dupes] + + } + + renv_bootstrap_repos_lockfile <- function() { + + lockpath <- Sys.getenv("RENV_PATHS_LOCKFILE", unset = "renv.lock") + if (!file.exists(lockpath)) + return(NULL) + + lockfile <- tryCatch(renv_json_read(lockpath), error = identity) + if (inherits(lockfile, "error")) { + warning(lockfile) + return(NULL) + } + + repos <- lockfile$R$Repositories + if (length(repos) == 0) + return(NULL) + + keys <- vapply(repos, `[[`, "Name", FUN.VALUE = character(1)) + vals <- vapply(repos, `[[`, "URL", FUN.VALUE = character(1)) + names(vals) <- keys + + return(vals) + + } + + renv_bootstrap_download <- function(version) { + + sha <- attr(version, "sha", exact = TRUE) + + methods <- if (!is.null(sha)) { + + # attempting to bootstrap a development version of renv + c( + function() renv_bootstrap_download_tarball(sha), + function() renv_bootstrap_download_github(sha) + ) + + } else { + + # attempting to bootstrap a release version of renv + c( + function() renv_bootstrap_download_tarball(version), + function() renv_bootstrap_download_cran_latest(version), + function() renv_bootstrap_download_cran_archive(version) + ) + + } + + for (method in methods) { + path <- tryCatch(method(), error = identity) + if (is.character(path) && file.exists(path)) + return(path) + } + + stop("All download methods failed") + + } + + renv_bootstrap_download_impl <- function(url, destfile) { + + mode <- "wb" + + # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715 + fixup <- + Sys.info()[["sysname"]] == "Windows" && + substring(url, 1L, 5L) == "file:" + + if (fixup) + mode <- "w+b" + + args <- list( + url = url, + destfile = destfile, + mode = mode, + quiet = TRUE + ) + + if ("headers" %in% names(formals(utils::download.file))) + args$headers <- renv_bootstrap_download_custom_headers(url) + + do.call(utils::download.file, args) + + } + + renv_bootstrap_download_custom_headers <- function(url) { + + headers <- getOption("renv.download.headers") + if (is.null(headers)) + return(character()) + + if (!is.function(headers)) + stopf("'renv.download.headers' is not a function") + + headers <- headers(url) + if (length(headers) == 0L) + return(character()) + + if (is.list(headers)) + headers <- unlist(headers, recursive = FALSE, use.names = TRUE) + + ok <- + is.character(headers) && + is.character(names(headers)) && + all(nzchar(names(headers))) + + if (!ok) + stop("invocation of 'renv.download.headers' did not return a named character vector") + + headers + + } + + renv_bootstrap_download_cran_latest <- function(version) { + + spec <- renv_bootstrap_download_cran_latest_find(version) + type <- spec$type + repos <- spec$repos + + baseurl <- utils::contrib.url(repos = repos, type = type) + ext <- if (identical(type, "source")) + ".tar.gz" + else if (Sys.info()[["sysname"]] == "Windows") + ".zip" + else + ".tgz" + name <- sprintf("renv_%s%s", version, ext) + url <- paste(baseurl, name, sep = "/") + + destfile <- file.path(tempdir(), name) + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (inherits(status, "condition")) + return(FALSE) + + # report success and return + destfile + + } + + renv_bootstrap_download_cran_latest_find <- function(version) { + + # check whether binaries are supported on this system + binary <- + getOption("renv.bootstrap.binary", default = TRUE) && + !identical(.Platform$pkgType, "source") && + !identical(getOption("pkgType"), "source") && + Sys.info()[["sysname"]] %in% c("Darwin", "Windows") + + types <- c(if (binary) "binary", "source") + + # iterate over types + repositories + for (type in types) { + for (repos in renv_bootstrap_repos()) { + + # retrieve package database + db <- tryCatch( + as.data.frame( + utils::available.packages(type = type, repos = repos), + stringsAsFactors = FALSE + ), + error = identity + ) + + if (inherits(db, "error")) + next + + # check for compatible entry + entry <- db[db$Package %in% "renv" & db$Version %in% version, ] + if (nrow(entry) == 0) + next + + # found it; return spec to caller + spec <- list(entry = entry, type = type, repos = repos) + return(spec) + + } + } + + # if we got here, we failed to find renv + fmt <- "renv %s is not available from your declared package repositories" + stop(sprintf(fmt, version)) + + } + + renv_bootstrap_download_cran_archive <- function(version) { + + name <- sprintf("renv_%s.tar.gz", version) + repos <- renv_bootstrap_repos() + urls <- file.path(repos, "src/contrib/Archive/renv", name) + destfile <- file.path(tempdir(), name) + + for (url in urls) { + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (identical(status, 0L)) + return(destfile) + + } + + return(FALSE) + + } + + renv_bootstrap_download_tarball <- function(version) { + + # if the user has provided the path to a tarball via + # an environment variable, then use it + tarball <- Sys.getenv("RENV_BOOTSTRAP_TARBALL", unset = NA) + if (is.na(tarball)) + return() + + # allow directories + if (dir.exists(tarball)) { + name <- sprintf("renv_%s.tar.gz", version) + tarball <- file.path(tarball, name) + } + + # bail if it doesn't exist + if (!file.exists(tarball)) { + + # let the user know we weren't able to honour their request + fmt <- "- RENV_BOOTSTRAP_TARBALL is set (%s) but does not exist." + msg <- sprintf(fmt, tarball) + warning(msg) + + # bail + return() + + } + + catf("- Using local tarball '%s'.", tarball) + tarball + + } + + renv_bootstrap_download_github <- function(version) { + + enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE") + if (!identical(enabled, "TRUE")) + return(FALSE) + + # prepare download options + pat <- Sys.getenv("GITHUB_PAT") + if (nzchar(Sys.which("curl")) && nzchar(pat)) { + fmt <- "--location --fail --header \"Authorization: token %s\"" + extra <- sprintf(fmt, pat) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "curl", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } else if (nzchar(Sys.which("wget")) && nzchar(pat)) { + fmt <- "--header=\"Authorization: token %s\"" + extra <- sprintf(fmt, pat) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "wget", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } + + url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version) + name <- sprintf("renv_%s.tar.gz", version) + destfile <- file.path(tempdir(), name) + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (!identical(status, 0L)) + return(FALSE) + + renv_bootstrap_download_augment(destfile) + + return(destfile) + + } + + # Add Sha to DESCRIPTION. This is stop gap until #890, after which we + # can use renv::install() to fully capture metadata. + renv_bootstrap_download_augment <- function(destfile) { + sha <- renv_bootstrap_git_extract_sha1_tar(destfile) + if (is.null(sha)) { + return() + } + + # Untar + tempdir <- tempfile("renv-github-") + on.exit(unlink(tempdir, recursive = TRUE), add = TRUE) + untar(destfile, exdir = tempdir) + pkgdir <- dir(tempdir, full.names = TRUE)[[1]] + + # Modify description + desc_path <- file.path(pkgdir, "DESCRIPTION") + desc_lines <- readLines(desc_path) + remotes_fields <- c( + "RemoteType: github", + "RemoteHost: api.github.com", + "RemoteRepo: renv", + "RemoteUsername: rstudio", + "RemotePkgRef: rstudio/renv", + paste("RemoteRef: ", sha), + paste("RemoteSha: ", sha) + ) + writeLines(c(desc_lines[desc_lines != ""], remotes_fields), con = desc_path) + + # Re-tar + local({ + old <- setwd(tempdir) + on.exit(setwd(old), add = TRUE) + + tar(destfile, compression = "gzip") + }) + invisible() + } + + # Extract the commit hash from a git archive. Git archives include the SHA1 + # hash as the comment field of the tarball pax extended header + # (see https://www.kernel.org/pub/software/scm/git/docs/git-archive.html) + # For GitHub archives this should be the first header after the default one + # (512 byte) header. + renv_bootstrap_git_extract_sha1_tar <- function(bundle) { + + # open the bundle for reading + # We use gzcon for everything because (from ?gzcon) + # > Reading from a connection which does not supply a 'gzip' magic + # > header is equivalent to reading from the original connection + conn <- gzcon(file(bundle, open = "rb", raw = TRUE)) + on.exit(close(conn)) + + # The default pax header is 512 bytes long and the first pax extended header + # with the comment should be 51 bytes long + # `52 comment=` (11 chars) + 40 byte SHA1 hash + len <- 0x200 + 0x33 + res <- rawToChar(readBin(conn, "raw", n = len)[0x201:len]) + + if (grepl("^52 comment=", res)) { + sub("52 comment=", "", res) + } else { + NULL + } + } + + renv_bootstrap_install <- function(version, tarball, library) { + + # attempt to install it into project library + dir.create(library, showWarnings = FALSE, recursive = TRUE) + output <- renv_bootstrap_install_impl(library, tarball) + + # check for successful install + status <- attr(output, "status") + if (is.null(status) || identical(status, 0L)) + return(status) + + # an error occurred; report it + header <- "installation of renv failed" + lines <- paste(rep.int("=", nchar(header)), collapse = "") + text <- paste(c(header, lines, output), collapse = "\n") + stop(text) + + } + + renv_bootstrap_install_impl <- function(library, tarball) { + + # invoke using system2 so we can capture and report output + bin <- R.home("bin") + exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R" + R <- file.path(bin, exe) + + args <- c( + "--vanilla", "CMD", "INSTALL", "--no-multiarch", + "-l", shQuote(path.expand(library)), + shQuote(path.expand(tarball)) + ) + + system2(R, args, stdout = TRUE, stderr = TRUE) + + } + + renv_bootstrap_platform_prefix <- function() { + + # construct version prefix + version <- paste(R.version$major, R.version$minor, sep = ".") + prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-") + + # include SVN revision for development versions of R + # (to avoid sharing platform-specific artefacts with released versions of R) + devel <- + identical(R.version[["status"]], "Under development (unstable)") || + identical(R.version[["nickname"]], "Unsuffered Consequences") + + if (devel) + prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r") + + # build list of path components + components <- c(prefix, R.version$platform) + + # include prefix if provided by user + prefix <- renv_bootstrap_platform_prefix_impl() + if (!is.na(prefix) && nzchar(prefix)) + components <- c(prefix, components) + + # build prefix + paste(components, collapse = "/") + + } + + renv_bootstrap_platform_prefix_impl <- function() { + + # if an explicit prefix has been supplied, use it + prefix <- Sys.getenv("RENV_PATHS_PREFIX", unset = NA) + if (!is.na(prefix)) + return(prefix) + + # if the user has requested an automatic prefix, generate it + auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA) + if (is.na(auto) && getRversion() >= "4.4.0") + auto <- "TRUE" + + if (auto %in% c("TRUE", "True", "true", "1")) + return(renv_bootstrap_platform_prefix_auto()) + + # empty string on failure + "" + + } + + renv_bootstrap_platform_prefix_auto <- function() { + + prefix <- tryCatch(renv_bootstrap_platform_os(), error = identity) + if (inherits(prefix, "error") || prefix %in% "unknown") { + + msg <- paste( + "failed to infer current operating system", + "please file a bug report at https://github.com/rstudio/renv/issues", + sep = "; " + ) + + warning(msg) + + } + + prefix + + } + + renv_bootstrap_platform_os <- function() { + + sysinfo <- Sys.info() + sysname <- sysinfo[["sysname"]] + + # handle Windows + macOS up front + if (sysname == "Windows") + return("windows") + else if (sysname == "Darwin") + return("macos") + + # check for os-release files + for (file in c("/etc/os-release", "/usr/lib/os-release")) + if (file.exists(file)) + return(renv_bootstrap_platform_os_via_os_release(file, sysinfo)) + + # check for redhat-release files + if (file.exists("/etc/redhat-release")) + return(renv_bootstrap_platform_os_via_redhat_release()) + + "unknown" + + } + + renv_bootstrap_platform_os_via_os_release <- function(file, sysinfo) { + + # read /etc/os-release + release <- utils::read.table( + file = file, + sep = "=", + quote = c("\"", "'"), + col.names = c("Key", "Value"), + comment.char = "#", + stringsAsFactors = FALSE + ) + + vars <- as.list(release$Value) + names(vars) <- release$Key + + # get os name + os <- tolower(sysinfo[["sysname"]]) + + # read id + id <- "unknown" + for (field in c("ID", "ID_LIKE")) { + if (field %in% names(vars) && nzchar(vars[[field]])) { + id <- vars[[field]] + break + } + } + + # read version + version <- "unknown" + for (field in c("UBUNTU_CODENAME", "VERSION_CODENAME", "VERSION_ID", "BUILD_ID")) { + if (field %in% names(vars) && nzchar(vars[[field]])) { + version <- vars[[field]] + break + } + } + + # join together + paste(c(os, id, version), collapse = "-") + + } + + renv_bootstrap_platform_os_via_redhat_release <- function() { + + # read /etc/redhat-release + contents <- readLines("/etc/redhat-release", warn = FALSE) + + # infer id + id <- if (grepl("centos", contents, ignore.case = TRUE)) + "centos" + else if (grepl("redhat", contents, ignore.case = TRUE)) + "redhat" + else + "unknown" + + # try to find a version component (very hacky) + version <- "unknown" + + parts <- strsplit(contents, "[[:space:]]")[[1L]] + for (part in parts) { + + nv <- tryCatch(numeric_version(part), error = identity) + if (inherits(nv, "error")) + next + + version <- nv[1, 1] + break + + } + + paste(c("linux", id, version), collapse = "-") + + } + + renv_bootstrap_library_root_name <- function(project) { + + # use project name as-is if requested + asis <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT_ASIS", unset = "FALSE") + if (asis) + return(basename(project)) + + # otherwise, disambiguate based on project's path + id <- substring(renv_bootstrap_hash_text(project), 1L, 8L) + paste(basename(project), id, sep = "-") + + } + + renv_bootstrap_library_root <- function(project) { + + prefix <- renv_bootstrap_profile_prefix() + + path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA) + if (!is.na(path)) + return(paste(c(path, prefix), collapse = "/")) + + path <- renv_bootstrap_library_root_impl(project) + if (!is.null(path)) { + name <- renv_bootstrap_library_root_name(project) + return(paste(c(path, prefix, name), collapse = "/")) + } + + renv_bootstrap_paths_renv("library", project = project) + + } + + renv_bootstrap_library_root_impl <- function(project) { + + root <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA) + if (!is.na(root)) + return(root) + + type <- renv_bootstrap_project_type(project) + if (identical(type, "package")) { + userdir <- renv_bootstrap_user_dir() + return(file.path(userdir, "library")) + } + + } + + renv_bootstrap_validate_version <- function(version, description = NULL) { + + # resolve description file + # + # avoid passing lib.loc to `packageDescription()` below, since R will + # use the loaded version of the package by default anyhow. note that + # this function should only be called after 'renv' is loaded + # https://github.com/rstudio/renv/issues/1625 + description <- description %||% packageDescription("renv") + + # check whether requested version 'version' matches loaded version of renv + sha <- attr(version, "sha", exact = TRUE) + valid <- if (!is.null(sha)) + renv_bootstrap_validate_version_dev(sha, description) + else + renv_bootstrap_validate_version_release(version, description) + + if (valid) + return(TRUE) + + # the loaded version of renv doesn't match the requested version; + # give the user instructions on how to proceed + dev <- identical(description[["RemoteType"]], "github") + remote <- if (dev) + paste("rstudio/renv", description[["RemoteSha"]], sep = "@") + else + paste("renv", description[["Version"]], sep = "@") + + # display both loaded version + sha if available + friendly <- renv_bootstrap_version_friendly( + version = description[["Version"]], + sha = if (dev) description[["RemoteSha"]] + ) + + fmt <- heredoc(" + renv %1$s was loaded from project library, but this project is configured to use renv %2$s. + - Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile. + - Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library. + ") + catf(fmt, friendly, renv_bootstrap_version_friendly(version), remote) + + FALSE + + } + + renv_bootstrap_validate_version_dev <- function(version, description) { + expected <- description[["RemoteSha"]] + is.character(expected) && startswith(expected, version) + } + + renv_bootstrap_validate_version_release <- function(version, description) { + expected <- description[["Version"]] + is.character(expected) && identical(expected, version) + } + + renv_bootstrap_hash_text <- function(text) { + + hashfile <- tempfile("renv-hash-") + on.exit(unlink(hashfile), add = TRUE) + + writeLines(text, con = hashfile) + tools::md5sum(hashfile) + + } + + renv_bootstrap_load <- function(project, libpath, version) { + + # try to load renv from the project library + if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) + return(FALSE) + + # warn if the version of renv loaded does not match + renv_bootstrap_validate_version(version) + + # execute renv load hooks, if any + hooks <- getHook("renv::autoload") + for (hook in hooks) + if (is.function(hook)) + tryCatch(hook(), error = warnify) + + # load the project + renv::load(project) + + TRUE + + } + + renv_bootstrap_profile_load <- function(project) { + + # if RENV_PROFILE is already set, just use that + profile <- Sys.getenv("RENV_PROFILE", unset = NA) + if (!is.na(profile) && nzchar(profile)) + return(profile) + + # check for a profile file (nothing to do if it doesn't exist) + path <- renv_bootstrap_paths_renv("profile", profile = FALSE, project = project) + if (!file.exists(path)) + return(NULL) + + # read the profile, and set it if it exists + contents <- readLines(path, warn = FALSE) + if (length(contents) == 0L) + return(NULL) + + # set RENV_PROFILE + profile <- contents[[1L]] + if (!profile %in% c("", "default")) + Sys.setenv(RENV_PROFILE = profile) + + profile + + } + + renv_bootstrap_profile_prefix <- function() { + profile <- renv_bootstrap_profile_get() + if (!is.null(profile)) + return(file.path("profiles", profile, "renv")) + } + + renv_bootstrap_profile_get <- function() { + profile <- Sys.getenv("RENV_PROFILE", unset = "") + renv_bootstrap_profile_normalize(profile) + } + + renv_bootstrap_profile_set <- function(profile) { + profile <- renv_bootstrap_profile_normalize(profile) + if (is.null(profile)) + Sys.unsetenv("RENV_PROFILE") + else + Sys.setenv(RENV_PROFILE = profile) + } + + renv_bootstrap_profile_normalize <- function(profile) { + + if (is.null(profile) || profile %in% c("", "default")) + return(NULL) + + profile + + } + + renv_bootstrap_path_absolute <- function(path) { + + substr(path, 1L, 1L) %in% c("~", "/", "\\") || ( + substr(path, 1L, 1L) %in% c(letters, LETTERS) && + substr(path, 2L, 3L) %in% c(":/", ":\\") + ) + + } + + renv_bootstrap_paths_renv <- function(..., profile = TRUE, project = NULL) { + renv <- Sys.getenv("RENV_PATHS_RENV", unset = "renv") + root <- if (renv_bootstrap_path_absolute(renv)) NULL else project + prefix <- if (profile) renv_bootstrap_profile_prefix() + components <- c(root, renv, prefix, ...) + paste(components, collapse = "/") + } + + renv_bootstrap_project_type <- function(path) { + + descpath <- file.path(path, "DESCRIPTION") + if (!file.exists(descpath)) + return("unknown") + + desc <- tryCatch( + read.dcf(descpath, all = TRUE), + error = identity + ) + + if (inherits(desc, "error")) + return("unknown") + + type <- desc$Type + if (!is.null(type)) + return(tolower(type)) + + package <- desc$Package + if (!is.null(package)) + return("package") + + "unknown" + + } + + renv_bootstrap_user_dir <- function() { + dir <- renv_bootstrap_user_dir_impl() + path.expand(chartr("\\", "/", dir)) + } + + renv_bootstrap_user_dir_impl <- function() { + + # use local override if set + override <- getOption("renv.userdir.override") + if (!is.null(override)) + return(override) + + # use R_user_dir if available + tools <- asNamespace("tools") + if (is.function(tools$R_user_dir)) + return(tools$R_user_dir("renv", "cache")) + + # try using our own backfill for older versions of R + envvars <- c("R_USER_CACHE_DIR", "XDG_CACHE_HOME") + for (envvar in envvars) { + root <- Sys.getenv(envvar, unset = NA) + if (!is.na(root)) + return(file.path(root, "R/renv")) + } + + # use platform-specific default fallbacks + if (Sys.info()[["sysname"]] == "Windows") + file.path(Sys.getenv("LOCALAPPDATA"), "R/cache/R/renv") + else if (Sys.info()[["sysname"]] == "Darwin") + "~/Library/Caches/org.R-project.R/R/renv" + else + "~/.cache/R/renv" + + } + + renv_bootstrap_version_friendly <- function(version, shafmt = NULL, sha = NULL) { + sha <- sha %||% attr(version, "sha", exact = TRUE) + parts <- c(version, sprintf(shafmt %||% " [sha: %s]", substring(sha, 1L, 7L))) + paste(parts, collapse = "") + } + + renv_bootstrap_exec <- function(project, libpath, version) { + if (!renv_bootstrap_load(project, libpath, version)) + renv_bootstrap_run(version, libpath) + } + + renv_bootstrap_run <- function(version, libpath) { + + # perform bootstrap + bootstrap(version, libpath) + + # exit early if we're just testing bootstrap + if (!is.na(Sys.getenv("RENV_BOOTSTRAP_INSTALL_ONLY", unset = NA))) + return(TRUE) + + # try again to load + if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { + return(renv::load(project = getwd())) + } + + # failed to download or load renv; warn the user + msg <- c( + "Failed to find an renv installation: the project will not be loaded.", + "Use `renv::activate()` to re-initialize the project." + ) + + warning(paste(msg, collapse = "\n"), call. = FALSE) + + } + + renv_json_read <- function(file = NULL, text = NULL) { + + jlerr <- NULL + + # if jsonlite is loaded, use that instead + if ("jsonlite" %in% loadedNamespaces()) { + + json <- tryCatch(renv_json_read_jsonlite(file, text), error = identity) + if (!inherits(json, "error")) + return(json) + + jlerr <- json + + } + + # otherwise, fall back to the default JSON reader + json <- tryCatch(renv_json_read_default(file, text), error = identity) + if (!inherits(json, "error")) + return(json) + + # report an error + if (!is.null(jlerr)) + stop(jlerr) + else + stop(json) + + } + + renv_json_read_jsonlite <- function(file = NULL, text = NULL) { + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") + jsonlite::fromJSON(txt = text, simplifyVector = FALSE) + } + + renv_json_read_default <- function(file = NULL, text = NULL) { + + # find strings in the JSON + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") + pattern <- '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' + locs <- gregexpr(pattern, text, perl = TRUE)[[1]] + + # if any are found, replace them with placeholders + replaced <- text + strings <- character() + replacements <- character() + + if (!identical(c(locs), -1L)) { + + # get the string values + starts <- locs + ends <- locs + attr(locs, "match.length") - 1L + strings <- substring(text, starts, ends) + + # only keep those requiring escaping + strings <- grep("[[\\]{}:]", strings, perl = TRUE, value = TRUE) + + # compute replacements + replacements <- sprintf('"\032%i\032"', seq_along(strings)) + + # replace the strings + mapply(function(string, replacement) { + replaced <<- sub(string, replacement, replaced, fixed = TRUE) + }, strings, replacements) + + } + + # transform the JSON into something the R parser understands + transformed <- replaced + transformed <- gsub("{}", "`names<-`(list(), character())", transformed, fixed = TRUE) + transformed <- gsub("[[{]", "list(", transformed, perl = TRUE) + transformed <- gsub("[]}]", ")", transformed, perl = TRUE) + transformed <- gsub(":", "=", transformed, fixed = TRUE) + text <- paste(transformed, collapse = "\n") + + # parse it + json <- parse(text = text, keep.source = FALSE, srcfile = NULL)[[1L]] + + # construct map between source strings, replaced strings + map <- as.character(parse(text = strings)) + names(map) <- as.character(parse(text = replacements)) + + # convert to list + map <- as.list(map) + + # remap strings in object + remapped <- renv_json_read_remap(json, map) + + # evaluate + eval(remapped, envir = baseenv()) + + } + + renv_json_read_remap <- function(json, map) { + + # fix names + if (!is.null(names(json))) { + lhs <- match(names(json), names(map), nomatch = 0L) + rhs <- match(names(map), names(json), nomatch = 0L) + names(json)[rhs] <- map[lhs] + } + + # fix values + if (is.character(json)) + return(map[[json]] %||% json) + + # handle true, false, null + if (is.name(json)) { + text <- as.character(json) + if (text == "true") + return(TRUE) + else if (text == "false") + return(FALSE) + else if (text == "null") + return(NULL) + } + + # recurse + if (is.recursive(json)) { + for (i in seq_along(json)) { + json[i] <- list(renv_json_read_remap(json[[i]], map)) + } + } + + json + + } + + # load the renv profile, if any + renv_bootstrap_profile_load(project) + + # construct path to library root + root <- renv_bootstrap_library_root(project) + + # construct library prefix for platform + prefix <- renv_bootstrap_platform_prefix() + + # construct full libpath + libpath <- file.path(root, prefix) + + # run bootstrap code + renv_bootstrap_exec(project, libpath, version) + + invisible() + +}) diff --git a/notebooks/renv/settings.json b/notebooks/renv/settings.json new file mode 100644 index 000000000..ffdbb3200 --- /dev/null +++ b/notebooks/renv/settings.json @@ -0,0 +1,19 @@ +{ + "bioconductor.version": null, + "external.libraries": [], + "ignored.packages": [], + "package.dependency.fields": [ + "Imports", + "Depends", + "LinkingTo" + ], + "ppm.enabled": null, + "ppm.ignored.urls": [], + "r.version": null, + "snapshot.type": "implicit", + "use.cache": true, + "vcs.ignore.cellar": true, + "vcs.ignore.library": true, + "vcs.ignore.local": true, + "vcs.manage.ignores": true +} diff --git a/nssp/.pylintrc b/nssp/.pylintrc new file mode 100644 index 000000000..f30837c7e --- /dev/null +++ b/nssp/.pylintrc @@ -0,0 +1,22 @@ + +[MESSAGES CONTROL] + +disable=logging-format-interpolation, + too-many-locals, + too-many-arguments, + # Allow pytest functions to be part of a class. + no-self-use, + # Allow pytest classes to have one test. + too-few-public-methods + +[BASIC] + +# Allow arbitrarily short-named variables. +variable-rgx=[a-z_][a-z0-9_]* +argument-rgx=[a-z_][a-z0-9_]* +attr-rgx=[a-z_][a-z0-9_]* + +[DESIGN] + +# Don't complain about pytest "unused" arguments. +ignored-argument-names=(_.*|run_as_module) \ No newline at end of file diff --git a/nssp/DETAILS.md b/nssp/DETAILS.md new file mode 100644 index 000000000..539697baa --- /dev/null +++ b/nssp/DETAILS.md @@ -0,0 +1,13 @@ +# NSSP data + +We import the NSSP Emergency Department Visit data, including percentage and smoothed percentage of ER visits attributable to a given pathogen, from the CDC website. The data is provided at the county level, state level and national level; we do a population-weighted mean to aggregate from county data up to the HRR and MSA levels. + +## Geographical Levels +* `state`: reported using two-letter postal code +* `county`: reported using fips code +* `national`: just `us` for now +## Metrics +* `percent_visits_covid`, `percent_visits_rsv`, `percent_visits_influenza`: percentage of emergency department patient visits for specified pathogen. +* `percent_visits_combined`: sum of the three percentages of visits for flu, rsv and covid. +* `smoothed_percent_visits_covid`, `smoothed_percent_visits_rsv`, `smoothed_percent_visits_influenza`: 3 week moving average of the percentage of emergency department patient visits for specified pathogen. +* `smoothed_percent_visits_combined`: 3 week moving average of the sum of the three percentages of visits for flu, rsv and covid. \ No newline at end of file diff --git a/nssp/Makefile b/nssp/Makefile new file mode 100644 index 000000000..390113eef --- /dev/null +++ b/nssp/Makefile @@ -0,0 +1,32 @@ +.PHONY = venv, lint, test, clean + +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) +venv: + python3.8 -m venv env + +install: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install -e ../_delphi_utils_python ;\ + pip install -e . + +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + +lint: + . env/bin/activate; pylint $(dir) --rcfile=../pyproject.toml + . env/bin/activate; pydocstyle $(dir) + +format: + . env/bin/activate; darker $(dir) + +test: + . env/bin/activate ;\ + (cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing) + +clean: + rm -rf env + rm -f params.json diff --git a/nssp/README.md b/nssp/README.md new file mode 100644 index 000000000..4bba6f626 --- /dev/null +++ b/nssp/README.md @@ -0,0 +1,75 @@ +# NSSP Emergency Department Visit data + +We import the NSSP Emergency Department Visit data, currently only the smoothed concentration, from the CDC website, aggregate to the state and national level from the wastewater sample site level, and export the aggregated data. +For details see the `DETAILS.md` file in this directory. + +## Create a MyAppToken +`MyAppToken` is required when fetching data from SODA Consumer API +(https://dev.socrata.com/foundry/data.cdc.gov/r8kw-7aab). Follow the +steps below to create a MyAppToken. +- Click the `Sign up for an app token` button in the linked website +- Sign In or Sign Up with Socrata ID +- Click the `Create New App Token` button +- Fill in `Application Name` and `Description` (You can just use delphi_wastewater + for both) and click `Save` +- Copy the `App Token` + + +## Running the Indicator + +The indicator is run by directly executing the Python module contained in this +directory. The safest way to do this is to create a virtual environment, +installed the common DELPHI tools, and then install the module and its +dependencies. To do this, run the following command from this directory: + +``` +make install +``` + +This command will install the package in editable mode, so you can make changes that +will automatically propagate to the installed package. + +All of the user-changable parameters are stored in `params.json`. To execute +the module and produce the output datasets (by default, in `receiving`), run +the following: + +``` +env/bin/python -m delphi_nssp +``` + +If you want to enter the virtual environment in your shell, +you can run `source env/bin/activate`. Run `deactivate` to leave the virtual environment. + +Once you are finished, you can remove the virtual environment and +params file with the following: + +``` +make clean +``` + +## Testing the code + +To run static tests of the code style, run the following command: + +``` +make lint +``` + +Unit tests are also included in the module. To execute these, run the following +command from this directory: + +``` +make test +``` + +To run individual tests, run the following: + +``` +(cd tests && ../env/bin/pytest .py --cov=delphi_NAME --cov-report=term-missing) +``` + +The output will show the number of unit tests that passed and failed, along +with the percentage of code covered by the tests. + +None of the linting or unit tests should fail, and the code lines that are not covered by unit tests should be small and +should not include critical sub-routines. diff --git a/nssp/REVIEW.md b/nssp/REVIEW.md new file mode 100644 index 000000000..03f87b17a --- /dev/null +++ b/nssp/REVIEW.md @@ -0,0 +1,38 @@ +## Code Review (Python) + +A code review of this module should include a careful look at the code and the +output. To assist in the process, but certainly not in replace of it, please +check the following items. + +**Documentation** + +- [ ] the README.md file template is filled out and currently accurate; it is +possible to load and test the code using only the instructions given +- [ ] minimal docstrings (one line describing what the function does) are +included for all functions; full docstrings describing the inputs and expected +outputs should be given for non-trivial functions + +**Structure** + +- [ ] code should pass lint checks (`make lint`) +- [ ] any required metadata files are checked into the repository and placed +within the directory `static` +- [ ] any intermediate files that are created and stored by the module should +be placed in the directory `cache` +- [ ] final expected output files to be uploaded to the API are placed in the +`receiving` directory; output files should not be committed to the respository +- [ ] all options and API keys are passed through the file `params.json` +- [ ] template parameter file (`params.json.template`) is checked into the +code; no personal (i.e., usernames) or private (i.e., API keys) information is +included in this template file + +**Testing** + +- [ ] module can be installed in a new virtual environment (`make install`) +- [ ] reasonably high level of unit test coverage covering all of the main logic +of the code (e.g., missing coverage for raised errors that do not currently seem +possible to reach are okay; missing coverage for options that will be needed are +not) +- [ ] all unit tests run without errors (`make test`) +- [ ] indicator directory has been added to GitHub CI +(`covidcast-indicators/.github/workflows/python-ci.yml`) diff --git a/nssp/cache/.gitignore b/nssp/cache/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/nssp/delphi_nssp/__init__.py b/nssp/delphi_nssp/__init__.py new file mode 100644 index 000000000..827935a53 --- /dev/null +++ b/nssp/delphi_nssp/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""Module to pull and clean indicators from the NSSP source. + +This file defines the functions that are made public by the module. As the +module is intended to be executed though the main method, these are primarily +for testing. +""" + +from __future__ import absolute_import + +from . import pull, run + +__version__ = "0.1.0" diff --git a/nssp/delphi_nssp/__main__.py b/nssp/delphi_nssp/__main__.py new file mode 100644 index 000000000..105f8e2d2 --- /dev/null +++ b/nssp/delphi_nssp/__main__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""Call the function run_module when executed. + +This file indicates that calling the module (`python -m delphi_nssp`) will +call the function `run_module` found within the run.py file. There should be +no need to change this template. +""" + +from delphi_utils import read_params + +from .run import run_module # pragma: no cover + +run_module(read_params()) # pragma: no cover diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py new file mode 100644 index 000000000..0abb68c29 --- /dev/null +++ b/nssp/delphi_nssp/constants.py @@ -0,0 +1,42 @@ +"""Registry for variations.""" + +GEOS = [ + "hrr", + "msa", + "nation", + "state", + "county", +] + +SIGNALS_MAP = { + "percent_visits_covid": "pct_ed_visits_covid", + "percent_visits_influenza": "pct_ed_visits_influenza", + "percent_visits_rsv": "pct_ed_visits_rsv", + "percent_visits_combined": "pct_ed_visits_combined", + "percent_visits_smoothed_covid": "smoothed_pct_ed_visits_covid", + "percent_visits_smoothed_1": "smoothed_pct_ed_visits_influenza", + "percent_visits_smoothed_rsv": "smoothed_pct_ed_visits_rsv", + "percent_visits_smoothed": "smoothed_pct_ed_visits_combined", +} + +SIGNALS = [val for (key, val) in SIGNALS_MAP.items()] +NEWLINE = "\n" + +AUXILIARY_COLS = [ + "se", + "sample_size", + "missing_val", + "missing_se", + "missing_sample_size", +] +CSV_COLS = ["geo_id", "val"] + AUXILIARY_COLS + +TYPE_DICT = {key: float for key in SIGNALS} +TYPE_DICT.update( + { + "timestamp": "datetime64[ns]", + "geography": str, + "county": str, + "fips": int, + } +) diff --git a/nssp/delphi_nssp/pull.py b/nssp/delphi_nssp/pull.py new file mode 100644 index 000000000..5769cca82 --- /dev/null +++ b/nssp/delphi_nssp/pull.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Functions for pulling NSSP ER data.""" + +import textwrap + +import pandas as pd +from sodapy import Socrata + +from .constants import NEWLINE, SIGNALS, SIGNALS_MAP, TYPE_DICT + + +def warn_string(df, type_dict): + """Format the warning string.""" + warn = textwrap.dedent( + f""" + Expected column(s) missed, The dataset schema may + have changed. Please investigate and amend the code. + + Columns needed: + {NEWLINE.join(sorted(type_dict.keys()))} + + Columns available: + {NEWLINE.join(sorted(df.columns))} + """ + ) + + return warn + + +def pull_nssp_data(socrata_token: str): + """Pull the latest NSSP ER visits data, and conforms it into a dataset. + + The output dataset has: + + - Each row corresponds to a single observation + - Each row additionally has columns for the signals in SIGNALS + + Parameters + ---------- + socrata_token: str + My App Token for pulling the NWSS data (could be the same as the nchs data) + test_file: Optional[str] + When not null, name of file from which to read test data + + Returns + ------- + pd.DataFrame + Dataframe as described above. + """ + # Pull data from Socrata API + client = Socrata("data.cdc.gov", socrata_token) + results = [] + offset = 0 + limit = 50000 # maximum limit allowed by SODA 2.0 + while True: + page = client.get("rdmq-nq56", limit=limit, offset=offset) + if not page: + break # exit the loop if no more results + results.extend(page) + offset += limit + df_ervisits = pd.DataFrame.from_records(results) + df_ervisits = df_ervisits.rename(columns={"week_end": "timestamp"}) + df_ervisits = df_ervisits.rename(columns=SIGNALS_MAP) + + try: + df_ervisits = df_ervisits.astype(TYPE_DICT) + except KeyError as exc: + raise ValueError(warn_string(df_ervisits, TYPE_DICT)) from exc + + keep_columns = ["timestamp", "geography", "county", "fips"] + return df_ervisits[SIGNALS + keep_columns] diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py new file mode 100644 index 000000000..7c5a3ffac --- /dev/null +++ b/nssp/delphi_nssp/run.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +"""Functions to call when running the function. + +This module should contain a function called `run_module`, that is executed +when the module is run with `python -m delphi_nssp`. `run_module`'s lone argument should be a +nested dictionary of parameters loaded from the params.json file. We expect the `params` to have +the following structure: + - "common": + - "export_dir": str, directory to write daily output + - "log_filename": (optional) str, path to log file + - "log_exceptions" (optional): bool, whether to log exceptions to file + - "indicator": (optional) + - "wip_signal": (optional) Any[str, bool], list of signals that are + works in progress, or True if all signals in the registry are works + in progress, or False if only unpublished signals are. See + `delphi_utils.add_prefix()` + - "test_file" (optional): str, name of file from which to read test data + - "socrata_token": str, authentication for upstream data pull + - "archive" (optional): if provided, output will be archived with S3 + - "aws_credentials": Dict[str, str], AWS login credentials (see S3 documentation) + - "bucket_name: str, name of S3 bucket to read/write + - "cache_dir": str, directory of locally cached data +""" + +import time +from datetime import datetime + +import numpy as np +import us +from delphi_utils import create_export_csv, get_structured_logger +from delphi_utils.geomap import GeoMapper +from delphi_utils.nancodes import add_default_nancodes + +from .constants import AUXILIARY_COLS, CSV_COLS, GEOS, SIGNALS +from .pull import pull_nssp_data + + +def add_needed_columns(df, col_names=None): + """Short util to add expected columns not found in the dataset.""" + if col_names is None: + col_names = AUXILIARY_COLS + + for col_name in col_names: + df[col_name] = np.nan + df = add_default_nancodes(df) + return df + + +def logging(start_time, run_stats, logger): + """Boilerplate making logs.""" + elapsed_time_in_seconds = round(time.time() - start_time, 2) + min_max_date = run_stats and min(s[0] for s in run_stats) + csv_export_count = sum(s[-1] for s in run_stats) + max_lag_in_days = min_max_date and (datetime.now() - min_max_date).days + formatted_min_max_date = min_max_date and min_max_date.strftime("%Y-%m-%d") + logger.info( + "Completed indicator run", + elapsed_time_in_seconds=elapsed_time_in_seconds, + csv_export_count=csv_export_count, + max_lag_in_days=max_lag_in_days, + oldest_final_export_date=formatted_min_max_date, + ) + + +def run_module(params): + """ + Run the indicator. + + Arguments + -------- + params: Dict[str, Any] + Nested dictionary of parameters. + """ + start_time = time.time() + logger = get_structured_logger( + __name__, + filename=params["common"].get("log_filename"), + log_exceptions=params["common"].get("log_exceptions", True), + ) + export_dir = params["common"]["export_dir"] + socrata_token = params["indicator"]["socrata_token"] + + run_stats = [] + ## build the base version of the signal at the most detailed geo level you can get. + ## compute stuff here or farm out to another function or file + df_pull = pull_nssp_data(socrata_token) + ## aggregate + geo_mapper = GeoMapper() + for signal in SIGNALS: + for geo in GEOS: + df = df_pull.copy() + df["val"] = df[signal] + logger.info("Generating signal and exporting to CSV", metric=signal) + if geo == "nation": + df = df[df["geography"] == "United States"] + df["geo_id"] = "us" + elif geo == "state": + df = df[(df["county"] == "All") & (df["geography"] != "United States")] + df["geo_id"] = df["geography"].apply( + lambda x: us.states.lookup(x).abbr.lower() if us.states.lookup(x) else "dc" + ) + elif geo == "hrr": + df = df[["fips", "val", "timestamp"]] + # fips -> hrr has a weighted version + df = geo_mapper.replace_geocode(df, "fips", "hrr") + df = df.rename(columns={"hrr": "geo_id"}) + elif geo == "msa": + df = df[["fips", "val", "timestamp"]] + # fips -> msa doesn't have a weighted version, so we need to add columns and sum ourselves + df = geo_mapper.add_population_column(df, geocode_type="fips", geocode_col="fips") + df = geo_mapper.add_geocode(df, "fips", "msa", from_col="fips", new_col="geo_id") + df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") + df = df.rename(columns={"weighted_val": "val"}) + else: + df = df[df["county"] != "All"] + df["geo_id"] = df["fips"] + # add se, sample_size, and na codes + missing_cols = set(CSV_COLS) - set(df.columns) + df = add_needed_columns(df, col_names=list(missing_cols)) + df_csv = df[CSV_COLS + ["timestamp"]] + # actual export + dates = create_export_csv( + df_csv, + geo_res=geo, + export_dir=export_dir, + sensor=signal, + weekly_dates=True, + ) + if len(dates) > 0: + run_stats.append((max(dates), len(dates))) + + ## log this indicator run + logging(start_time, run_stats, logger) diff --git a/nssp/params.json.template b/nssp/params.json.template new file mode 100644 index 000000000..df989ede7 --- /dev/null +++ b/nssp/params.json.template @@ -0,0 +1,30 @@ +{ + "common": { + "export_dir": "./receiving", + "log_filename": "./nssp.log", + "log_exceptions": false + }, + "indicator": { + "wip_signal": true, + "static_file_dir": "./static", + "socrata_token": "" + }, + "validation": { + "common": { + "data_source": "nssp", + "api_credentials": "{{ validation_api_key }}", + "span_length": 15, + "min_expected_lag": {"all": "7"}, + "max_expected_lag": {"all": "13"}, + "dry_run": true, + "suppressed_errors": [] + }, + "static": { + "minimum_sample_size": 0, + "missing_se_allowed": true, + "missing_sample_size_allowed": true + }, + "dynamic": {} + } +} + diff --git a/nssp/receiving/.gitignore b/nssp/receiving/.gitignore new file mode 100644 index 000000000..afed0735d --- /dev/null +++ b/nssp/receiving/.gitignore @@ -0,0 +1 @@ +*.csv diff --git a/nssp/setup.py b/nssp/setup.py new file mode 100644 index 000000000..a6cbf640a --- /dev/null +++ b/nssp/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup +from setuptools import find_packages + +required = [ + "numpy", + "pandas", + "pydocstyle", + "pytest", + "pytest-cov", + "pylint==2.8.3", + "delphi-utils", + "sodapy", + "epiweeks", + "freezegun", + "us", +] + +setup( + name="delphi_nssp", + version="0.1.0", + description="Indicators NSSP Emergency Department Visit", + author="Minh Le", + author_email="minhkhul@andrew.cmu.edu", + url="https://github.com/cmu-delphi/covidcast-indicators", + install_requires=required, + classifiers=[ + "Development Status :: 1 - Planning", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.8", + ], + packages=find_packages(), +) diff --git a/nssp/tests/test_data/page.txt b/nssp/tests/test_data/page.txt new file mode 100644 index 000000000..af5f25922 --- /dev/null +++ b/nssp/tests/test_data/page.txt @@ -0,0 +1,65 @@ +[ + { + "week_end": "2022-10-01T00:00:00.000", + "geography": "United States", + "county": "All", + "percent_visits_combined": "2.84", + "percent_visits_covid": "1.84", + "percent_visits_influenza": "0.48", + "percent_visits_rsv": "0.55", + "percent_visits_smoothed": "2.83", + "percent_visits_smoothed_covid": "2.07", + "percent_visits_smoothed_1": "0.34", + "percent_visits_smoothed_rsv": "0.44", + "ed_trends_covid": "Decreasing", + "ed_trends_influenza": "Increasing", + "ed_trends_rsv": "Increasing", + "hsa": "All", + "hsa_counties": "All", + "hsa_nci_id": "All", + "fips": "0", + "trend_source": "United States" + }, + { + "week_end": "2022-10-08T00:00:00.000", + "geography": "United States", + "county": "All", + "percent_visits_combined": "2.93", + "percent_visits_covid": "1.68", + "percent_visits_influenza": "0.68", + "percent_visits_rsv": "0.6", + "percent_visits_smoothed": "2.85", + "percent_visits_smoothed_covid": "1.85", + "percent_visits_smoothed_1": "0.49", + "percent_visits_smoothed_rsv": "0.53", + "ed_trends_covid": "Decreasing", + "ed_trends_influenza": "Increasing", + "ed_trends_rsv": "Increasing", + "hsa": "All", + "hsa_counties": "All", + "hsa_nci_id": "All", + "fips": "0", + "trend_source": "United States" + }, + { + "week_end": "2022-10-15T00:00:00.000", + "geography": "United States", + "county": "All", + "percent_visits_combined": "3.25", + "percent_visits_covid": "1.64", + "percent_visits_influenza": "0.9", + "percent_visits_rsv": "0.74", + "percent_visits_smoothed": "3.01", + "percent_visits_smoothed_covid": "1.72", + "percent_visits_smoothed_1": "0.69", + "percent_visits_smoothed_rsv": "0.63", + "ed_trends_covid": "Decreasing", + "ed_trends_influenza": "Increasing", + "ed_trends_rsv": "Increasing", + "hsa": "All", + "hsa_counties": "All", + "hsa_nci_id": "All", + "fips": "0", + "trend_source": "United States" + } +] diff --git a/nssp/tests/test_pull.py b/nssp/tests/test_pull.py new file mode 100644 index 000000000..cdc85a908 --- /dev/null +++ b/nssp/tests/test_pull.py @@ -0,0 +1,59 @@ +from datetime import datetime, date +import json +import unittest +from unittest.mock import patch, MagicMock +import tempfile +import os +import time +from datetime import datetime +import pdb +import pandas as pd +import pandas.api.types as ptypes + +from delphi_nssp.pull import ( + pull_nssp_data, +) +from delphi_nssp.constants import ( + SIGNALS, + NEWLINE, + SIGNALS_MAP, + TYPE_DICT, +) + + +class TestPullNSSPData(unittest.TestCase): + @patch("delphi_nssp.pull.Socrata") + def test_pull_nssp_data(self, mock_socrata): + # Load test data + with open("test_data/page.txt", "r") as f: + test_data = json.load(f) + + # Mock Socrata client and its get method + mock_client = MagicMock() + mock_client.get.side_effect = [test_data, []] # Return test data on first call, empty list on second call + mock_socrata.return_value = mock_client + + # Call function with test token + test_token = "test_token" + result = pull_nssp_data(test_token) + print(result) + + # Check that Socrata client was initialized with correct arguments + mock_socrata.assert_called_once_with("data.cdc.gov", test_token) + + # Check that get method was called with correct arguments + mock_client.get.assert_any_call("rdmq-nq56", limit=50000, offset=0) + + # Check result + assert result["timestamp"].notnull().all(), "timestamp has rogue NaN" + assert result["geography"].notnull().all(), "geography has rogue NaN" + assert result["county"].notnull().all(), "county has rogue NaN" + assert result["fips"].notnull().all(), "fips has rogue NaN" + + # Check for each signal in SIGNALS + for signal in SIGNALS: + assert result[signal].notnull().all(), f"{signal} has rogue NaN" + + +if __name__ == "__main__": + unittest.main() diff --git a/nssp/tests/test_run.py b/nssp/tests/test_run.py new file mode 100644 index 000000000..72346cff7 --- /dev/null +++ b/nssp/tests/test_run.py @@ -0,0 +1,31 @@ +from datetime import datetime, date +import json +from unittest.mock import patch +import tempfile +import os +import time +from datetime import datetime + +import numpy as np +import pandas as pd +from pandas.testing import assert_frame_equal +from delphi_nssp.constants import GEOS, SIGNALS, CSV_COLS +from delphi_nssp.run import ( + add_needed_columns +) + + +def test_add_needed_columns(): + df = pd.DataFrame({"geo_id": ["us"], "val": [1]}) + df = add_needed_columns(df, col_names=None) + assert df.columns.tolist() == [ + "geo_id", + "val", + "se", + "sample_size", + "missing_val", + "missing_se", + "missing_sample_size", + ] + assert df["se"].isnull().all() + assert df["sample_size"].isnull().all() diff --git a/pyproject.toml b/pyproject.toml index 2ca230476..e194a54b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,10 @@ line-length = 120 target-version = ['py38'] +[tool.ruff] +line-length = 120 +target-version = 'py38' + [tool.darker] revision = 'origin/main...' color = true diff --git a/sir_complainsalot/params.json.template b/sir_complainsalot/params.json.template index b07b197a4..e742c63bc 100644 --- a/sir_complainsalot/params.json.template +++ b/sir_complainsalot/params.json.template @@ -50,6 +50,10 @@ "hhs": { "max_age":15, "maintainers": [] + }, + "nssp": { + "max_age":13, + "maintainers": [] } } }