From fa10fe0463a4e6d033d7db504f1da69a13aa1e30 Mon Sep 17 00:00:00 2001 From: Kaitlyn Johnson <94390107+kaitejohnson@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:12:08 -0400 Subject: [PATCH] update from June codebase (#26) --- .github/actions/install-cmdstan/action.yml | 55 ++ .../scripts/get-latest-release.ps1 | 18 +- .../scripts/get-latest-release.sh | 55 +- .pre-commit-config.yaml | 1 + README.md | 2 +- _targets.R | 11 + _targets_eval.R | 65 +- _targets_eval_postprocessing.R | 332 ++++++++- cfaforecastrenewalww/DESCRIPTION | 4 +- cfaforecastrenewalww/Makefile | 16 + cfaforecastrenewalww/R/get_data.R | 2 + .../R/process_model_outputs.R | 13 +- cfaforecastrenewalww/R/sysdata.rda | Bin 72975 -> 63467 bytes .../inst/extdata/example_params.toml | 8 +- .../inst/stan/renewal_ww_hosp.stan | 4 +- ...newal_ww_hosp_site_level_inf_dynamics.stan | 10 +- .../man/cfaforecastrenewalww-package.Rd | 2 +- cfaforecastrenewalww/man/compile_model.Rd | 2 +- input/params.toml | 8 +- .../command_line_eval_fit_hosp.R | 4 +- .../command_line_eval_fit_ww.R | 4 +- .../command_line_eval_post_process_hosp.R | 21 + pipeline/command_line_eval_post_process_ww.R | 21 + src/setup_eval.R | 19 +- src/write_eval_config.R | 30 +- wweval/DESCRIPTION | 1 + wweval/NAMESPACE | 17 + wweval/R/combine_outputs.R | 14 +- wweval/R/create_hub_submissions.R | 172 +++++ wweval/R/create_mock_submissions.R | 48 +- wweval/R/eval_fit.R | 378 ++-------- wweval/R/eval_post_process.R | 389 ++++++++++ wweval/R/exclude_hosp_outliers.R | 45 ++ wweval/R/filepath_mapping.R | 4 +- wweval/R/format_data_for_stan.R | 29 +- wweval/R/plots.R | 663 +++++++++++++++++- wweval/R/post_processing.R | 7 +- wweval/R/sample_model.R | 28 +- wweval/R/score.R | 210 +++++- wweval/R/wweval-package.R | 5 +- wweval/man/combine_outputs.Rd | 10 +- wweval/man/create_hub_submissions.Rd | 57 ++ wweval/man/create_mock_submission_scores.Rd | 2 +- wweval/man/eval_fit_hosp.Rd | 8 +- wweval/man/eval_fit_ww.Rd | 8 +- wweval/man/eval_post_process_hosp.Rd | 18 + wweval/man/eval_post_process_ww.Rd | 18 + wweval/man/exclude_hosp_outliers.Rd | 35 + wweval/man/format_for_hub.Rd | 42 ++ wweval/man/get_box_plot.Rd | 32 + wweval/man/get_filepath.Rd | 2 +- wweval/man/get_full_scores.Rd | 9 +- wweval/man/get_heatmap_relative_wis.Rd | 35 + wweval/man/get_hosp_data_sizes.Rd | 4 + wweval/man/get_hosp_values.Rd | 2 +- wweval/man/get_model_draws_w_data.Rd | 2 +- wweval/man/get_model_path.Rd | 2 +- wweval/man/get_n_states_improved_plot.Rd | 39 ++ wweval/man/get_plot_hosp_data_comparison.Rd | 2 +- wweval/man/get_plot_hub_performance.Rd | 46 ++ wweval/man/get_plot_quantile_comparison.Rd | 9 +- wweval/man/get_plot_scores_w_data.Rd | 13 +- wweval/man/get_plot_wis_over_time.Rd | 37 + wweval/man/get_plot_ww_data_comparison.Rd | 2 +- wweval/man/get_qq_plot.Rd | 26 + wweval/man/get_scores_from_quantiles.Rd | 11 +- wweval/man/get_stan_data_list.Rd | 6 +- wweval/man/get_ww_values.Rd | 2 +- wweval/man/make_baseline_score_table.Rd | 34 + wweval/man/sample_model.Rd | 4 +- wweval/man/save_table.Rd | 2 +- wweval/man/score_hub_submissions.Rd | 45 ++ 72 files changed, 2752 insertions(+), 529 deletions(-) create mode 100644 .github/actions/install-cmdstan/action.yml create mode 100644 cfaforecastrenewalww/Makefile rename command_line_eval_hosp.R => pipeline/command_line_eval_fit_hosp.R (83%) rename command_line_eval_ww.R => pipeline/command_line_eval_fit_ww.R (83%) create mode 100644 pipeline/command_line_eval_post_process_hosp.R create mode 100644 pipeline/command_line_eval_post_process_ww.R create mode 100644 wweval/R/create_hub_submissions.R create mode 100644 wweval/R/eval_post_process.R create mode 100644 wweval/R/exclude_hosp_outliers.R create mode 100644 wweval/man/create_hub_submissions.Rd create mode 100644 wweval/man/eval_post_process_hosp.Rd create mode 100644 wweval/man/eval_post_process_ww.Rd create mode 100644 wweval/man/exclude_hosp_outliers.Rd create mode 100644 wweval/man/format_for_hub.Rd create mode 100644 wweval/man/get_box_plot.Rd create mode 100644 wweval/man/get_heatmap_relative_wis.Rd create mode 100644 wweval/man/get_n_states_improved_plot.Rd create mode 100644 wweval/man/get_plot_hub_performance.Rd create mode 100644 wweval/man/get_plot_wis_over_time.Rd create mode 100644 wweval/man/get_qq_plot.Rd create mode 100644 wweval/man/make_baseline_score_table.Rd create mode 100644 wweval/man/score_hub_submissions.Rd diff --git a/.github/actions/install-cmdstan/action.yml b/.github/actions/install-cmdstan/action.yml new file mode 100644 index 00000000..a1df87cd --- /dev/null +++ b/.github/actions/install-cmdstan/action.yml @@ -0,0 +1,55 @@ +name: 'Install CmdStan with caching' +description: 'Install CmdStan with caching' +inputs: + cmdstan-version: + description: 'CmdStan version to install (use "latest" for the latest version)' + required: false + default: 'latest' + num-cores: + description: 'Number of cores to use for building CmdStan' + required: false + default: '1' + +runs: + using: 'composite' + steps: + - name: Determine CmdStan Version (Unix) + if: runner.os != 'Windows' && inputs.cmdstan-version == 'latest' + run: | + chmod +x ${{ github.action_path }}/scripts/get-latest-release.sh + ${{ github.action_path }}/scripts/get-latest-release.sh + shell: bash + + - name: Determine CmdStan Version (Windows) + if: runner.os == 'Windows' && inputs.cmdstan-version == 'latest' + run: ${{ github.action_path }}\scripts\get-latest-release.ps1 + shell: pwsh + + - name: Set CmdStan Version (Specified) + if: inputs.cmdstan-version != 'latest' + run: echo "CMDSTAN_VERSION=${{ inputs.cmdstan-version }}" >> $GITHUB_ENV + shell: bash + + - name: Restore Cache + id: cache-cmdstan + uses: actions/cache@v4 + with: + path: '~/.cmdstan/cmdstan-${{ env.CMDSTAN_VERSION }}' + key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }} + + - name: Check the CmdStan toolchain and repair it if required + if: steps.cache-cmdstan.outputs.cache-hit != 'true' + run: | + Rscript -e 'cmdstanr::check_cmdstan_toolchain(fix = TRUE)' + shell: bash + + - name: Install CmdStan using cmdstanr + if: steps.cache-cmdstan.outputs.cache-hit != 'true' + run: | + Rscript -e 'cmdstanr::install_cmdstan(version = "${{ env.CMDSTAN_VERSION }}", cores = ${{ inputs.num-cores }})' + shell: bash + + - name: Set Cmdstan path + run: | + Rscript -e 'cmdstanr::set_cmdstan_path("~/.cmdstan/cmdstan-${{ env.CMDSTAN_VERSION }}")' + shell: bash diff --git a/.github/actions/install-cmdstan/scripts/get-latest-release.ps1 b/.github/actions/install-cmdstan/scripts/get-latest-release.ps1 index 8ce0f9c1..52af26c4 100644 --- a/.github/actions/install-cmdstan/scripts/get-latest-release.ps1 +++ b/.github/actions/install-cmdstan/scripts/get-latest-release.ps1 @@ -1,26 +1,36 @@ -# PowerShell script to fetch the latest CmdStan release version with retry logic +# PowerShell script to fetch the latest CmdStan release version with enhanced retry logic # Initialize retry parameters $max_attempts = 5 $wait_time = 5 # seconds +$version = $null for ($attempt = 1; $attempt -le $max_attempts; $attempt++) { try { - $response = Invoke-RestMethod -Uri "https://api.github.com/repos/stan-dev/cmdstan/releases/latest" -ErrorAction Stop - $version = $response.tag_name -replace '^v', '' + # Using Invoke-RestMethod to fetch the latest release data + $response = Invoke-RestMethod -Uri "https://api.github.com/repos/stan-dev/cmdstan/releases/latest" -Method Get -ErrorAction Stop + $version = $response.tag_name -replace '^v', '' # Remove 'v' from version if present + # Check if the version is successfully retrieved if (-not [string]::IsNullOrWhiteSpace($version)) { "CMDSTAN_VERSION=$version" | Out-File -Append -FilePath $env:GITHUB_ENV Write-Host "CmdStan latest version: $version" break } } catch { + # Handle different types of errors + if ($_.Exception.Response) { + $statusCode = $_.Exception.Response.StatusCode.value__ + Write-Host "HTTP status code: $statusCode" + } + Write-Host "Attempt $attempt of $max_attempts failed. Retrying in $wait_time seconds..." Start-Sleep -Seconds $wait_time - $wait_time = $wait_time * 2 + $wait_time = $wait_time * 2 # Exponential backoff } } +# Check if the version was never set and handle the failure if ([string]::IsNullOrWhiteSpace($version)) { Write-Host "Failed to fetch CmdStan version after $max_attempts attempts." exit 1 diff --git a/.github/actions/install-cmdstan/scripts/get-latest-release.sh b/.github/actions/install-cmdstan/scripts/get-latest-release.sh index 284dc434..819b7fbd 100644 --- a/.github/actions/install-cmdstan/scripts/get-latest-release.sh +++ b/.github/actions/install-cmdstan/scripts/get-latest-release.sh @@ -11,32 +11,35 @@ else exit 1 fi -# Function to get the latest CmdStan version using GitHub API -get_latest_version() { - local retries=3 - local wait_time=5 - local status=0 - local version="" - - for ((i=0; i dplyr::filter(scenario == "status_quo"), + baseline_score_table_dir = eval_config$baseline_score_table_dir, + overwrite_table = eval_config$overwrite_summary_table + ) + ), + tar_target( + name = baseline_score_hosp, + command = make_baseline_score_table( + mock_submission_scores |> dplyr::filter(scenario == "no_wastewater"), + baseline_score_table_dir = eval_config$baseline_score_table_dir, + overwrite_table = eval_config$overwrite_summary_table + ) + ), + ## Plots---------------------------------------------------- tar_target( name = plot_raw_scores, diff --git a/_targets_eval_postprocessing.R b/_targets_eval_postprocessing.R index dd14ae2a..c9e31f52 100644 --- a/_targets_eval_postprocessing.R +++ b/_targets_eval_postprocessing.R @@ -44,6 +44,7 @@ setup_interactive_dev_run <- function() { tar_option_set( packages = c( "cmdstanr", + "rlang", "tibble", "ggplot2", "dplyr", @@ -51,7 +52,9 @@ setup_interactive_dev_run <- function() { "cmdstanr", "tidybayes", "cfaforecastrenewalww", - "data.table" + "data.table", + "ggridges", + "ggdist" ) ) } @@ -120,7 +123,7 @@ combined_targets <- list( scenarios = eval_config$scenario, forecast_dates = eval_config$forecast_date_ww, locations = eval_config$location_ww, - eval_output_subdir = file.path("output", "eval"), + eval_output_subdir = eval_config$output_dir, model_type = "ww" ) ), @@ -131,6 +134,52 @@ combined_targets <- list( scenarios = "no_wastewater", forecast_dates = eval_config$forecast_date_hosp, locations = eval_config$location_hosp, + eval_output_subdir = eval_config$output_dir, + model_type = "hosp" + ) + ), + ## Flags------------------------------------------------------------------ + tar_target( + name = all_flags_ww, + command = combine_outputs( + output_type = "flags", + scenarios = eval_config$scenario, + forecast_dates = eval_config$forecast_dates, + locations = eval_config$location_ww, + eval_output_subdir = eval_config$output_dir, + model_type = "ww" + ) + ), + tar_target( + name = all_flags_hosp, + command = combine_outputs( + output_type = "flags", + scenarios = "no_wastewater", + forecast_dates = eval_config$forecast_date_hosp, + locations = eval_config$location_hosp, + eval_output_subdir = eval_config$output_dir, + model_type = "hosp" + ) + ), + ### Scores from quantiles------------------------------------------------- + tar_target( + name = all_ww_scores_quantiles, + command = combine_outputs( + output_type = "scores_quantiles", + scenarios = eval_config$scenario, + forecast_dates = eval_config$forecast_date_ww, + locations = eval_config$location_ww, + eval_output_subdir = file.path("output", "eval"), + model_type = "ww" + ) + ), + tar_target( + name = all_hosp_scores_quantiles, + command = combine_outputs( + output_type = "scores_quantiles", + scenarios = "no_wastewater", + forecast_dates = eval_config$forecast_date_hosp, + locations = eval_config$location_hosp, eval_output_subdir = file.path("output", "eval"), model_type = "hosp" ) @@ -195,18 +244,36 @@ combined_targets <- list( ) -# Downstream targets------------------------------------------------------- -downstream_targets <- list( +# Head-to-head-scenario targets------------------------------------------------ +head_to_head_scenario_targets <- list( tar_target( name = all_raw_scores, command = data.table::as.data.table( dplyr::bind_rows(all_hosp_scores, all_ww_scores) ) ), + tar_target( + name = all_raw_scores_quantiles, + command = data.table::as.data.table( + dplyr::bind_rows(all_hosp_scores_quantiles, all_ww_scores_quantiles) + ) + ), tar_target( name = all_errors, command = dplyr::bind_rows(all_hosp_errors, all_ww_errors) ), + tar_target( + name = all_flags, + command = dplyr::bind_rows(all_flags_ww, all_flags_hosp) + ), + tar_target( + name = nonconverge_df, + command = all_flags |> distinct( + location, forecast_date, model, scenario + ) |> + dplyr::mutate(convergence = FALSE) # Add a column that indicates did not pass + # convergence. We will use this in the head-to-head comparison to statify by convergence + ), ## Raw scores----------------------------------------- # These are the scores from each scenario and location without buffering @@ -238,6 +305,10 @@ downstream_targets <- list( name = mock_submission_scores, command = create_mock_submission_scores(all_raw_scores) ), + tar_target( + name = mock_submission_scores_quantiles, + command = create_mock_submission_scores(all_raw_scores_quantiles) + ), tar_target( name = summarized_scores, command = scoringutils::summarize_scores(mock_submission_scores, @@ -262,27 +333,32 @@ downstream_targets <- list( name = final_summary_scores, command = scoringutils::summarize_scores(mock_submission_scores, by = c( - "scenario", - "period" + "scenario" ) ) ), ## Plots---------------------------------------------------- tar_target( name = plot_raw_scores, - command = get_plot_raw_scores(all_raw_scores), + command = get_plot_raw_scores(all_raw_scores, + score_metric = "crps" + ), deployment = "main" ), tar_target( name = plot_summarized_raw_scores, - command = get_plot_summarized_scores(grouped_all_raw_scores), + command = get_plot_summarized_scores(grouped_all_raw_scores, + score_metric = "crps" + ), pattern = map(grouped_all_raw_scores), iteration = "list", deployment = "main" ), tar_target( name = plot_summarized_scores, - command = get_plot_summarized_scores(grouped_submission_scores), + command = get_plot_summarized_scores(grouped_submission_scores, + score_metric = "crps" + ), pattern = map(grouped_submission_scores), iteration = "list", deployment = "main" @@ -291,7 +367,9 @@ downstream_targets <- list( name = plot_summarized_scores_w_data, command = get_plot_scores_w_data( grouped_submission_scores, - eval_hosp_data + eval_hosp_data, + figure_file_path = eval_config$figure_dir, + score_metric = "crps" ), pattern = map(grouped_submission_scores), iteration = "list", @@ -300,12 +378,14 @@ downstream_targets <- list( tar_target( name = heatmap_scores, command = get_heatmap_scores( - mock_submission_scores + mock_submission_scores_quantiles ) ), tar_target( name = final_plot, - command = get_plot_final_scores(final_summary_scores), + command = get_plot_final_scores(final_summary_scores, + score_metric = "crps" + ), deployment = "main" ), tar_target( @@ -322,11 +402,27 @@ downstream_targets <- list( name = plot_quantile_comparison, command = get_plot_quantile_comparison( all_hosp_quantiles, - eval_hosp_data + eval_hosp_data, + figure_file_path = eval_config$figure_dir, + days_to_show_forecast = 7 ), pattern = map(all_hosp_quantiles), iteration = "list" ), + tar_target( + name = box_plot_by_date_and_scenario, + command = get_box_plot( + mock_submission_scores, + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = bar_chart_n_improved, + command = get_n_states_improved_plot( + mock_submission_scores, + figure_file_path = eval_config$figure_dir + ) + ), tar_target( name = grouped_ww_quantiles, command = all_ww_quantiles |> @@ -345,11 +441,219 @@ downstream_targets <- list( ) +# Hub targets------------------------------------------------------- +hub_targets <- list( + tar_target( + name = metadata_hub_submissions, + command = create_hub_submissions(all_ww_hosp_quantiles, + all_hosp_model_quantiles, + forecast_dates = seq( + from = lubridate::ymd( + min(eval_config$forecast_date_hosp) + ), + to = lubridate::ymd(max(eval_config$forecast_date_hosp)), + by = "week" + ), + hub_subdir = eval_config$hub_subdir, + model_name = "cfa-wwrenewal" + ) + ), + tar_target( + name = metadata_hosp_hub_submissions, + command = create_hub_submissions(all_hosp_model_quantiles, + all_hosp_model_quantiles, + forecast_dates = seq( + from = lubridate::ymd( + min(eval_config$forecast_date_hosp) + ), + to = lubridate::ymd(max(eval_config$forecast_date_hosp)), + by = "week" + ), + hub_subdir = eval_config$hub_subdir, + model_name = "cfa-hosponlyrenewal" + ) + ), + # Write a function that will get hub scores + all the metadata + # horizon by week, location, forecast_date + eval data alongside it + # for the models specified in the eval config + tar_target( + name = scores_list_retro_hub_submissions, + command = score_hub_submissions( + model_name = c("cfa-wwrenewal", "cfa-hosponlyrenewal"), + hub_subdir = eval_config$hub_subdir, + pull_from_github = FALSE, + dates = seq( + from = lubridate::ymd( + min(eval_config$forecast_date_hosp) + ), + to = lubridate::ymd(max(eval_config$forecast_date_hosp)), + by = "week" + ) + ) + ), + tar_target( + name = scores_list_hub_submission_oct_mar, + command = score_hub_submissions( + model_name = eval_config$hub_model_names, + pull_from_github = TRUE, + dates = seq( + from = lubridate::ymd( + min(eval_config$forecast_date_hosp) + ), + to = lubridate::ymd(max(eval_config$forecast_date_hosp)), + by = "week" + ) + ) + ), + tar_target( + name = combine_scores_oct_mar_raw, + command = dplyr::bind_rows( + scores_list_retro_hub_submissions$log_scale_scores, + scores_list_hub_submission_oct_mar$log_scale_scores + ) + ), + # Rename the model as retrospective + tar_target( + name = combine_scores_oct_mar_full, + command = combine_scores_oct_mar_raw |> dplyr::mutate( + model = dplyr::case_when( + model == "cfa-wwrenewal" ~ "cfa-wwrenewal(retro)", + model == "cfa-hosponlyrenewal" ~ "cfa-hosponlyrenewal(retro)", + TRUE ~ model + ) + ) + ), + # Filter out the states that not every model has estimates for, + # start by doing this manually, can write functions if needed as + # we expand to other models + tar_target( + name = combine_scores_oct_mar, + command = combine_scores_oct_mar_full |> + dplyr::filter(!location_name %in% c( + "Virgin Islands", + "American Samoa", + "United States" + )) + ), + tar_target( + name = save_scores_oct_mar, + command = readr::write_csv( + combine_scores_oct_mar, + file.path(eval_config$score_subdir, "scores_oct_mar.csv") + ) + ), + tar_target( + name = scores_list_cfa_ww_real_time, + command = score_hub_submissions( + model_name = "cfa-wwrenewal", + pull_from_github = TRUE, + dates = seq( + from = lubridate::ymd( + "2024-02-05" + ), + to = lubridate::ymd(max(eval_config$forecast_date_hosp)), + by = "week" + ) + ) + ), + tar_target( + name = cfa_real_time_scores, + command = scores_list_cfa_ww_real_time$log_scale_scores |> + dplyr::mutate( + model = ifelse( + model == "cfa-wwrenewal", "cfa-wwrenewal(real-time)", model + ) + ) + ), + tar_target( + name = combine_scores_feb_mar, + command = dplyr::bind_rows( + cfa_real_time_scores, + combine_scores_oct_mar |> dplyr::filter( + forecast_date >= lubridate::ymd("2024-02-05") + ) + ) + ), + tar_target( + name = save_scores_feb_mar, + command = readr::write_csv( + combine_scores_feb_mar, + file.path(eval_config$score_subdir, "scores_feb_mar.csv") + ) + ) +) +## Hub comparison plots ------------------------------------------------------ +hub_comparison_plots <- list( + tar_target( + name = plot_wis_over_time, + command = get_plot_wis_over_time( + all_scores = combine_scores_oct_mar, + cfa_real_time_scores = cfa_real_time_scores, + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = plot_overall_performance, + command = get_plot_hub_performance( + all_scores = combine_scores_oct_mar, + cfa_real_time_scores = cfa_real_time_scores, + all_time_period = "Oct 2023-Mar 2024", + real_time_period = "Feb 2024-Mar 2024", + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = heatmap_relative_wis_all_time, + command = get_heatmap_relative_wis( + scores = combine_scores_oct_mar, + time_period = "Oct 2023-Mar 2024", + baseline_model = "COVIDhub-baseline", + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = heatmap_relative_wis_ensemble, + command = get_heatmap_relative_wis( + scores = combine_scores_oct_mar, + time_period = "Oct 2023-Mar 2024", + baseline_model = "COVIDhub-4_week_ensemble", + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = heatmap_relative_wis_hosp_only, + command = get_heatmap_relative_wis( + scores = combine_scores_oct_mar, + time_period = "Oct 2023-Mar 2024", + baseline_model = "cfa-hosponlyrenewal(retro)", + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = qq_plot_all_time, + command = get_qq_plot( + scores = combine_scores_oct_mar, + time_period = "Oct 2023-Mar 2024", + figure_file_path = eval_config$figure_dir + ) + ), + tar_target( + name = qq_plot_feb_mar, + command = get_qq_plot( + scores = combine_scores_feb_mar, + time_period = "Feb 2024-Mar 2024", + figure_file_path = eval_config$figure_dir + ) + ) +) + # Run the targets pipeline list( upstream_targets, combined_targets, - downstream_targets + head_to_head_scenario_targets, + hub_targets, + hub_comparison_plots ) diff --git a/cfaforecastrenewalww/DESCRIPTION b/cfaforecastrenewalww/DESCRIPTION index 067b5432..8997cd49 100644 --- a/cfaforecastrenewalww/DESCRIPTION +++ b/cfaforecastrenewalww/DESCRIPTION @@ -46,7 +46,7 @@ Imports: here, yaml, tidybayes, - cmdstanr (>= 0.7.1), + cmdstanr (>= 0.8.0), zoo, lubridate, ggplot2, @@ -81,7 +81,7 @@ Suggests: knitr, withr, testthat -SystemRequirements: CmdStan (>=2.34.1) +SystemRequirements: CmdStan (>=2.35.0) RoxygenNote: 7.3.1 Encoding: UTF-8 LazyData: true diff --git a/cfaforecastrenewalww/Makefile b/cfaforecastrenewalww/Makefile new file mode 100644 index 00000000..923deeda --- /dev/null +++ b/cfaforecastrenewalww/Makefile @@ -0,0 +1,16 @@ +PKG_VERSION=$(shell Rscript --vanilla -e 'l<-readLines("DESCRIPTION");cat(gsub(".+:\\s+", "", l[grepl("^Version", l)]))') + +docs: + Rscript --vanilla -e 'roxygen2::roxygenize()' + +build: + cd .. && \ + R CMD build cfaforecastrenewalww + +check: + cd .. && \ + R CMD check cfaforecastrenewalww_$(PKG_VERSION).tar.gz + +install: build + cd .. && \ + R CMD INSTALL --preclean cfaforecastrenewalww_$(PKG_VERSION).tar.gz diff --git a/cfaforecastrenewalww/R/get_data.R b/cfaforecastrenewalww/R/get_data.R index 20b99755..cd257a8b 100644 --- a/cfaforecastrenewalww/R/get_data.R +++ b/cfaforecastrenewalww/R/get_data.R @@ -473,6 +473,8 @@ get_stan_data_site_level_model <- function(train_data, # duration shedding autoreg_rt_a = autoreg_rt_a, autoreg_rt_b = autoreg_rt_b, + autoreg_rt_site_a = autoreg_rt_site_a, + autoreg_rt_site_b = autoreg_rt_site_b, autoreg_p_hosp_a = autoreg_p_hosp_a, autoreg_p_hosp_b = autoreg_p_hosp_b, inv_sqrt_phi_prior_mean = inv_sqrt_phi_prior_mean, diff --git a/cfaforecastrenewalww/R/process_model_outputs.R b/cfaforecastrenewalww/R/process_model_outputs.R index 5e936137..b0edc060 100644 --- a/cfaforecastrenewalww/R/process_model_outputs.R +++ b/cfaforecastrenewalww/R/process_model_outputs.R @@ -1076,11 +1076,22 @@ get_raw_param_draws <- function(stan_output_draws, model_type, mutate(value = plogis(p_hosp_mean)) %>% select(name, t, value, draw) + autoreg_rt_site <- stan_output_draws %>% + spread_draws(autoreg_rt_site) %>% + sample_draws(ndraws = n_draws) %>% + mutate(draw = `.draw`) %>% + mutate( + name = "autoreg_rt_site", + t = NA + ) %>% + rename(value = autoreg_rt_site) %>% + select(name, t, value, draw) + posterior_params <- rbind( phi_h, eta_sd, log10_g, i0, infection_feedback, autoreg_rt, initial_growth, p_hosp_sd, - p_hosp_mean + p_hosp_mean, autoreg_rt_site ) } diff --git a/cfaforecastrenewalww/R/sysdata.rda b/cfaforecastrenewalww/R/sysdata.rda index 8965de7c01ea7e3b4213c181da877ca07d375c92..5fc4a3bf7dca789c7c52e934fe8f015ed0591d1f 100644 GIT binary patch literal 63467 zcmeFZby$_#);GN9mXcOM8U&=fJ4L!vx=WBo6r?)@Bt+>3S#)y9~Q-D~{DZ;Ux_6xX%lV-wS%RnyU{cj1PM+rRy9Wrcp( z_wa-nLSfM+CZU~Ft{O`@2O$9gf?W6&b^rv66a=!jP{IT8!mbd3fAFtBpvN6CApsx+ zClClX00hzofiOyu13>eDf-wlr7zCmM?tnmW#2`@l-38NK3E|y0KCGGF!vBvGZ!u)6 zf1HC5e316tR9j^RgD{$C_3t-L%j;}D5f$x7St-e45Ssm8Ld%!g<1O)!Q1-MvSi*tu~jN@kufA zQ5FJ~gy_U1YcT3)UuX!YJ-%WJDJLiAC>F>I2TSc%UPDgpP>eX$tQ|R(lhkZNF*pE0 zS^@|8y#y6Lq7?`jgmpY_y!0E|NKPwL>)c3IL@Qz?VzJx+U|Q(06ryTs$77r?B-^+I zmDo5ZsQZH7)247L)8>7m&C72SqMhu1JC`Q6mXS&uS&*A5gegjjKcOT&_6Uj8D<$_e zDOV_<5>{iI%2os)jF z5^_Sy(rS}JiYc6&LWQsL*16^V3YJquk8R+pu{f=7atkg(JZ17H`pMKLy;8ZedOIIw zk;(nwJjSPh>}6k=F!pDDVF5eyF?GJ%U+T2{97pt_G^8~$HAy;~h02*={CQ;TP@2la z4{t4FN-w2TpNhS!l3vEicEGUTQ^hS2`z#jci-n(NY}6@Y zpBh*%UHzT~y)EPUyh&}tRwQPVmtBAU5}7rtcYel z=O8`j_%zcRjj}DO^14KnoJyRNy6MFp@kl$d${FjU<(E+?bUmI`RgJHB9KM`}L)RVZ z^HW;Cm|9L+BjSX3L`%itq$E$t#ufyfU~s9T)$e`@;8J=sPGT-6=BXCYhlN^@#8hq` z`$-A2O|`eOa-6e*ho@~8A&Y=XM>`?5WBM@Ks6)8>^W2VmN%0)mc_p?z&7qHAc@yKx zZD{%|Hnc4k0(h^&~pE@t&7JVuY9WNA}H~P5D99CEfZ7r+bcV`e0j};Oz<%dO!fESsz z2-iE1K_NQEI%Y%e%}`?hB! zAdKC8T^tj8WI~N)MvSAL@I3Z8JkDDU9ue8}?y60HUZ|31<^#{+WM*Lxq#UWz-q^@S zu7K4J%Zj&2?VUyDeeai-z6=Dv^0`=btECQ+2+5X=<%B|4`uhgD=f_fBI#g+$MZF25 z{!r=kt$m82w?W)ac5SK7m3_coKNqr}TjMC}#w#h;GMX^&;2;^>a3;kg_gP%YoraBy z;Tcm*{D@(3w&c^fgw7C-GmG~wO6?SQ*w3OIqn~<-9NtsT|0s`LUOqomM@d0J6%CpZ zDKHXqabiuxl~`+Ab#BM3!XNZFKe=#gwEFQ=Sc)=;-i)0Vl{<22sZE9G`4kW36lTOd z8%l(x4|4TFh8Y(c=+iYkfs*!L@AsimkHBjbcB1)(l4iFJLaj@;ixvnXj&s~c9?Xmr`1a{>G%@&Vs$OO%w=-M{UIv!=XmTth91iaG zx65T5TBq2GV=J5K-8(m(a3@Zv)IM};i&hF?8x*T;i|G@WlUKugKHcSLL0;KgL_*k} z^ax7(O`4GHdqHH}E|(LSL2+8`1)q|AMFUokR_^LV+JZ@IBR4m%Ynr!m89&ygOIlr& z5Y^7bb_;~ZM z{T_`Zk|6So`Qp!(-WHb;qds{4(t?rvD4*-R(z6c@j&ks_6{ga*)S+DT&gYQCy_>Ub ze=_$@6bw`a#sJcvNRmV*jq7oNC8$}Uaa*Mn6oJN_NR$*Ik3~K`@8>{$zLxJghoBir z^`V@1vGncfMGHd5Z5ep(YJ{NTDC5lQ0^+~5vq?!h1)p6|-;Y`;GxA8P?`-?x-qOHT>MhLS?W;E&p0k>40K8(13mHaUbwV_&XY=fha}(F zq6~d0WrQT7+S6a~`~YcE7<9hn7EnZmhf)<9GTy_b0(*GC?6mC2Tm5^3`AI+Pr}qun>-@r=qD;iJ-y zO1>*)fy3MKh5ettn8$ii^87-$W0?#6wC`S4J0lpkuIdw$5$j>R?j|B+d!1GO0)bfH zST)AlnSMO`@rXS62*#Hfr9(9nxQhYwR~K@^=ForX%HIKrxvZsRwFv+WO0aal`8FCPX!JMy?K z*eB|>z0u;?X6L$o+H`EYG=+2jXK=Yuf?g(nFS*?Kp231=|DzW>KRs0) zD{1PcXb$mV5zk}1wCqU9tEwD1F-AN^>eNa#nwY)Dup^h)2i#B{TlOzB&HhHttQ7Q; zZuc=z%?WLa`6DmrP$w}SU#O|%&s4Q`4 z!=odWe3$S%Es?FZEO~t6YD6O{zlFd*$WJ*FDx8tASLnwKS37y6#1x5(BZiAOO=(!2 z5;?%0{c%g@`t))0a3Rk5pkTZB(4w}9;&aGDKOz_JuP1xuA;D*8-fS@hM&wwM=kLBS zhj`!;N}|$$o~sMocA~G|kH+Ci?JVh=zDAQ!mUutvZ$$#M%A!ZeZ^#()AYMg0EKh+k?H2e3>88 z-F7ag{G)FyeFo1bzlfQ#XYOxaS9^*+nw!n?8upO+)^uj0;U!suwA{i$IR?VTk))W+ zq4<0sc}F&Nh{a=TLXt&Zj3l2pz+gqrtGf6#W%gV))~Z0;eLgPp_Ui%<@Tw^;uPuHx z>O)!uRLlcP!!6YG>QQEq10;v!_S!vOL3vElg3mX{nieeQ+l@?|d;^)gD% zitr;n@N;W}p9#M^q+kg>C(kFSnBL zTq8-2YLAzQ+rY6NMyo_#jyPFfB$H1`7!$FP5IiRvG5W$tv)!zYn&`B&4n4k{o}qMf zB^xk~ z8QT!ZVNya!+B2xzz0!fan)o*sFW%7|yk}x?2oTXtytZu%maL&Cqnj?1U?-E4HpPO& zMPIA$rIp65CYN#fs^ub3Tvo>ay_wA_hF?da5W2YK(#pmifI4b98u-!vQn{@Ylwfhi zJzST8d@(mUe;jNWO1`K1Sacs!dMu+5Vw6Outt?!1xvb$*JT06Pg_C1)ymj6HLA$3% zEs16KZsfYNeQd58Z9v8NVFsF>=1S)QLwJOl!y<=pU1ymHRYvA^t_f+DB;q}bemE}P z7Oe0U(S-Yye(X5E<-s?{rYT5kEw8JDE zv);=dg0@*0wsWpeFsRt7)(avmRgP40KKi|*Gor|B94X(txK>;-p!w=+^^E>S%%}K| zgHRIgdy(m0HDS+DR0PH&ZA2wQ>_1(_I;PTo*kea`ZlRK}D!1uaJeQz>_O0n5_EKbvzp>2Kk(Q8Vf3jT)1U@v)Ifa{l<+b>hHdo!{>b3Vp?yhC?s&xjjVuZH{3 zK6QL&6;Y`a{RUP)dj!`()PxL$di#C%OGM)lf-pl4y9 zdmaT3m>m0)d>aYz{@|lKq`qGFD93BPOIP;$BF23YDmXszGjhe>Uw7e!cec#u%^-p7s%Odp0PT5{$+wWKkE=qS64T^p)&GX-H%f9iYyNu zuf_xxwzss$Ny(F#-Pcy}dG9O&0^}%ytdVH;P&EgOM8c|jQht86z13YxvaFv+k5!AtNEa@dqHM3?<)*Z8Ps8ZmfFkuyZ(wOKg^MaIK7 z9CW%g?D~4W!BCi0b#fAMg9r7s|k*o-jQzg zUJ4I^{DLl7GOPkWBw)B0T=NevMJ;kJ#u zV~#$VO&&P`(pY5Qgma%*yy?*@g5--dBFIM1gEq1E^X9JjzK!OCqCY1a*49j~ug3DF z8YI4Wwz#sCqJ2H+Oyt5hM-6$Cjb96i?`@|_4X^-FPF@jjznWS&f+T;Y#1Too$HgSOr z|JAgY;sJ~Ho=|7_G#h6naiWE^;hZ=MN?A(F;~L+UFyRiCDQXG(UG+R3;;l5>_n_V7J73uS6XYYSccxKAwfmhKx8Xsd8q8a$ zkqJ@DF4Yg#$8dJS#PCpt7VVyIv#dknoC_R%u9ivA#$5BO3X{v=*DguL=GWex(OP%fmeCAp=h_+&GUDaHJl{QrU3f@^~4So{|z>CDo=p1T~edXMYHPDmv=) zftlBKc-_H?tctglE035CH%_?TUc_r=H%e~Mocd4YV>>*WUB4f>N{^CmP~M2XRi#_( zwwiOXquM2M*T-dMkxio{rEz9^gm4U%ACf+Vdz1OZKrKEL^l5)nr3y*GzjLh>+WB}4 z7Z;Z~Othj=6R8!)aYh>DP{Kb87xNx-iWPUuoSU*?-0mgg;%n5?%3HNDSy#!OrYcr` z^Uxo)M3g^6GB(X5W(mhS440OS-j8t3o>m~#Y__EL#Z8`6w7gBF+M&xId zxSG9?Le7z@ndXEzX7|)#!d<7?K#JtEufE@8?O3VrPnJ2q6+4V}t`jhKHWw)8`l0RE zfG^^;*-KG_pHdU{z(M#X^yZu0Y-%lnz3&SFJG=@Tu6M6eP2%LUY`@Kyq&DYZ!%voI zaZ)`>pI}_k9(X0=;>HmD7QWnNYtQ^lJ($Vc5#fDEYsd0tmZ&2C76q!R4nkq5Dn@rG zQS%B1gLidgf=)xPPBXIxaUt#*U5bCx_hHM-un%=ECZ8X-ndj{spmV6tlux_XDjsrp znBcl%EpUckjK<+~tr83R_*5fBzdn-G!OcK;lC!9K*6bOJrcgSt)R4Mp)Z^{7ghZBb z|CUUulIXb#-)fgnDHk)tB)gDsx5H_TYHuey8E^!C zDC6I1%oMhuETBC6;=~ zV8fz)cmk4x%&W(4>GG7`U#M;&MKq$<&=*ZA`5BPmgX$sigg}8@V##bP6Kx)&+|QWR ztJAG-t?5@#eJY91Y82ySncfd|)t1sLQvXBPUw0>- zJC8jOCJaGT&K!@aNO!1&7q|QTHuGTZ(*hbXJ{_2^YQ8ERWas}z&rxz}tza`MbuM5I zVS8UKk8so?DWiwDq}(*UYG96+YfI`%a#!!-Cu@P@NEsCkF12vH=?(_pC(?OO;!xv- zY`ML{o_jdf2k|3G5iCo3Upqh0zInZ+DSzRe%mu*?RK7<_d!G|~?7<1Y6cZ!9!QKbM z^oQxgbVr%KGzS}v;u~4(#yKjSWfllhY_y?C>{?GZ&*b{~eFjj_nSIEUZpWQI4UGx% zewR3{K<1i1)dvO%t?y3ZTiA$8H$apMi>!dfKy?1 zpF0V0meTZy70N#Qpi`Einv%OCSpi0?c=2XuH%nkH=a z%+fVBaF*AKYN3QgGsHGl=nNxW!12K+0yiI#5mh0Uwe?T9rl{|Pnq;mQT}Tx>*!e-^ z-T8^hv~6V$)iY@8T*-&b56MCdlRO+# zCwci=bcFIu;;o(0nL#cw&b2LgAn6jS;P12a6ZpAPwlYaxpO{W>3R<%MSZ8%4JH=Lh z_?o{}^4Wf2=<1DQ=jQx?f>2YomD#Imm?Y_B+a8b+)sUXaY zFW9D(bZ@?naYiZN%F||8lvm<5(D%6Zd909L1n{NCiYY{DKfcfNHbzw6;Se&rm1$-B)TsZ!W6*3>T^*rjXH4O6F*G6wI0$ zJ_Et~x-6>BW+fFH7%Q(Ds}{v|fHxs0`4X4;6(-g7!`MAZ5AE~(%xBXD|@@e^w%AB{Uj^n|b$QihEi*0;=0c6W+P$1Bqo-;-QW@tEL9>MqZ!Hp2!w1 zDdGF+QqM?^I^*)aP*=n#(bj(*A$gTwyRMVf=WsQZ7D9O6GV{$J7!N-fFe-fRO()9(llaOW%k^BjPMg} z&PQx5VjQP*%Y~sptYK1yr@?)nnS_(4C)SMpponh-5~hQtSNvy1X8nskY~)!zN0Sg$ zLCT!AOqM~Uhw`o%tGBaj9JW!0GK4@Vy&6+!cdyKuoj`$?|)3t=1nUg2>*(0 z?S6S>foiSjp1UWI($cHiFH|`u^hKgr2JPn`0#Vd(J@~C%JBN6>)HoXBrhU9 zcPO-Y);nC-C|_fo?bM||&DnL@u;QJkPpOPSCliD&^tp-1hGWpi?g=0EZVarPwWM%) z1V{uN#AMXh1{q8E-JVTnxW57yr!i}*QRP>Qv%NSroxc3izcF1`DKpF?p~VX|rAkGM zV5d)-3V84!!HIf?tU^)WN%iUWXQtRmcYV0-!Z{2c+^%5yJ@O&@P#4 zKs(z$9O6!-Sh0^k-Y*!bHA|6^kj)wFsP@jWXvQ3P6d~y(f^~0D$n0-lQkfy z%^8bu=g53O=Z8z>a`4cQraS*t9siV!kWBbngL;v1PKm5&&Z^sr_*WNd-_tMknsq%}yd%4Eq2Jc7(YG6gnufn; zbT`~~EO06@oMA-H|J0umB6>o0zUA7xA9qY-loBTV3T0UUuQS_IfV;T01cheQ;VOSc zx+Ak1!uU3zZ%I8TrlWwu>hnoFs$AOU$q;}1Bz*zjDdM*IFqev|%{D(;Cl`{tOuI(+1+Stg-;Ql*$PJlZD{?cP$# z?E8$u^<&v;QW0TL2`nwc;EABj((mvIbuR4y(sS7XCEDV1b2v;9w&r8uo7f z{aSZ1F_jRtV*KolYtinH;vI#N%Fa^EUX>pbA~AI;bF6TXWzcsI>`ZUAzJ86M?UG(< zf^A=AV}|l1LX*6XX+wq7g^hsgM0B-cg8%&*OOtnc z8Cb_JFbY7c%ksFP+?RJX)y8@4aG}F`|o&VV(Ina(${}35kTybvfQyi6NmQkU>Q9ixun*r(IT) zLV7lKC5ZVd2vI>ItG2w*w6UvAm`S$xH`@(Q1Px4}_za0C@Ipa>$xg)l6oDXa^-UG6mIV9RHqRWH=1Ax#_z9hrs=sb z$kfwj?dn(C*W|M`;!$)HKbD6O)KKZ2mioGF8b9dkJGEo^DE`?vxsbtcEVloA7D3b2 zondiy{VFVDxco)3d1Z=d6cdLYV&ABtOC{x3n`9NY93IAg1n<&=AJ9m1W?A24q0Y0b zrXdX)q@S<2Gdt{#Z=&UGXssoK9vbog=*2_{QKu;RN&nF>n8o4YpcGmR>V0)3PqTA{ z)%ja{Wt)|a7FDy9%%Z(Z@^kVlkVG>@O*w))onT@U;1li2;S;);P*6(e*bCliX*hY} z;>$hvN?fV!VWE#z84+F;<7m~svTaG#GRMQYw5^+9hkRIA(_*pPTFh#yYFWaJ12UF53H%KRCh8?$aej<}qc5bi+0!i=LutJn? z8qe_(d%5?v;dwZN=^<7R;~NjAjG8~&X=3l~ne{t%y>#j`i$*S@3OY10%YC+c6V;v>xVAP7dBtvj4?HefV}R2CSfBLXBizUoD0b4wLsGq0dF9 zw8+Yhu|`fJ!FWjSAqlo8*SJx{#V2XTQ@Bu_0G9Lg0oU(rrA$%BJ9yc_r17PzofTh5 zC|nn91)=f{7UC68?wJN%%bIHbWCgz)rDWO-r88lgJT{M-E%uY*wb$e+Y|A14S+~-D zo}}1R;*2MiaXV{ME8K&qYkqKL&nz;%c3l)G8I{S(9tIhx*qspWt**O2DR&uu-&s7l zswuZj+ZUVBn2V_ud1d*2JZk)yZOBF?!C=Yhhns#MMua12xAV*|@>f-e*ILy+(F*Th z?UlZj>599wjZX#dHfo@o8$*6th2LD)Y94u0ax5>xk@t#4U(*34dGNd%3Rk` zM)bCNW%O1uGwWI0RbO=W*M1|B#_Nu=mZ_HF8<~UaQ$Ghw5r2{7Td|9!Gx?&h!a~Me z#@fR7g<<){arEf09h`gLj+>0z?kntFIKlm2Z^R*=HSKwWczy{kvyzf1 z>^`<2oOzNC>;s0_|J~`aU-yRb429wXO8OSM4lH`MLAzuhV0Waf7-wqTHb|`(&q@Qr0DXTP|3^5J|8ZBa0S(S`a`ZIqB|+>P zQuXYISMt0UPYA4?cF7jcng){QR=k6u0>B*=UG(bRhkU`Q9!*;kN)Q{9c}^az!TJAk zHoADI6oG|kaHCc=!0+@wY2G#^<-H$gzn(W>5p_A>`BPGlracwSZ5S%Gt}urIY9SbN@!>jU;q zO&zZ3z@7XOB;B)nA3BtP$$iY-sU8G%Y68ah>v2Q>t7kB5k_$92I^jOI0Wy683=q~& zddIV~rXP&sEAue3OlGcbKf@aK0PbXeA+Vk2cg5n|^PUb)1J7_^wh{bKAI8NH2Z+Ib z2Mlr8NP@V21%B-&DP4VmU;z9s>)=|$mAYZzj=E-GiTAr_{~TRP0BQ*^=d&+=|KPp; zHxnuu;;}FLb}o2RbzZh|YjEANy_Ach2u32Rz$9K20aEx*U%J9>Q7!adC0o9R)Ka0U`bEiJ$?@ zf38zQ=c_CPSPsy27EId_IQh@qn0^4q2f(jbz-qzu{gU3hi=+ixO^7RS`UqOL>m8D^ z1+>{X2=rQJ0PCk|&-HGcfBDj#v8aX>{V^Lrzxq3{Jdexzd*o$;z;tsrHqM#?fVPpn z&-OStc+8G~K@1K6b8)aA;NcrV)?($lZ0Akp znO%DoCJYLL$;fkTJ?=bWvkF|uZmDvD^ZgOc0=X*E-FJ!j1tV^jJR5YgoeRTGDyZ+# z5PAZXj_!evx{``L248a>H(p(oxrs$ScHy`u&H(RDZf&%5EHLf3+Yo0o-ES{CeQ3Z9_!v-|? zpkcrshLGZ>j?Xa6cTT)i9|TEh7y|C4dUz!V+%6kMgatmUbAhl*XxxoGy&dol%oUsq z8XO>84K0{!aILkr|5KwlQ+I=~bjo9$P_R?t(Ez5Xp|JztO@EhE&yG4|#R=jFID;xg zE?t2OE?&^22%h@j{haJ3Wm)iliL@WYOtaJKu2Kf|A+Md z4AN)OmGM6_uc6=k7jC~DX1@NA(d3(4VsR^8yA~GB%ShkjFW--yB=LoKoh3M^>H+$? zxtBP%Ip^jN?1^ecfzc)wY=B#+>)4MS0pf-;8;jVmB z8mBVVW$kdD>~|(Q83p0sN7UVzyv2bJ68HEBL{1_(F#EY7!eR!?;jI7&Qk@GdklthXL1b>}T<7vw`4{!*~+kF~As|Gxj0UI$`0`@*u43Hws-|ATqd z1)%&)AF$W(g?0D!-}Uw%%$5U00C<~R$enA5qiKhD&*xMl>4p`4A@B~uxP$pwLoi4W z<`W>c8UOaF?d{0iIG*LXiZo_YvU8S#i6>pn=ax*quzV}m($u!W;1?8Bnd2b21MgrV zxvTI7B@C;ez6}^`VPJ)k7YOby{Kot%cwfvcTt6ErI03wQ(PB4qk%<;VJYS3lcnL|$|`vk<7@e}!K5o8bETmw*e-W8PO)rbDlGK2}_0aevTD z;z{$y+x*alW+}P(0m-qZ$?T-4G2e@|QsI(cURY)i zK*{6(080M>NUoDz5!fLbMtT5FP5}!2TbvZUeqUI4$AeDe5VKeoLdfDeb3WswhDKJ= zJiXo>5o2}%jQt%vBM_Vfp$>HdFP<^ji)m&TmLM6fh(k=R>kEx77z>+VZtmZbsZsYb zNzz>51&nda|4mQYCc&ilhOQnc%k5cc+KHf|?LSdgwsMK(dg(Y&#CX_5K``@?{PpD? z`KwEjQQ;-mU$O)JfxoYU1K11*Ss4Z*5*W;2HUv6{;SyO9pxJ@nVP`x|nNiL~dZyYB z7;s=%yLWK})M5V>CwGGM{v1|1`oiiC_GF=m!kT-e{0sfZK#Cv21xxUOWP0SURUDpO zWO&`d$S*C16?0o`ch2a7Fo!5a8pB!6C>xok2_yo!b9SsdbP#>~C3F7f<0AbTs`_<~xx7o`n8T^1ZG~2_{@DY+(j` zg3rHj)gNlcf6u{Ia!-GS0Xnzw&bogKGR?Pfzr)(s)PGN%qjA5{Ty=%AL0);g zd4qpVzAx(k;+E%RDH`1F!zn2ngh!vlfaO|H{&L^q)6-`6H>*@XZ*Jj`c+A*IFoV_B zF0D@|_Kx-r{yQ|nfB*=)S+%gI@WkX%)!I3*bZ|#wW=DV>|Cey=k8iyP!w?8aS78S@ zSnh}OXPf>LhUe2y1b>4h0MPxPkR$~O3|e=lkMPUYIK#RF65r|Sraj|-GGsh}vb&`9 zF7^H&fYEaemTq_aT9dn5ESyVAOItjJb#j;Rr!_aiAPD^F9(W$^jvMbF$pKTqyZ~fy z@G|t&XzRN8rrR&{(xc43!}AK|cu8dWL`3{n?~FT0^!f6*kSo`W_9?v@6KUQ!Q+(%q zb0|N-GRf?w}10uVdL5qpJ^B5~(x?C1 zX5eN2%xu9BM+eopWgzs{i&L0qf$^gz;y*l&!(Tae9Pmm5uvqlpP1>5eseK)m0vO8? z{4K$L0U+rDBiO3zybbcM+gyRsRZFhq$mm69^p6ZLwD2&k!ucm;Odg>GKQjD*87%QQ zrq@iJZmVh^e;@Z320Z~T{A1k+1r{RyWQPTB28W)%F<53~`NQ2iJsmLiW5BQws$iV= z@79e#B>$5THPhgP3l=p(?pg+s3;UcOE&p0HDn|ga+z(J*m@RvL`vU_fKkmA2K1A^R zkKh-P22kJz5JK7mi-s`5s{>~ISIO{9Pn`iun*PFtlQ8fp{z1cf_R8^V!0=!LvZw}H zXxM^5jsvzV-umD!2)?^!117-lSd!mGZ!zJsDL zlfi`X78}g)-oq}j0id1ln*m4~{*_*t^~GI!`F;A~?vru(o9a9Sr>p+Sb;DGmz|!O& z%vFq1OQC|tFsxamNOa6 zw$PCjSTa~+0HlHDFi-I>ay0mNso(*=3P93qfAnl$a0v4SuwGzb%VZxBoB^mVB?$zbwz*DDR34-|k(uoHzH}j?~X=E;d7YYl;poN;Vsrx`S^~ue3i-n>k?H z5OI}XO^c~-GOf!nU2v`L6LEBfOEUcs$5?4CZRtMDS=t@bF)7qJEqw4 zf+_w2@vjvXZ6F5p{uP_SD&E!oXXb_pGuXSZ48R#oYGKvkn$CeOh;86ZXvB5?44)3h zzJsvsjlWq#xtp#h)H3~lPugJVW@z+Xuf1^g)xyd^GqCnCVeSJe1mnqH`x>xqxwAVc z|2bj1z`Sgk^Ytw|zfqp{U%7;?iDV5;EJYcos}2|#pnXezhgo%oXR$8s^6{sF?}|4< zkWudx?;N(&zA5@+aYiv0SK-W2wzSvd-GPqeY$rX%SlEqok&NU#*9ulB6%OvN{XGl5 z?ZhIZBbwnYCrM&&s7sF9q>y(x@-(YSbV!;99^x@5=ZzQw**&od?9G^w#r@w40L~hJ zw*7D^o5Iz95RIOFm%2ur9Hkr?5JNc|jb>K^pn&Zy{4&JCKY@)p45*g;do87gK;;it z@1KIJ3t$cU2VnlxAHa>y|KJudBrLy(I!-c<2P0{2_3ovJQvKgcJo6e-f1vF4=f42^ zKdbf@2BVMm)dUWN5PjhFBw(*KP6}Ym{3jQC8@JatHP>`E{?mRlSLtfsEhH~q zavRU55XV+8GbAO5|3cZm^HyF4pW~p@T;U~Kp=JWb!CdjK{x@;1LvH>FYl~lNlqnX> zG{x1C!9!`kHtqueBT3nSSr*J(K)<}aD~D*`FC&eoY4M4#s+xycO%_mo?;K#SU++33 zxdSf#F4_14OOr5iO92?l6{ZZr`u&0gW`-gF$i8^m7k8*QY6iNqyjwN+V=YD16IL|< zR?P$Gtnqhr$pKnhgDo$@?qk^+dO|Zip?BkT@4efJ1Oq!hu;mA1n8Je(za-%90?Jha z1ZTi00^KCAL-NDyZmoxd9pJ+akO#~g{8e;kT3|Qsrt}BTHo>43)p&&J1ns+%>II}RETf7t=BF#;2LcLyBxJ{@ zZ+?%cZ+|>qFvPDZ^uf`((f4bQS7EVuuOrU*_$CQEmG<)(vgu}Cso7+&NV3eg4lkP_ ziXz6NJCCBAeI#Y{q)a z%`&zrBVoz3o0^eem=WVo8N}qYz~!T4y3H}P;22BBm!7=nrolk!n&P>A$oZ`O9+}-O z{dY(n+B=)aXN#gKTH6R;k*~bEo_a5vCoE7ok6$l}7Dqi^^#!-yJn%Z%+5}%*>PJP+ zv|!x~JtixP{~6sm@*<_Hd2nnp_Q%-4!;cpy7*AW&-h?suQ_nIW*33LBDm0VF3kwBr!lO#jI3&B43(~V^{-7INBB6iGZ_|R67e`Fl+A8Sb4w;b?()qG#p z_Ly}dmgn$vW9l6-u_F9K(iIUKs;ol%>5b%t)6Eut$0$#zwb@IT;6+QK9CzAtt(t@< z0@3Ln%Fkba*05FUv1U0TM}S-H?l|+3G1>}rxk%adkCqIHsMamGP2~}?zL83mZ|drt z-L2H;A|FsZ|EQhT#TXoZp5pQdSCFi6Fgv%-u;gC)jm*=5m=!z5Jp#vOJrvJz=|;iq z7f+udntnQd5TS;E8uxJ5mvq2-gVL|}B(4nHuz1=uqsF7Z=M+EUYoUFh66@u6ZJ%=? zV-j8T^GOQXL;vE7YE(zH2{9nD5-xQM{tL<<|0kgW&5CKG#;MoOlB+JWNx z#laVhE|sf?L^JBN%v#~A3XkV0uQa0|IBmbm-$0kbJ^#$$1(BWS{sQ|;eMeV+t$Z0i z-X)fd7Kse( zOd7lDNIw(df>I+d{2Ob2u!5ukr>CaY!n5T9<9wgmqL!w%TuaZ?2t0Fyu1x*PW;MFJ zFiU~CAp4;ox}{(X?b3A5K`aK_l<$sAAHXb(uAz4@ZhWq*r;Z_o@s zf|It@sEKpnSULa7fx2;Sg%E9xCzXGp-r7i{>KI8ghsAp{kqgQ_-{g)=^;Dy0!Yz)C ztW!a7kpRDX;^7Uj3v}}L_=&UNxiBn=r3JR9yau7?2hs)8-Z7UkXUu1mvWa{Fi~n8RPo=Q))e(0EV6$w zOdwk45KKcZs@ZOueau)5Nm@U|-9NDQ>4}2Fy~%rGyu2l}nZ{^=uIl*gIdy{J&nxDm z@1gQD2W;=}1u@$1onPDNH$aNI@S+EU5?9))zLN|n+lb-z7R|$34!wyi8Oy0~Ad0Fs zoKQ5)+vK^HuUdCKv3>gGJ80SK!D$XU7&@e&ov)6r9NS%3qA7tg)DMx>d| zQ>)dj=9MU7<71xW*|sTvn*rV^W)e+#5S?5}!?1DGTc19Iw%_YUKg*Hv=o6{nTMt-x?{!ag5{i|2R zB&Uw&x|k|?nMw~=gDh~L@sWC*4GGhZI{*9*CF(sj&okH|=XmyAY-^l*$~Upf;MT8F z(j5_V>Lfge%&P>d~{LE%zhMk{{0Hm(9-{i zaG-41NG{<7LZV{tH_xd^+?M(cFWyJ@fi?7$f!GW7SKL+$xq4oXQ!=M1}HBr8`jzukoX zFY?|pEU#T@0Hr{&7I!J`THK{bafjkiq`14cP~6?!-Q69E7I!F8+}*h^RQK8YeCO=_ zefPOPZr(g=O_IrElFXXOdNT|7Le`E6)*~o>+L1E?feDO1_n>&x#SxFWwmk+jSN2RP z!yxL}s@KtR>c}o7mxR_W`Tosmc$Iw7ywmuX3e~!)O+%%#b3?sD4ep?4D<2 zC|#4ki*~HTY89ifuf{mZI#vkGH_qbN-ocvvf-y$KQNhqI1|)aMH}9#B zo7a4!MPlJOO%Q`E;w@@!Yhyd9H@z*}^a~j<%#=gtTIlj`t8`|BSLo04q+P}~C5NEu z#v^9pwcA~Dv6OI=PdOAY$D@4FZF9{ghVNrEqStr5_=U3fV9v(2H1Ts_=O`1l zS&6o6^f!jTNDv;s!y96*HDK$H)XXnpOJ8*@-aH;DmTB(Cj(a1PcS7d4{GPU<^u6ZA z!giHwr@{2cY`>=2Fcyw;ImlF7@OT?)|`;9m@Jn+{%1)n?Y@p39;j_=iW{8h9=3#l;JZAvitLF zakDN~hm`D69OF*8N~wBeTZLIvy5#ts-O>@OhO7a}`kVG$0)@}{@2=78u)vFG>el7% zHFFy64VsD0WffC48?37@_nv^vZde7hzM;Dy&bjT`UbWwQeYhS!A?i76HCNQV_xjYjDYnuiHS9AG87HF;A9}?%KWpsO%kPwZuEN1?)f-|q zndsfOUf(h`wn;}n7$n(mY^<2Qw&pZlciJmbb7`}AVRsw6gS7zD-xAeFy{hsvEe(spP3I*HVXAh2%YL~-n`MR_-uN)_`*_TyV-L}xJ8VNusS*H_q?NexI~ zWNL|E4(XBnRxjY)-o=bvDC+bmp~&P~i4kj*d?L=pBS?1tGZw~UxetJs9kr6I-?`9LHO7$FH_hcz8dYYLTuWh$A`Sz!s zvB7pjgN%%Ax@kjWaMkd{>h#dfRV>)hNp4wjxdraPWI|-J2NC?svjZV|fifu9KBVa0 z73iPgbefH-Srg90o^Jc+H_6&8OgrcAYHqxKqfP<+B}F~I=F(B2{S`fHXvqOtBTjFS z_#%#ac)qnGcm3|%^d>Vj`MH_|NNQ!t$rWPU^Aka*;rK6Yg4|LK=Wax%aznBkiLP8& z&5DVGbEfZWV}lPj*YYdbRi>3>LgC@0Ya?c<#5NRMILIXzOxS}Kj+mSk*s-_~4xN4b zf?Nxzae{k3(y2(VWl1)p)NE;UPirESf}=(*AST>dStPN?o|E2I?r|)ji_8e^?Xn4-+^HkMCbU-9NVXW$E-MenvfJiOn|zXt>1oECmR9zjcW%m+7K9^b}ciXEof=-%v%p)d%vHQsG%9pi5D zS)~e>IY{rz>Kl^R5HTS zqNoVAIKSkSMG?A%o(yhGeIb6@(moWRG{QoH)riFVydgw4WD8Zo;+h78Zyy&Zc3E*H zKHN`>_zqFo$g3NoG9vcStUJC77B!j8yt295jkIHD{<9-kkZ1iy4DgLuxByTb6Uob^ z#AAH%piiMR$+8uu2HrcL(FTMQh^~mYUs_5&$*(};z#C{p<|{Lh+#JnH8cvlXdJ!Cd znM=KFkn)KRG@Q^7h(xtGw*3&nu8jEbus^v*S3Fj2u+}%{rLms+`^D4C`vZsIP4~zo zRA^McE5hq~Yv$K!mTG2E%6KcC&(P@+ghH=I4{T}QSOy3Kg=o8+2L&7{zhhXP9o3^7 zbC>dQ&Q6>kW-Ar?EQ+@ec%MVb*q7>Dig~RN73dZKk6>-))bR0O3Bp(uIW0;lhc0`{ zOtWJlV2^6npy_nB{+^)p{TyaF?717Gnj>qH5B@7k5{`2Ckn_vKYX%CZ6rz%Qt>baZ z3URln-Dvc+oh-~ogCL&`v%Y;OGzp|(zlBT7^kEz>iuIOz`8_=YggLtxA9!kG5P4md zVoK-RA`}Crr7K2gWbXJ zv{NS(#o^#g?Rb5z$8J=->=H{5Al;k0&}k$|3A;+Mu$ATs`}n|yw(o{Xzvn)Zx>xLp{sebHFAnK>na0p0omxLRVnWpC$U zzFNst=9M=yo&)Q*Cg8(KG{ub3q4Dp|^bkp9mwk#b%Sh+#l5*p0WwiC)m^llgfJHf8 zN4ATnY)IaP*Lb(i7lIxeI@AK&3L3iZ-3Jo2rxyBDbh_dN%^yV0?3&w?OSVy7!%3Wv zGR&o7W9|f4v0L*T zexOJ<)BB+q_VwohtO{H5Xv*fM&-GhBB!wHfy`fZ~ymw5*wS_XCAm0ope~sq*qK+6b z_Of6`G!nNFJzdNxDAbGXJ~cmTJvYu-*9A6+Pr^l*Ep*|9AnA6c_^jO(j+Wqt`{KaM z(e_<#a&P?cDz-)Vs~A~>Fv@16>2xFfq4o;~ob>ve^yG8x0GGh^!FkQp2yq-Sc@lwe zMD#*@Xa%QSXLc|UQ|$zrteaWv0q#$|lInahdA>p;C|YrHL*gDS@pQq8tfkSVm=b{l z5P_JsRcp@csIF8zVqI_i8*AncB+S)JjQIB52zIeyn~lZZkOrI&+H_28MdaQP=|R{G zf_M>TL_W*>Fpa6@AC{wko5@tGNWDR{@N}WMy=P{IPTDwEg;NZ5hmjb#?XsPr&!%K} z>~=`+|Gh-y_b=u(J)li#ZtMjMGxEah_FDQV?p^d&Z<@$vNJVg~`G0Xxonc*mfxT|K>+m&2qh9i|j+5wYz&9e_ zBIz=7g&6;kaSHRcR90=EobCAiBo{#<;_G4oN9S0xmIxQSz59S!i(d6$-j0fVM^9Ly zsfkUytN!AwHzFT>l8vk#uL;KQDtViZ-W0jq)^e>6cQs8|Rci~1)_lr^9fH}a+DdL< zMisvUHj7-4xjHR8Jta`9MMTcCGU_7n*9*JWr$?ceR?9$+&wwV**U@v&HaD%gL@LR(IT0e2N-=&|NliEViqGU1~!WiC9yC9Clo37{6{&~in7rWjyjDf&s}f(D^)1;s zs|bq&K_$`o&Z85;07~FB-Me?%`z~4UP$De9yy$p`Bftz#BHSU`D#Fzy4C(FZvt^lA zaz@dziRGA$Y=MR;Evr+vzaS^DA)+e0C&?$_YMyMNP0v}YT;}m!DiOTPq+T>-xh4Xg zD>0Fwg87_s_G^0}+Caj%#OH+&0)20A7B_V5rMU@9OAnQcQNl{IgnKlCK+hBHiS78} zxlrnJH&no>`vuS-`0?ApC*%2eI5=^z_Z_cwY}$gg#M&XY2;QX;;wE@>%fG-Y5hvSK z2X(9E>ex|4LDg%t4n^P>QUC*W#D$lnZ!skZCwLdL*@18+ML;Vy@YqGV!#D=oZOzHLp7X`>WBM;FA92bx_D$v z62ejo;ovB?rRkB}2>S^3*_l4+!46sH2>!yYM+;44Z?X?5a^2}Mg%D0L=}!zvhA+W& zj9_`dwXfFYmZh7-GWqr^Q;b9ypYQ0-d3oq(lHTE6PMwvC3bJ0|P0d52^%}lz^xHnD zG2QFR7a42e+svRbL}XNOA{#6)uZy9& zA)-u2S*s7VD~d#sp76tu>$IcJZ$Yyf(dXq3FO7ySDlyyA-W%_pmk-Px;vg0WGK(HZ zbDTQpwGDDDi0^5zSCM%Ikpor-fqey@?}RI*tnKrkzk8axEXqGsXSDTZAS4uu_sdxj z9ex*;fYr)1|2`h{&ddZ!7=uxJ3g(mkv^%280F?Zz?ua437Sj9m{151~ZDijFSg zeNe70!s!m>%B{#%E;QN8ZZ#;Fnv3F7>56@PVQMw`tsKl*Nu9HUSYU!zygu?!bChT` z6pJ~fs=LT6_O5EpnGxdLgEKxm@r98QHp5r*TaND+X-;#u3n~TN;6LMf94^`!v&5=i z?L&PbV>jf?yJpyZ7IKYHTU&oWD>f1~>)q1UA>|;BLMr_^916J+j6~^8f_UFvMF0F3 z-91tQ`cNxT^1W8>TFhElOonAOc(!NR98yx6QGuQQQk!g*gFY|L+({zGHp@HA9BE3} zbhXV%d*m6K!(W`h(Y-KhqMfq;x=nI^x1?i}^C3IJe(r6LZ#iYI*Z$IWL83g{M+F!v z&=&1GRxBD$&dcbmVPWs~7BeAQfSr!2<-?fOP?%#U_by$}mClfl+1nr8C+LLy)CZhP z_eU$>p^2SOC9K;>inI6@$Fna^2v)Y7EY_C+7PNz=W7CCK-RW|p%LUXpE0(5~lpCR` ztv*oA`jz;6y6;o>JXkE|9UUUV7_d} zbpP?bvL?7!G>+A)ci$X~CYTuiJ)dWm;b*<1w;9f-LG$;mPX#oA6JBF@4N54>%rdm1 zn*?Y8;d2aIY|N7l7UBjEsHR|L!ob06`1Fi@(0fU0$u!KuApSB1-5EVN(ztRjhKRuH z@WS(|h9_RFV6!u1xAW+7D(ZkwjUc|9vG5!koBcp2E65)&@A|M7yaMBNkA88cM-*{g8ATwBKg#oYJ+h3 z2)$F3Vj_A1Ug)GXq&ItYtc{oUd}A>H0qe$Ej~c#-u@xl)sM1`WzTmn3R(~`Vr`5-g zeOsMCmUfPs^R@gbwfvv?_=fb=#uW@G5|wy_r#{2XGD^HKV@+5|PHIUaX1}l$0#~e&PX}=VfJeUG^Q_x2;aZOWcq%#Qx*LEpm zMq{+Nx_{G1ge3M{9yh$>boJ$g?kOplMJRIuyz|Lpy94Qx=}Apq@$j8=A0L)4FLS4QObT$iwEGgflBeMh z5rqUaV{9zp(Vh=5_Rdq(w~cW)abAZJ!|bCq+Qpnd$s1H4DdV538!@AVh`1h7cV<^F zL@^VaA$u*t;9JJrm6LouvD^C80KV0akGIG%n=_Pb&_`f!81mwadD@#@xuP1odhjst ziz&P0ZukXk4-~QX=e?ooa?b+Ot*)!C$zTd|E;Ka}EkAzH%an{ehA4U!BwcEmE_e`f zC*^%Vff8S0#a|(&&+xhsUn%^4Dkr_6gosOc@RK`jbqjo3IxnN@M|y?AaSSh!p15GO zL@i3l;2?cWcty0_Bt(iC64)FsL*A^KP%8FXs1qXw5$ci2XIEw^WORY);T-rZtKtN$ zY36L7Hxi7A2z6|znZr8E6NK+yGFYOfv1Go7yNT+?9G>Erk&MN43?MdAGb~?wi=XWt z{8B1Hwtif{VW9B7!ulv4sZKM1TF>fh{$y>@Yn^I*N>R664!+>=!MMe) zLNKSXu!!x=c7e7PyfRmqhA#$@WOyV|WCENrgwhMI5>#TyS&YD^TUq1My6V6gHoc4k z)6>LQc1=wih9AL{%s|CBEA|c+As5$5SqOW-hU4fuON}7d|Kb^$# z{vAo&vn=F9f|t z#`J0gvogt!e~@vN{mq2nQ#DQv>gzX-$&fBCOIoy4h#q+@Fl)J%28tN)y@M%T=;{S= z-DOxxLc)WStfcXx?2AnmKI?AH_HsmEmMsx)Km)0Et#L3Y@X)V2Loq9+*S(5G6i}Pa z&FK|BQe{42gdt7T*=HGb;5rz<*STMm)g7l(iE*GG(!E_ChLYWOBzsEfxg{L>=4gz1 zhK*f^s&WGzL8SatzM9u5FZ$x`3(6YHtpY`0T-=?6-Ic7LpCi5#!0-fL*+YG`Wu7O3 zp7i>wzbzasMeNMTc$4R?HbZx$8lj}+6eoIdTO`dWGctsHY)+Iq?zsgH=X(sh>>Hq& z7xi&#pEQmVb}49zR6-4C7n!j1!9pEl`7(fNsqicFdKpw_%+hvHpfvq*pFBOEuX%H; z*k28vG&EWs$DGtuM}j##ZeQ41s3*-tNazxUF=6FE-z#}9jtat=Qq&ni1nA&pF?3KE;>e!>>VFI+^cR*vcUXb~Ssv*kU#qSUB; zXJv^Jvj#GMwBR$>lxOc;CMC1>YDm+~M4yQTrX4gJ{_S8gX61U-zHGf5oLfvGUd)&w z#Y>-{v5q@_2>(q(3e%m9RZq^ndzfy6+&Fm#-ev)TUf?(=Cf>wJzU{0|=}uC;SPbmf28feESN)(b6u`TW(zwzDh88p| z{WcW#RJP1Ri8gpOBtNFacL(w_x938A0jX`S&^vnWR8{eA;I>FscKVBrRRUIcM6Ms_ zD-OXDCEiK5MXDnQ3(3oloC{QLmn^;JRFIxVY&UI-LD+3#58XD{ooNMmQvRuL zr3SJ*X>CEElem`0_`nv3;#@7xT0EX_*IQu)5h~;6zHl!sI-;_ zz#qR%5<$%!y9?gBCVQTZzmHD}$7>_L3RqaX+rhj~i$dNRJQ1|QRDEU88-(n}gJmQ_ zXxyw)c@keunY$a=*e!J~5#@HUYnKM0GD~u4i3W{M1{Wleti_9!Ff`xI8ioh9W zp(CTvabHImOyJ(*@}JKMY>#IKn-nizYs2|^+&QuN%J8oj`I1l4!@oB!iSaTr1{fkn z$T8UHanLb=@(%d>dyMH5+nS4+k}N^~`_N z8<`IsYV*wH+R*y7M3Nm9R&MG}$a;s?R&+LJHm z8&`TWF2j<(2com*wjcMiYm+47{twIeE2f>f60mP^#I7al$PT$6om}@myPTZUe>(nK z%>NZ(y%#454fCgG;T1`E-ILzQ2JCu17N<+=GZNFwe=2m&<$R9gX2f;~Ze_cDs3Nhv zgXQYrc$dWZ5QeLP<63x@gDEeG{UmR<=YQO}o$Tm5EJ@NzV>Xh&g=KPe$5B|b zd1BMo>QGueva>eb+_G*tNx-t(ZW>TClmDnXus!~7l*)rK zTg{sVNoEg{78HK9%g_4#+vd9+c4X4)brRspZl!SzlB{0h{Ch2bopxe86mj+_7`Ir- zW^GRYb*DWr2l$HO#&j~-F49#pXyX+t*P_SS->d$f|8O~%>)X{s&R{ zW0UO9qo(URXIIE>IU}5JtX{h!&i;ET`(xHeVO*@E-25&7<=@XwBietFG%T6B^w9j# z|DgFBhI|3dm-N8?|E_r7(ap9yjb`^-@2`v9fj5JwN2AANi+z`{?T&G$>e1Poj{Vgc z-^860@?j*U4G@03J&dGb!o-iGEA8LAfo;~A#y5=INxDK_{5`?F7TA0kD6_{7q5*Q> z_YdS=U9aOoZXe{v`5(wl=9=@rEB;=(Sow!?`*#oH{tBQ1$pFc{0B~@Dl}z*ZpBzAb zM1bUT5bY0tI=UgL{!7*P7ef1}L7T1Bqhk*r6i1sn;KBX>bMc7Tqn0J6_;@#qBIL}@xp-)p5< z{nFIG8rlA$`xf;ZA7+*RC(r!<$13N4TfE1E##0E(rJi(nM>MlfnNhFyH6q%mp)73I zE=oqN=%`n;azRPhw!V*ydU7Aq@n0VKKc@eC64!12`mFq$BK}0e^&t9R^u&MS@&^fx z2BLp?zN;Q<@gx1et*At@1nSZ44W`9t+R=f!vB_|GyzX6)9hD`0&>Yi7Utr#Nz6C&w zcc*?+-@XsG=|_c*{0Mi^{{34cjpE6zid{Y@4d0)7;V=0wt`Qx1SOEPR27h(iUuO8( zbItaEoLnS;0o-16F+Ke2g`~;Po@Z(c>nm6qG+$@2N?;wtTIU)KmiURACTZu*+Me)R zGK`fs>rig@=J8Lph~U(~I%rs_T=*tmK5yKy_;n+8zGHM!cDkd<#tF-wxv5FBzm+QU z3O#yEf0r)c5OYAkuI8ZCmc^Cx&%uE|Y5#$6`$L-cH$;oMfjQM!R%VtryMycrfbmCo z@8E$)Ku(&`b^7Ip*_cP;70EnKvZNJ9GQ(rt(p-iwzb(GLB}!(~Jq*^#uGwD~>~E68 zkJ=YGTqTl2$QCni>MJfYFE3#ehJ4Te8&dycIe(dWFf6!xC~=E-E_}SXpw~1SM|R$U zlfkvgezH%$1z4mu;Xb4-W&;MeJwKO2@J-O)mqHIK$WXhKycy7q+4D%(%djKhH0_PCM}Ri!hCFcV zUV5Ao02babj|*=kz#`l2+af!WMH24Yf?5`^8pnan25%M|}dbC&$@uw>w77d<|B z=(ResQI89rZ|iZ11QH*t$HhD#@(|?P!kx`L3HRF~J^Mcw1pc_}_p@H&_>H6sH~6p8 zAJ$V9w_sl{FSx9@Nag{11qVUlN&hF?27k5k{;h1rw}vn2bU&3v^n5sg4D?T$)H^De zL44W3pt=^Bqeg-~P*OWN>k}mtb+KTHFJHuno!rgsh~G8c>({vpol>b<(U6KK?2+Fn z7WHN$X^TMQ<%P-hB!?yeOqmGT^Yo}lmHx$JiPm=sR3DQp*LdkKhDdhjQyF^+rgyTw zQh08L9v#x5IfgFu(a4qKwml=|kqQV!fn)E9Q+n>jI&%x}ROpE~UE#pUAqI_Cw3+}* z>WRq6hbi6LT|kC`p+Dgy*q~F>>_6XU=X^A>%rBOE)`(A`Scfa3kX?Ej=(e!&yn4Ef zZdAcc_DgrtT%tCpH!lH^yKSMGldBqVRm_Nyj^uMe7DP_wIx@wuBD2n3)*9AyURwqX zXt0lb-NC5A9QI2~9M*?-LRT(a$AA~HTc4<{%AZei`gEk9Uk`v(s9k;dP-IDEmr z9JF{a0#fob+mqu2z`1|(pW(j`obNCGA_s)>cv?H+CmC#jm4Wrib|JtC>Yj(gp8u5t z0&2Q*o*W8n90Zu>mdpj%4tp^F5&v*WSO%Oh{q9d!`T4~=0B6a+yPt*rw*V{$=aB2I z_rsx7vk$*A0so!eAW3FrWdvAqVP^eB0O;8dOW?o7{SP3>n_1KShsu5ZQ8kmQTNSfz zz zAUCs;F1Eq)OdFaCf z^pD)%5ShPCAKU&1bKhIxzozhLmhTTe^;_Vf!kpKW&B?G2ZM?|w0qHO40?K@_(y~rM z0ywhnwD>#Dqb0v@#>>7(KX%vh?>I0IJa_ya^Y}~v#tM+dX4e)VARLwWxCiSSKSqxS z)IWCb@0tG+Mwx9cE-ucub2&~}pJeR=ScLhn9UOq*L%jeJ_s|M}lmCyEGXEI_20UpW z{4(DSDED))hf zLYUe?v;3A~x_#LwNQokCZJj1N{&qEM(=G9D{8JpOXKMSL$caF1n!zKh;F(dqmPw3c z)oRk1lk@5+F6vfo%rfZskhsn%mi7v$+aOtU?a$Vq1&eJB(^V~K8ZgMF9 zikqL3R5DP|YvK1eFaA2CF1n<^za8ewpIX|4j^*?BN`uofLefnw0}Zxfq?!3rR#U(zd#B27kdAS^1Vn9`fsoH0GpJL z-2C~%Y*uC_>5?uCr++G7dM>C~yk>(!mdYfYgxR z(+MB8K>;z3FBX4Z|6v4)-A^Y!Kd)fBH~#p6BzgJy`pWN+o$2m;;%weGSF}6-9m=n_ zGyD$j&HwmVReiPX_qczC2q^0RP4d4Op8bd10|9NT<^073Q1KKXkizd5j^jdGGiKX* z=J2^OeI75EpOZ;AlzJFVPs%X_o@0dgXmi_U<|(;YAsx1PbQg&ht=rgCGz|IuLG&;F~|J? z1^L4v%ggg;lIMA)_CW!8uVMJ&&0Zdd9tbZ+s~ZQ_Ykb!$@;|VD+(7_Z`p@nGTP`g& z7M9JZkx9i|7gMsI5n52_aBs7BbGKHhwd!r?fK6lTQ^R$|05* z!k_=`l(xEf^=wt5N<1MO9zn8e1|xy;m{*38a>!VqkqpM`qFe&2k}-)0IWu2iv!bGU z^A{aVfX{bB8hgk~f;^NB|8Oxtk9UZQ`CwpV(uPZs;a~DZ(c=vsvM(q;l8+3YP$)G0 zt5U6pziW_Y30l0GxN;ne0GZ$0D0S7ut64JGx7upx;S(Oo#XnpvqbgYfO=zs6UA zbAA<2DO~t2dhbv{N4NAD%LS#zd_DeGy8kAK()@z}&9knb=h)$K7 z0ynILc@LkhRG6WT^!BwV)uxp#YRPyvwKdm1L?9fZTdBQKIb$v#aOS+uOWhJoM6HKu zEY7;p(CuAK)FjSeD+&mw-MT2^U3_|%Zknokbjc8Aku)KTC~dlFNMe47nl^zx&^k9&lv;n3^jN zh$Q)d5)T+V|7DkttYtz!w+Kw&1=cT|LO^_i$6RZ4+be~ zoq7<9z4e^EhL_&X8oY5!lSx8d$wo{ztO8dzJV`xE@-*3@f@MH43(rdf^XvdnE&r+| zP`3^gVDmv^uTQb_d_4N%kc8B!!sY@|NU0!*Hw(PM_8ZI^Mqn6JXv+)7eU0aD>En*f zXrZK3RqIM1TpbHLmRR8N*NGOD_Eh+C85%oM8vq;Ho^X7QEXyrEka3z>EoDM3qK63l z*pV5ghn)~U@oXd-r(Dm5T6jQ@%j(PZUj-2Y%~|hR4-ct*`8;EQ79c*dB7J_WbxsF+ z4@N=ghMy`Ot}ZR?M5hnNxs`6h7w1-d%0C{9_nM=^vPD*eRVsPV5SrE8s`#^S#XF%D z`dek9P?lmxF=D3k*^U#+J|Ddt;x{KOs+BXm`TS-iijI-i$<_`SF5daGc4~U)<(KoX z11}W9zw2a@_$N!%F#r?WOviu`$7|s&5YWMFKVT-@0%Yv4#&L{5gEy2NA%p9e)18qSTER zl2>Cv-|_Q}tDFKYT7ccZUtY08D=n7~$?t(#!FGLi+bN&vq; zZ3!4niY(}W#yk4lS26@%chzb4*mUU$>z~PL*;Jp!iVd;#@f46LM-Kx*wIOL^3zG)& zcJPN;af{ufPMJDNQ@C;LXq9QpfX1vWkL(6lnPZ+i;oBM6Mz#{?t=C+{d@>pBQ06~3 z(d712q+!TrkDkQ>2APX9{D@dV@?(oSzR0{CON1Ex(wc^lEg1Ds7)S+%mpN*-g}r+v4uVj0p*L5 z>QK~9p31=vkqkINUbdh)S}3z3e*KDIqW*k|HF37TQQKn_2YS4{7=Cq# zX@4OljDLIi6;KpXY$_@6DEKslju>%f96_-jv|!k02xYirl_k^iSVu;EpUvH~1Z)dg zPGx%tx|DFFum<}KO}#r)|J!4iBMT3v(YSKeV2DuR09K;u~#uZ!cY2#xAev@Ag9c79Mi z6&$^G=7wi*ecxI2aSY_RWl8{F9Y0VItkgawqWH8`A)UGmjnt52uo_H=0nxN% zY`PgXpNAEK1&J^~?z2mPL*h_z&;2N2jYYd+()O`NRBJ7ZmD(vs?6xy$+;^ z9mWt1Rds+vSY$)mu4>j)7I!?xcJamfFmIbtR`zBuwH#_`jJ|@s2G~So@qHPjyx8@2 zj-9`TDMw-gjuvgeQ0pEcpL&Ie0zDN z;BZiWju*lbGCyGvgaZfP1fnNT5`K~hi*5VC5G1eC8nJlRARDx&-JxU0Sp;0lx#8BMkko>EQ!9v}w_P2ShUr{)qTJ zYqZel55$SWwMsL_d?PWvZzTnkecPT=`o$tTU_I+T4X$q+#FG%acUyK5JaXFj9(Ftl z$BrX-F9CIC$2vQYUs_5>@va9UDz-nJ%NbxwGzGQ5#Y$r`v0+Nr3RT>aTVr?Cjo_tF zp@s})%4;MXx7sa}#Y*wHMmX%SIvK;JQL#6kpGg>^9n`VuPIy$95iMVJ3}IX$LPQ{r zqkgf#4{;yvgnygL8}pP5Y9FSK5s76e$Q|f}bFC~5LPsA5*N8l&?xNs<>Av3@Fc(1`hnrDcY#mW1|~nJ;pLD1}vB< zn9+R`+Yd<+Kjl|c8d*H8#VUDu(>(=R^+_1m>cDBy-gF&j9x9>$g}&kVq#Qdyl88-+3_;v@8VID$uRb!{T+AJ(xjl&Q=-aFXbN(J(JZ6xT`%H^3KT`9fTvWO`W&lA2jeba0`X+&o4$TfLX;)M zc)x|-{?AJC)f0f`@-(D|_Y;u&IU6QVEA{F8Z8j_~pbhW$FdP2qy9khfn|;b7KTM}1 zXod0~X0~e57&7Fkzs*jw{sQ?w&Ap{M0TWpv9|S;EfI)t%4nWZYgTO@)K|PEFhExI_ z?*!5Ua#DX_00w~++~20+01eK=^KCNzAil4}KD-)eGo6|eT39_D6WU4tgiP<#qTzTS zw!gHrVb7?sK5?ub+$Rv%Dk1~y3E_L$*&5;ZLSaDHyIgwj1@SpVaA^@8Yr!$3(}PpngWE@qcCW?W|g!wT^2=aA1P=+V*s~#l79CvBq9y*DRP_(H1=PR^f(VguOxDXb*8mqp~S@7{8u_)a!dp_I4 z8qbBl3llmBU6NBKJ;?jfpdOXT&pmn(05I?&noC@ij@1r6>Dn0nDcU>l>)|mv;B#Rig6V?GQX*`8@V+i6^mUB-+m>Y4&zto=Y3_ zwPBHCicrk?WRaCYC{H==2qBqpJC5}Uhte0cn3$NjRL7&&gj$atUrWnoyNaz+UTXE? z^D0p)cDUhNZb6^p`Xolmx|54}GhU0^I{iuf6nlm!eiHMGY-?V8cC=fRPjh`Ra;{VY zIp1ari21)J1_dO(0NH2S_}S#t){=!kk`c?kFII^l$atY)1$tQSPCWiq-bE&5?Mk3_? zg$`&Kb67!7OP;Cr=Eq}lU*a#Vqh&?aF_~_6+uo$aI@36ri(H+<&I{i9e|;V(scR!} z;5xL-hCOw`Cf8_tVsU-@uw5xH&-3Sv)Sof1bzg6a=I3YY>+S4J_kO)b;QjSJ;pf|h z2LQ3Y+Rk=w?=gqkyu9+>{P<%T@^9*^ZN5SBfY&|WA)|V`@6dllwB6esF#pNcUjtU( z-v?FB*MEl!?M%16f%$5)b>AZ=ZTG%I{|s?{zP9ME8Tc6s^S!SB+gksJ=9#UpW^DOc z=wAX)2LE=g?GjsnK-jARtv$Fq2Lwc4ZwGi|x;OtAX}UKz`MsQPpCR~*LID}y%>w34 zdbWE{5@)OJOy}o#g?{|R!axZ<{^I8b3D`9H(+~GYwm;m3|M-gYF@$$@!p*WCDWwMN z4eXlQi6NSUm?wgFV(syikzjPH#uv@9Lm1pB18yEPn66a5r3^Ss3JL_`L+F~)PzSDM zWdXz%G&lp2GwSPQBJ+}{S8hP;Ah5N)*jexk$)Pm@qx53=`+AP=x%ZNq}W$4B`_}9sb7M;MwU|*0hYuQDrLbZHAAAk;v zfZs*(%lU0<%;tZyGqo;!^D+Vy{P4%D2R{NgFV&7MfzKFK#Sop=6C0rfJ_3W;>!v!Ix!S}PNi4X_`$6W3?t5v^HeQ=d5cQi}Kx!QH*4#bgi zBN(t%48dC>;yF(uB;9!;%}l%*0@_i?ksat8SDCFI%`hu#puq_?5PK*_v&@dR5?OyJ zJh?~14uxRqqYtq%J}77f@4ijDIg)2x2c9a6vuffSdtw>L&3D%w&XK?THjR;m9#Pv8 z<3(%a`q!`^mwl}?APUON!3bzm>lwo|`bqa=Y%V@~{|&Tr3gtuxjKGduH4kwA@e}q)jNHvlcGv zH%&SYLE`D-ch=r3uCMv$+AEp-Wz*33uNx#Yk&vkbzMc#(7^Pgq-BDo zsKGhi)oqw~!9IEEDpIu6{VWmpGgw*oXr~KIRLL0A)~ev9 zHA_yk-ZDE-xmNe30c#5etZJYbkMSXzZ^|Zjm$PKdcHEjRLI%7ww6p$|jf2yUZ4~!X z%KP0pdlX7LnCwtihmI>UV44WuEpI4WX1eZVsIA!_SX zA?HeDAKNEK3DL`{Kq^t(9!Da*sLIy&&Vql@jbR#TqYR%Q&)cX!)za!x;v$ge&nNi$yPI}%s!mOw7`vzov7tmw3WXO# zr-*^DqR4y_2%I~x#GdAw&<$xVmR#>CSVyN1)mZB24@VTn&7J)xXANdfNiDh#a~!Fo z*SP8jN2TUOMM(0_xA?!1*zhW*eCocvDOQ*_9Xo3<6fDnlAP({8a}ZixhSv5YQq9|2 z%TsqXgkKH?hk~0(mgpJV*1Pe}W=%{DoxVEQl8kG+p%TkR+c+I=J#yGEOh9~*-!*RD zeS?a9<EK*GOHDdsd|XvoD8 zJe>nE$DcBZ{`aiKlD(%owKxWkBrcnEWmypo_tshKi5PNE2}zwm_CK!QM62EMHl{e0 zM~AI8ZEKo|1diPC?WUs5)a)dgEJ<;@^>GrFN_Y|^QUs)*a)&fuD9|Q%-0f)fxA9_t z#u!?5=`Bv^o9=pjV@T{OBy{~V_79k0%!C|Vf#vV#FnGQGlgw4VcUOP_5WlGr1Q0|e z>K9$s7qAiYCrZW}(0KfX6fgrO1N;pEYoK=pfuku=m5TryTRWf79XUbx zYSO|CVi2|h>Un$FQ$u-5+)P>On7L|6%>7bv_4WA55i~g4p?2JZug=2C>w{Y zp8F?IdHGJ7k~gkv<3HR4;kpRKUMfJkW~)2k6D>qU_9YKtJ&GOBz9K8V_JbUnU5028 zkagn9^>b17;q)?`@};|+&a`w3O;rowGUtnj!ieZr-F#)J)i! zqmL{Aa;T?y%O{xfmxx%@i&g35q2q7)JTBu0wX{rQt=u{x<|Tf=XaS&sL)REXP5od* zwBXNFtkL%)agKA!1U|s)dh2ZvenjIVx^-_hrfm#0E3dVKa7gitAh?v^prK|f?{7=2 z2mFS=4V)=eCT4TOd|zrbyL2;KPZfRbIkZdlnrh(5(&;?vOHB7fC05UDaFoNsb+D|F zH4eL@zXW!#XvnkThDe=l-RSz#?0n*@=4y$4tZk%R9$_ARXASESl5%8-o2y$g(Qy{0 zs=d0cE(8&n^7daVCE1`9k|j=6y6vo5NuibeI+-1Lev9|q9^|Ub4m%%gJZdNe7^6EP zUSX*HoJMUzNup1;F?=WtnkF(AU+V7|a~|HZ)z;s#_N@UdHAGSbYa5FJ7rse)uX>wvukr29Q7Z|=X6lJUYZ|&w8!U`f8Zma{?V@nw;Lm(( z@6?0QIp$>W^^b(3P)&_V3qP#CB^pAJJ-+Q{U*ffKSVKJ4 z9QzC*MTNJ}5j8Pb4x%KKEG63n>CN0B{K2+}C(+{9{eH}J=~me@1hv9b1NJk&8n2=D$%R+@DKO^_jx%Y~I0x*i|c--UYc8e4s6dSFM^jx>QrJaT}9&H4Xw zJ6T943T0x``tFE~y&1tx$VG%s@kHM7r@AsB(@9O&Z%b6K0Ovz&K_jZ9oiq=5#W z)F)kPXPtvL)t(vf{nKcwXd-rCeX795^B0?Lv)n{XOxmXDIxmDG7l+qsV2O;BK8Ts= zcb7_jR#hpRzDr7~w+O9t>10ld2&8+T%0;NLERvSiv5!`;U6rfs*nR|217rkMUlGPjMY zw7^*q5oDl} z2$c!TqQ1p1FKoPC`5L4&6s*n(#@|qufD%9%aqo%{7Pdad+b{{o=CsYyGAVfb=7pYD zw4sSYz8M4*zMm^e=he_~rl9nQNww5T< zEu1rvFf4@dO@|@Tu%SwNCCWxJltu5+Cudo*N%78ibkiGdCJ7UGGcCti90VR_@_$$y zMkr)Cwsgh$l>)HxS`lNr=4LC$+E<#Er&iU8o|V-uk~%e zhg}or+r8)Ty3Pb$G7J%#6x!`Oz22v#_Jv-hq*GY>SwbxcC&OkfVHlrBZIs?ObqkH4CoLeI(=Von~Hr#H6OwkL1R__uYrmP}1!8Lr%6;naS< z6wTYgmZPCluXTB}ZB@hhymokGCW1LIgD{;#@+?!zYc#yEtXv1QOOB|;Ob?JVG;fUi z?ptZ!hzKqPIN@w=wCHd9F8GOEE)y99weKeyiRN%~_)V@A zX|n%(&l*!pyF~jrB3HM+5>XP$B4}pyFPxK#0oP-cJfSX$H$;ry>*eRzS%_P2QWrY~ z4D74+5$_)-9Py83JQ-%X1amu#+c3Emb?tgDmIn5d16aE9ik{%Ok6&&4_TicJG@J+| zqqVfVAGy03@j0Iz!dD|>RISuSDF_D&jeGC5Fer}9$j_-Rb#TWm65!K^#XjHKe`VjY zdWWbPyu!jip{F&gA<#skPp%WaY^YBcgZ|CyIVGg7XsL{g9Q6U0kX#UKT->*R0h?0y zO|YUk(k~k~3o+BPAu8)HDlH1dxdBQChq=J=Es%oU%A!U4NAiRJIqm&ECDA1sX!H}Zi1WujzO4m<=*Hv_#pP=Co6s$zGGl9CIPOI84&F+TE3LE!Vqh#&7G&}t0CB#kr z`NIA=*>Ka%$A7?uJ3B?mTjJvs6Hg?DY1)iD^%Vt7rP;F?0iV>KM(D z$H}3BrDd*`9w&FC?_S(O*>x{R6ZjS1ioqg3R&%FIYHujD7(76UJK^v(nCn6gNS%O0 z?%6)Za&!re=6FjH0`qq&e zU%tDwD;o4@0xIBkQ8M?NsN;J!2(_XsvCBeO1WM7o!?OdfT}MHz-J)%p0BKj)`6H=ul8ny{T+g#P><)>9-mgn$tPgJ+tl$_WIc+a`;e-ybO&p zBgIiTNEKZtu+ifc`x(&Cv7YPTyo+ua>?em|JODI zV$7H;Y9$Q{7JTN3({j_vRDyr$fm zpqeC{y*j_210KGLBZC_VsNhnr^b!V^q(vV5Vea9)Y~MXJ_Gph5m;gI9U%d?ztlNtk z0ptUJZLn|K@sp*#_df8^9}y?G{Q})=9sBpxYCRf+SHC+waQbnRjSg`9RG$uaYUqG? zXBSjm5P`=|`>y0Wj)bd9Zduasr}fJ<&1pUD_ThEkRdGygGkC}Pq6z19+LxYT%U@?; zUt7>w0}>J0s1Hu+5+(*&8%Rv-PEglsiBeMrdB=dd<(#6V0of z?#_&8UDkW8Vn{KBY1T>t;kuk6WaqX{x#Z4kIt7|&&`oDLJ((0r4f1?szo(c2uk66R z#R%o)r(CL-W_&dWd7dNf9V=Y(Lkl*KCwYV7Y7N)7WzIyxbUH3rU%f|ChhuBE-E;tg z=j-k8um*xjqP&=W$!m5NL?L9fStj;-xX!Y~jtep1C)z7(`5OjJuU5^9)N)};eH6LT zn`E$e-fm{UWczgOQYf2pa`<9t*f8Nkytl7=zcY z9xRMaacFV1dKS6JYYL1wu!$9D_fvP>+8s(SFooXu|8tCc*=OG>wD$VyUa#w?WQokn zB2icR_x4`Gc%>MOe;^O?CHr*)LUIu@7fv}<^F?!1B6+G7t^C!ixuQg!Yb!3p$q&P{ zkt|5scx#RqtPRa3^$)Dk*JJ@qqeO~Tb<0;{EUUS>@q6~dclIqER{8MM2k};zqF|^(#XFlhYs}lur>$J; zn)XyL;ExO~jr2BM5{U_2x{+D#zdeI+H}U;EE8?5fa!#IE?tc#F20bHmRtQ*EheTo# zKM8OAiT&uzpOf;_R60cLi*hY{LOL|o2+_=Hf zSqEH+N7waai{~Ob_liIuTp*QLhK%2tZFhflifuD`_@SToa!Z}!!=}w&X9_Di?|1#} z@>Na7h6Ct=Y~QR=Tes@B76YJ^6&(ivd*6sxxPAq`R zA8gxm5fhq3*Za=-C8hHeivynR6WTZMzx=(Dz3)EWl;hJ{1f9~cXi#y*`ffNM(BXV> zcyIagv$b@S!Gd79bm6YG;cDh=TEkVnp1dGKZ_!+gFJ`AJxcHadUYwl{59unY5$o6g z0>?!YPAYeH=$pRr30(i5^b<}1iP^(vz(~&C{VPasTn#)Aiu;?iGceiCVbv+cJXJ0q znp{P{dC#;RGV_{u0+(}hx(bLB0^sMYzktd4-}3&zg%y~yqqCtz>Hm`Y7{9I9pit~f z(J1bmAA|xUwr=R4fnKM1mrmZT5X5cKETu!rDZT<58H}D4`_1Uz@I$XbV8RgzQ>M54<7(M%*Z#$NhZtB@?UEG`*X5I8oH<`BjkS$y3 z({^ZtPX7YZlaP(k4tFSYhAM-;7JI>qiVFb zwzzw_7kyf$?6@9>9H(xvg4Q*q+ilZ<$xu?im5V`wWGGx=E57~Vd-D%UYMS#Fe3t-r z-;Fwa<-KUHB9R|Q2Y^}3!ej}GTx`9C0(*oA~Xn#0Yt!qiM<+g_AezA|Tl$bH}KDbs7)Ich{S z`4a?)KgXg{Wp|ootdk)?p0}Fv@Q#X9|q|u!%7KL}d(>#ogdw zym~r+j3tLdeA(E{k#>35NUa#1ZMcc~OTGhUfIi{+2%G-(h}Pq9hwN-&zt_G75tm}a z4fpZx?z1gxJ`(w}Cjx}1?;qqNS$hs1z49t*Zbh|GNYVR{Zvdj9wl!huzhXeh8ZqYc z?fjZ@<$RFPRFNqM&O1thm&tgS5h&64j!xQ~i3!Tw?;}&ekFWi^1pS=}qu!@UVb^ZA z$_^S4c@sd1v+z-oKWqqtpToWT`g=S6)F!tWmhr--eXlT(lg`#f=O$}*)Ak;kixG-p zU2Cl%bFOB1J9($at$$0%iAH~K$`<3Z&$li$ZIx!=T)G2D(id>QdQ%-vhD9QuIA2OI ziBA2Yv9OBeE`b(|_%n~GN54*bm+Hk}a>x6%P$f|k{|Vo=8H=@-Q#Gr?oJZ<3A-7de zjf)x}p00KYPmc(GVol$C{OIwyPC$s?qjiAOeFzD*OsO4nK_9(C0D#kP&YiF#az0NQ zWpCf36KC8ro5)slej%rxV_dlZp26^)U)jSy=A@Xb`AN4+G(y0qCBrGiLAx)mz^16bGk;VMHI5hEbmf|pQ3L` z?r@AHJ|>2};NG7O(TBF=c+a3}EUO9rHAa>j(L}eopwTJE<^nO-fcM$BQz)cYbnc*Q zVJm!boBQf2vbrXDPAzq?-6+pzZpM4fls4f5I-@s#rFe_~_2h#laMxm(Q!-TDdAo1q zq9S^3R#2T5pI>;aD54Ov68?RP3*) zf#O}zOjuX%QM@6vrC!2f$d_FW&I`s+8Jv7bI*vXTG8$smY7Y_!~aNn+_>{Jp=$5+5JdkLd2pv1m-y=_05y#-kDe90?3!+^BmK~^^KEToZk=wk*50png zH6U5u{qOV5eXuG6fNUoQR_M5c2zm;0S^M?fMOjl5D3~IRke7-4YVf#-Yd}IhYV;BB zCQO@+nlE@fn`TL)wyZy>L08Cra`~IX^hNr2`#68b!=rq{U^d>5M$6`4=I}RMaaD8* zL(gVxnS$-pll79Z#p@Ag_N<7G@i&ig1D1J_dna?sq~hqlF{owB3T~2zOQ_eGj(DeE zA2|)&71O&?AAcyB-5{IekIY;|shLUA8o3z;QScn)fg9nx<%<&O5=B~NEAQVo8PvLa?M0m1WaU2Xg!^zkY8GGtYXk2ZOn z_tM7Rwbo;tNA1vNZ|?dDjUokP_Vpk|)$e40KZhJru`*_En}yE)Cl4tDP+y7rWMB1D zX;u%D$t>kMWV%-T3Ep|9M|OGnA9_?cx0!cGP8i&y?ZJdZ&1x*;R1cJAyR-Q3(rlC_#T%8~GsEQbYuHSm9JM%gNCmNqtnMnPtO6j# zVFgovuP>6pq`|{u|lpvvP$;a+`|DStQw8Q@`)~He(IXQWD);Z09av@e!A#I|^ z6&+%Dziolo)Da@+Brl3DVu<}*==nR9L`v7D#=!at;1#V!X? zv&bT8t$YJVXU~OFceQb_(0u~p>XaGnCRqCtay4Q<6p6pTn(!J|p)MtC+MV6>MT=*r z^x3S^80$^;m8GeFADg?VuLzt7TP*>S0%uEoA0bcPWI<}GBC%VW4nMmmk2iy@M#1Ww z=2F+&f^^3IQ)Ru|B+0xHBj^hW=J~s#Je0Lk_q^XG&&uq9D;sDiUAA|N!rs+YIGoI18~x*e}&D9GEK%NB#bocWI264h6y^WP1P z84!p>%P}mww@*#?XZil;a@ltG){4&BsG2Xsb6qn;i2Y8zg)JN1XXCb$BGUThPTPe)De6p37b zwJkx=yxy-)+Zf7vBd^Ugn!kQ&5d`Cdea&qmm9*Blv%GNOMg0Xkb4q~hdbt+eWcq`m z(hhy|EqR&$n(W~m7d@PuXM3BWmuLlD!!V`9O_a||9BTbL!n{A_t7w&@ZpFgiUcT>! zJvp7sc}T`^o~Ls%a}lI+m5lXvOwX9)mi0vx!UwvmO0YO2Oll_M*cN1#X++4ZiS;*c9HNYrInZCdbff!OYq#%s|IW7z_mBu> zMJ<@B`l|Ok6*;smmoJBghea0SJaHrNln38_De)QjKf8atoXlLVRSJ=tx1bjcQC8W~ z`~HA^pXL(h-@49oPQSYU?jSqi3*H7^%q3IKBH0aJMD4RHv5fX7k zoC zZ;z=^9}edObYC)Dmsvj)OM)Z6?>W8B##3##BVhYjUEF6~7~Do-E&SZKn4wV&jd z&iPcW6WnIIlRE5ZCSoYB8J&Y)Bz1Y^lH|HCu;whW9Tte5p0sg<;6=#wx08+|j{=f8 z#C&jwoGf7a;`P^3tt$yQQU#R^ziTPdH8#nLe^$Y^;#kT3(uv-Xct~|FsVl#eLE2P; zsmohx4!PQa(?lvT!zoKA6mrBzdRpl$+}k!hN1L?2fpeXd`iteIqBQUPS$)sKJN7du z1|yuX7B3BB4Dlamsmc7iehcNZgXh8QVQ%?yP@?2UPiC9v`^Q}}Q*H|1tk(*Vi8^wK zsax(hBr&g?lVo}sA^Gb2H9oryFOjwOPVv~A&w6>z_v1Vks#b!3kbk@U@Gx}y^YxOC zb+Kw3>$bOVX401FPo!)rziUL}D>#tkZ)w)GMshQ_y5(|WgEy?P1yZ(ms(4D*L)qcg zyYh72-X8|J?A?875`DK>%2gHDr{ttCwlDiZRO-ZHcF~sI-J;LC&oIjO)xku~s@GRO zR#!vD>sfOZ=WnI*mfFvD%2&2LzqX*&EmZeeH?Fj_O70%t>&5sdm&txziwl*rZYg+q z#Y^QgfPB|Zx9{eqzq74zV#ig0q`RaXA0mlnh51%!70m~?3-iZ+yM1l@_k;QV$>L>j zFEY_}9wy5}w2{0&cW7#*(p~X9{?z%)cO3}})r7)-r!%R``_7(T?P%X*b6SU&m_>N{ zn~TMJ35AIMJ!_#kS#Ae6nLy)zf3jYhUgO!KrE?KF)^i z=33T7p;96*x*+{Bg?b+{7N7CM{SL<8#H{*HE{pk4# zdXDLARgsx#o_P1x*8gg^iutRXP?~TaGEsNl4K~8*6|27gDI5j)J`XQ0LvNwTJU6Rx zI`O$0jYH#pfz{)S-#;b7!DpjskQh3aaGycid@AHmedm6U(C2&I?{n!HpPxPFZ#TDF z{F7&UOM%yP?J_jLbN7L9Zndm&(U`To`;otC?HdHypz@w$_` z;4^U-{U2XsT48@#tX7h)#af|+kI`$+V&5y~C)&oIds2S08|uz}PNd>q8EI>t?B^fxc&@1W-Ko1sv8+b*Rt(UYQYb=-#T zb?NS&sQtyrcy{tvw+*c_^^#%I>ZN~Iyhvl29gTxBt3#uFCYg0rCz!RbeRQgP7=(Dw zeQ45gQ4L$VR2R4R)_dr>Sbdb&m;U=>5~_j%~E}T(H&eM!uy=QgvOVH>`G4 zG*qMN-K!wCsrT(haAoGNtcR&u8>-#B3de0!db-pP>cYGkT$_s`H-*H=&VDT9X8vnO zL&VHB>NI!S3i+iRs`s}@#@-*&zhPbl4Dx+7^>r-z^qi$$|Kq07e)OB;jjP7wb+}gj zZNC9XitIN0)t&ir44^s7IjKKArMnlEquVcG?DnxTF?~3|5L9LclINm$TXw0f_I6DZ zYx%~}?5&)-cdvGr#UHzB-?+5R6B#|gviQm@X5%nZP7JOY3mWEP`>Gc*T3hn-m3CwC z%^0+)H0qowFP_Dbq9)~a)P6hdwYxu~HkVas?@VbFxQvEee*+8A6{1yU=?B97{TbA; zZ&8d*{a8yQ9jg=vmc`foOx{%1ce8Mcf@8qJOj8>sRhyeCuP15e%DZa7L0v8)xx3eR zXD6jI%Fq`n*2k(l8C4WTYvoobe1(v=sIV$}qNV4Yb>*D@(fhkp@%Z~{u6$W*$F^#D zhq%bW-thmZ5FdG0S{t^G(oK}7pp^BJf%l{B%6ER#59h~kDsxvut&aN4#^tChy>{z- z`3~2gGsp1MtC5t~Cw8p%{3aMv>*R9x`p<&K!!=Q6%QjKs$E^8&ziH;OpB9HhdzoKT zn~9{cU%6#^yJzl3dxGUr$1wk^#dY^IEDMvsTq3=kDK7TADaQSG+PL|@d6Aq1G@GE0Bo z;_X|&(|_lM3iDR<<|_VcobS3AUOg7tVpb6ivs6uTC81 ze4PyNX85-v?zMi#wblkB_Y}OJx0GerC#K2e3{Ff$TW*4^sGex*`DsIU8*;UD58oO} zNP}K|5iuYZ(!yokQ%?K1)L>lKXY) z$d--I<9dmO?7<=-!-5(Mq9NBuO$$g>_Q@%vgMY`ZGOFAP6!cE7Zeh!|$rC5<^2F!b zJk`2#yZNl}A*|JnWcP@iaRm+}NbqiyG(`F$QmOkprvsBC&7i89yE!6CcNYoLX;f4Q zr6~X?KHc1xf|7W;tjtS4>v%=+F0t*auf6kD%2OsBayl64sVVaqhtNV;;p;O)*bG> z&syj_uDT*#pGtb$i`n~#;XzKn9od&ARip^JpW}rf&$pGcFJQJ)-*Lr?tZ-y5&F3&a zFBZtq+?=cF(NY3RB4}tpIRx^-J42LUnLQ#>)_RjP2rh}s)@+Tt8-C$EcG^N(Pi?b_ zurD@m+X5PFhF+5H^sca{iiYJMDBoCU$IdQ}7L)j%5geykJ{{`ew!K9O!}2ZmoWwv1 z#5pXomPyUSj6+mAw#P5=WN}2W&CZzf_+$yV>QWsGhjz4ufsU#D{j+%7vmccg%!3`3 z2Bn(snp_9s@Rojl-gU83-@t+xNSelut-x!skLP3|@s8N~cV7 zx()Djv`v3obxzmtNrG38Jfmd7;dJQ9MyI{}#W6!%N0|8&Ur725a(S>}UDix+NOh>vba{{?Wc;2hFh@M?J=tnKnYWF0qAy zAj`?z!^NV0vfs>H9je7VdAMGRx7SYMcyVn>?w?7*`fJ}m7%Y78;nPN_a-x~%Wr10H ziR<_>1BtHaS4#R%sc|{CG)euUelJp+PGBT;RvcoJdtx|l6oo}0A*W)_m5WnP_FbiY zAZw?@j-N6_2MS!2YDr)nJ$QSc1GjKzfYNed_n*6&QkTR|=1_ikDE2Ed5{?##Wq{`JQK}*FqNGXKS{43W88k9tR6NR2R`s3RJn9hzn@By zlB=Vj&vuzlY41^fC?8TAQn8Vx@}>32m2a)qzb8gfX1zcTQ~Uw4#ZKlSG`9^RkqMhw z{FJ%HqEC*81Z8x~7Hz8pjz_OxaF%h5$h@C{{-OJGCa;@mG9Xx!aS-9n%CD^r9wTaq z=Q%>@NH3ColgHECsEt|azt&$Nh;}G4x|7@syI@MEmiodn}$A0L~|Vc^)qd? zl+$)O6KRu;r11dj#x*~(6o%U7`LC2*WRJs3?SBfoxoSj63U1oKiLInyOgExa@1^5( za8J&U(tOn*IZ~$dJ)>XeS-$XkWkF)2_3|O{`hp;1Gh(GzZ?ONyDVWu0pXu1_5j*V^`- zECr64y5u+)ss}6XLXRW3XQ0#DR=?I9x`sYopo#P;5p{O5sFP#O0LIc@p{>jruNJxF z??eyPq)9uBw}bn7sn$oTB~B?yhD9oJ%6I`ewv6)s>TuU3j5%w&=X(!hmNbXB*N@34 za~`30-C^K$hkEc@!>-oa0-?2gxM>j@p+S|pQpK8MYcHj-52oPcY{W=uh~jgr%V-%d zo}$EXJ5?(BCxe(MObe3$YVesYH}u@(@8_o^?L%^d+&Udk+hKK$wAqCKic*vaCuV69 zLRnnKyfyp#CYvkl{?kP|#yYWz@S)eVY;#+m;UG^0FxM^r4>4(t$V2nHmr^6cMf9DE zJYfK@l_Xli)%=*Eh?iz|qLL`KE5x=5%pdkc@B6vD0q%jiWw8Vd2!-vHw`BCo*nIn?B~==sHWH2FA4ql zcUM9nQv@k86joVP54I4Qh!pxOXH2(i%7j38X_oxEB zJwY)oClGMK62sc6r!Q45#nVPiZlpRq+jZTss6oME51Q)Q8&PyM3J}>vU`xEakQ0w} z%-l?5`hSnz_Brk%N854j^k0nV{xU{_mRS)+U#8jc3Li?e;|9udsBiBI52fR7?gPNv z4T@TcwbTJjJcohdp)zmQ9=eMsq@WOkyeh*izXa0s*mBY=IsXjt14Ef{;YhmP4}pi|E+yQuo6E%e|z~duogct#a_eDNhsaSf1XqF%W14 zF%e!fc(=fDovxNuvJ3INojt`;={r6~#Sym>@Ta3R!>mxY^(ZCm$gXW=Ju9Hn+^!Zc z%ML~>42ezEHKF5?JIJ192&x0}{#o=Q`EWicm89rxE!QITMb~%1_Gul5{toCwr5E%Q&Xygnh*Uhe1gG z>5F&3(P@@D$M0MAxYp$djK`a0ww>gl(JVxp>b{7m*qs%ri7fD*4lDq&{+ik%AAWZs z4XM+_l?Yk>uA*O`sW%&iiS`rU(iYn=pJwuHs&j;NH$(#J0sExksA{Ek1JMPE8D&H5xmoX!u|L3JzhWkL zbWPM;T%Av{TNkRKbR`|>sr?_{@jI69=#GcW?tsLVVkZ*>T=Z)VRhY1S|LwLDp6P3~ zKC;kmYm#MSg{MT9jYRJbZ@sR}L78L|}?S$d>`j+CN~Iq9{SCl<&T!_0V;b>1kkF zPWKu_RnH^yk2beFjnnXM zC#!BE`7vO$C~xe5UsZ-K1t_$P#&)siq+~On4%7UsM!0QxZf$fQfMq6o#lJdhPh{VJ z$d+wf`_P@$2$wH0fkes8`?|%t?QoWQ3 z^xVjlahnW)qBKluG64RM@qCb3HXb}H<-);BO<)e206ai#RxS}*s>f(26 zV|^>?>&!H_>%P!OPVW-SMEXXeGap2RQr(ZfShCr48+VG0 zZM_ia$I#yi;Hed?#-mU?ai-IV&2~q*!9cRBD4{g}XUMAnY<2$DLJ{6Lzg}*FJi*r_ zXJNGEeffA_`CHW&RVK^#LwlKF)tH5K-I};KxhpRYe&X8mzaQ9fwsPx!9+iprwu!%} zOF)h^l>=HocFLg;NwL8pCDfuALkVd5Rb8CXGh*#G@%LoAXxn*)kC6o86+9FPB2gKl#cSrq|d*b&PMOH6ZBt_rm6s`RI z|NP0GIYgU+%s{P-y9^I|9^wqtn|_WuGd}cfoE?073>My{xRB(38~HvZCnD*ooVjVf z(G!Wb4Pp#YeO%{II%B~R{LCCwO8a1W6oI$3n()uK7Vd)GWKFtjrN2kD6RwF)z&5qc zlgIHN)XEYS@k=9*lZop@!$05Y?hmzg<=dM)H=T}t4eyxa8g3Sv(@KF6A4g{YfRS?j z`@1z&qlH$VL<4u`Lp;42-dZ;8BB5FW>JR$l@3DQ}PA02lLZo2dkS=++W8e(0Cz|A? zjqY~ClKGp~-OK6_AwSdh)eh2yqWHrxn8Fq~ZoaR~^iQj_iy*|(Aw)iSJA*lH=N;$3 ztX%nTmmgs983CmHz9JzwyKLoI6d`3J`kf}CY+j_;YIw2jV`;f{?g*BAU29mbySf8P zWzYEhSg#oB7S`csaC@IEk@i~GuWae=6j*PS)bvLY@Cp@T+UT^ytpiuKF(p7*5{~#1 z)BZ03!#*puYtM90{8IAINSr3#9E;Vq`$Tp0U^1 zFT3#J9X|*%^r}TlP(mIxGv>3cLH^Mh#M-S|9pk{vrcj}#?m{N+deP2m?d&TT#(h*p zck{>VO`ePqqeB)+c~|jKBBJZjd(XCO-T|sGGdSuiZjh(vYb}21rt(3B84;p;`f3FP zQVaA$vnT3Z2VWwcS{w}>(@j-1(oGvH>5o8LFS^*g=S!6-qdVzm6ZT7|u}i6+uB+Sd zRuan1>sW~ny;4fMU+O%iDunI3r#>PRJjXao#UJy&wAtEVaqe~0N#?@IPlU9ibqXl9 zdw+-DMh0Xg{j3aH)o^*aNA2<})L(W9LUJVF}m z1qHsU9RH48s<9I5YiyH#NVCrSJMeo$mQ@-ePo`@G9!c=ADfF*-7Z9JKaM3s*W%d;3 zUIM9Ls?7hTiywNO+JoI8;G3v?P*Qv}f4vuhG( zV)|#u`YdZMx{KPS`J8cb&Z|b;n>S$mjO?BjRM-OXPJM+(fxdL^zj}mYo$$@VWXOAe zV93L_+s(SymLky~2hy*!800Z|-LydiqcNxENRHd?sYn3Edu5 zVHaH{rMnMUs;=kmnG7Q+8fQ)g9^G+r5}4TZO6-C<8)JuszC$xaOm_2D;OOr-RP$xy zt{JvYoV))|2Kkd|A5^GHPgx?OuCNvM8Eqa=%qT=&T!g&6@H4Vi@SQd-4aJ<)l^L&^ z-}a)th&X`O`y8q4BaHKDmGG;%1aF|xz|anG`7yHPs{6QKzSuHeHG~J2T`nAfrhFM$ z4Gf!vF9Y}La&7bXt#%oBPx_N5XOwF#cRH)UMhfTHK9%-p9chay=!c`JMNqF=4{DFhth%e-7U)$T=>6fq`ZoHmw;bSY`?)U;dJ>dp<^gJtB1!GX~cRy&HUt2m798%Zd@H*R{Q71J-PNy%AW*qLhqoZzt z_OfSNIctAZwAcF8*kBQsUWgt%X8Vig`F)xCsD}x}a}~}Qqapk-%$WRUa`xif&x28i zp=0@>ciXIhXrJQ#PSME|%EiGW=j$qJzSk|EP&vU}woPfR(5Kqcwy9KuJ%Ct3u!zTn z*A&|oLXPLt)YyL4XEPd!w{uaOIDL8~;&x6E{T0fb$j>*KDs}*&(XGDX@`Q`T9AC)x z9w`wL@O}bl5}`Xb8YTcu>z6tDhl1jH=l_bQo;H$SoIX+7ISCj;M_GECxLKt=_+PyZ ztH%Wrn{p|rgDq}V?g;DZ(bQ8GirtI!Xn5=y!#Nr|zZ|66gW#_tJczdi>62R2<<_|%yhWIBaZ9xYtNT!XiKIOZ$c~ve;(B zY?SgLj=3wA83>#Hmv`AS0vddp-z_%}H(85?Rh?4|e{ zUjAHSZqDSrWqi>5vA9k_kCw#sSpd$CB7eh{XZFx zdFkaAfW!3(HA|el1O^CU5zQ2Pl}A?2LE~YS@lD%gsqzl4Gn~F**sq547_YbQ$v(RSe|E`vH ziA0i~`7Xqv`VY(qu>hn>ORpf_>w6Df#$^{&B03xgW<_+)s^J$qWk6qRg7G+DaDB8A z|LoOmm_QthDZyHS%+yIlBc1anDeHVhsVbw&S6qS5@1ihoHFaavMARWgM5#mGS^P|; z|FMZk^Jldy=e;)yKY z-q+xkTtr|jbpF^iO6Y$@u(@514}$U1^0+5TD`t|+?>iPKpQw&vM4 zV>71gJixs68Q8T)baJsEOSvL8)Hd&Y#!Hk@N4TjBa2q&R(HLLZ%%Ifeij}vuL2ko% zl|I8FKl;5B`(sAxq?nz-?Gz|R8F8@)l?)wm27_%=C|H zAGtC4HF(j+gSv#u3XwhU3O_#S2GY}&kDtgaWZmr_| zy*5NF@5dbypzI=Okt|N$%JL$`!R^s_R~zO5j)T(OH;-!N8u$e3t*^0+13u`ltyVjpOq9$->UB;3eiFu<_Y~4QKUw^Vfkq{6zNKxY5vC8`JP+0SK z;nzL4Oi&Bs>dPFna_!*pRr0o2Tfb6sDX$%e8u^SI28HQJm<^4+>uS&gfW8G)Y@tBI z7-E`k$xWAZ>+xJ`olO=}5?Hdtkv95uSaF<`xx8dbDiv>s6Ud3A`*jTw_h3OeB}T`H zCfHqIY@X4lF>jswFo~EzL=g~4ul6U-e=FmS`4fzFx0KQPdv&GFe|q1cWJIv2YdqAg zv~Nk*MzZHU#9odAk=tl2q@T{5u1@~`q9h_G@t*QQi9D6P+PTdZfSj{%sj5@TjV%*cTi7`L-R)CiBCU;uBdC z3j#|>qFOYbR{1%QO%C{U%av0Jrl34i&fPoxhInw?Np8QE*}9^ZPt+UL+esh0_1cjc zx?%p5<~~NvXJkv}gzB!FbU)f(H?MSX0t1WKl@92%Giv3TYB?+;AM< z`JyLGSNmYK7zfK~?QONq6DmYTw51FAXS|bgsFFZZl+veD!*%}jI6vWk-KOb}cg1Ha zn5!3CHX;CsAvAMKS`5hnu`U_|e)zbCWX%{4*M51E51F)QEb@RMM!92LTGZotp}zg6Y_C_x|D~^W^{TD5cI`>%A!-9b zV)gub<+47UzE)L$PQ~+R{{*R(8o^LES?bbPjBf6-hLFakjxIIS)M!qe+(CtGUn*M| zlUt)Q6?M@nQGzF<+;H?U)|aXcvMZRUIt^>D;(#$>t-}oA^vv8a%UjpwW|p;He2wyme#T9aMjkBmOoS5NFU!4i*|a=8hXDJ$4j z+eFuw^N}dgoy;~08}Tyh{^<(>Tq~!^=tCFU|D5^)gh^c7;@$L5L%#32*q-y^aJU>_ zxPiOGqzF$BnwCVKY&NKd*Ae)-yhF}h@dKGlypqHn)MNpaXZyNU_Zf`;DkTBpEvAcN zGO9)&6N$v458IgSV|0#0)Hpgn$EkFF(jaNb)sY7H1L5z~v4TYCju0TgF>2?bfG|ph zsiIzT?YHVghK?5HV_g>=Rd6aMabybA{qw;7%bW`5$t}kgS!D=9!YKMKEK-gH0w!7E z?M4;IDWAK*PSJEv8AOUTptYsel7{D%R+UIG9_yBb?b`}ak!z~}zTmg8T!wDGfhKkOF%CtZe5cz<{ z32uo70Kmv5!(e7YWo4qVgRekk0sx$G_cw7_%1Oe*04P*W1z-yRwkJ|4yM+cDnhll> z03ZF14M0Paq5^QCJP|3g2BwOIqkeR?>82Vcf8q!c=%Ylu6R#45MlN?aW^b{t9_lNff04x9i zAyud-l1wy0&!A)gnt-rpdS3QOZUlgk69Di3ObL9{lL;7!0w9OdXIoI1C}60Z#g5Y8 zoW!2L6p&3(p|MD|NCyayop`Y2<^O!0D<{BiSr91jlAY$PCyQJ4R2hWEnND?&?5UtK zQiaC#Qb2+JdwRHk`iEnQXp+3I^vQ8jsHueKy`Mik2Ofr)^-$PDx&0Jab~huAGD3sO zjh(_%NYqV0Oc*|sfnn3ir@5SDYu^yb^wn&F$q6R<*-k=;pw#jr?io7S)^z}83vDj~ z0-1v*$=pH968io-Xw+9N-O~BS8Eu-Bq_5r}t=)g1do9$;P4@k{GX>BP9SznWVYn$E zd4IRPj+=v+;dH$JO_}}3VrprTU!6%`0b<8j6WzCT<5|%$#dW$}xU<>Q1#y@GeS5G9@ry7*5hj$}s)e|q@H&-SH2RoH= zQ<1+6YmcVnfZM_|%f{Zej9~soT|IEY*gz1Ak2v;X!AjVopoa$Y$%3QQZ9Vr*Y-+5X8`h0M;LYVZ zabHc|_>Y3~!@|9_6Nj3~v((j_j1R|y*MdchnS!NG6A6*2dMWypyA<+9yc~;*wwtY+ zKbz*R%Jivb8!it`qmO;5TEf$AAAPHJwT{i2%eFsy@oxIusQZp?{xtAy_Tf(8-%Q!5 zA*#73tG7Ei@CGti~uxZdteQg|8{-f-e^kdO@;&8tq&RX7Jt-i$q$9Qe8`KG#U}C6cwCK@( zdG&@X##6HvzxDU}LlA1BlSxQu&%!ee7_lGbyNjxUF zJT40_cxO(h^ju31)TtdemhOywkh&RU$dRj!^m3bMSWrEi_V!*8Q2-@vaH65&FD@ol zv7Jn7d%|^RQ_XJhuD`{|^>9~%v~##8-&jb_P8K+XYk`VFEz96IinP!!#(>E;E6EXZ zk}+JK(8<}orgpAWb=C3Z#L?fJlDkQV&p$IbZTm3eNRVBP6P>{Lfb3-mwa5m?rw0 ze_>4TkBTw>q+V@=$+3gn5{>Ga^trV{%6BXm3)odiUz0>3?njl^)h+JxR`%s`ueI?> z=y+rwBSu#MqY7Pg#t%j;Da^=o+;~2tvJJL^!0k|5;!Po4;cW1>8OOOkJDR($1fqE(|6 zPXAfdK~vS)o{NqJ`Ol@AO3cy2&AzjTRX4eOEJ-HK=ZnW3)pM!#8=Ug3}!F6yS`gm}5j#T9)|)hm=iUqX`kSr8ije z`(#fRaP8+h%QXUnZFRgEvSDNFQo~{|Y}`J{?Gc_O9~A2kbgT;$IXoLxobig^^fYZq zL&ENHe}PHqXXg_%ssx*&zrn`8uuAC0Pa{(kIleJ-5<WH_sV)W1U|Q4S2R8GbD9LsZ5l=-3Q56=iw*Ay(B9?S^s;Q`7 zOe<`1v0Q{^2-RB?ED44RpnZ}$RpOTd3X5zS21)y*w^STKNMw#Zyfe{)rZGsn`Pdox zF;Y8Z!EqH>S_~9Yh}ly8T2#UAdqxyg@*pV+s!co$V7h!e%%IZYUQ~>iXAFflIn{AI zmJqI}Njxz2mud)QNw$C~r3{rbDT@)FG?mXYmk^91OT z{`Fa8=|OGy{eyz$ufXEV5rheP!W~aPv^fc$T*+WE#GnpL$q%g`vI?NE)ZiDt$DXT7 znq4D*v$i-h8vvCJhio#DUS#q3W|f{Ftym&_Gg_p7*O`5<({yg&P&_l7;vdu@l zo3-s}aHOWFV@)QX*m~G?$M*|-&35#Z6wsCUN(cQxqFs@R?Ca=-<}%`Pt)^U2tda=c zeL+5#E+c=sb!~`G;$_?I)^`QDmBaK^#^P-#+2G=D15M36qs{`)v8|}y3r55MtEpn- z;n}FR>7MsO?`DSJWn;o3I!FR=P-59CPKuQTs~dM&Szg9aTHYz{qFH%Nt&58)M_#eM zi4rt}p_;DrZ>V}t<`2_gJ}#N$r|#J0H7tqFbH5T>v-vY~ezZPl{#{fPw;p`lPJK|D zw4qooHQWcesNw?sM@ikfFSB*NtIFLG772I8AjSb3k54@~!B9=gi*h6|Q!&Cc`@-?m z@Cav2t``X71O5S{IcQb@Oi7gr-jsJp(?_Ii#Y7(^e3$V|PG7f8;(w#;<#P~=8O(yk z3Y92cU-j1uTiF=~E!J81=IadUGWZ(2z~J9;u<0yDe^{3()sgEr`x5Q9+Nf8;9K1Wh zn*6XhFpVx-12XZJd5GQu{;cVJY^aNU5Mi=GMP3>TS}fbI8Z+0lS4LWe$@!MruL6Z% zcZ9&AgE-be2Ly^>07>I=$g-8=;7cweWbu(0Cx(ZMVQ*q0b}-`W#2<-vZgb0WQ~FGC zS_2ea?F^ZNz87oe>aVC%_F}gwNHdaDEQ)b#m|}cqJHkKF58a^R;ajUEFLu`yt~M*Y zf?{|7Tv{=Mo_oPQ>FANJ0JT}RU2oVktG^s2`nm`wM2hNh6?r^z#$@bU905IjQPvxu zM*MxpAs<=(rCnd~9GnGTgvLOdqQqMy(;dvC6{zEuxn8=Xt!@lfny~FJO0=(9p0iTb4Y%obz^f z()gLRI9YW&PFCy%9V*4gD<~>nP1N(ab9o#VNXRtF+@7zJ_~R_!?jN8luoqX3#vE*2K}`{8keGMO$eL&;aGb z8SKoCAwIl zW}5>By2?2`4x{jbQ%ww5d6Zt7<|LuLy&`5SS&iE=7huqSnnD#C=%!do9f#^WkxGG43>650A|8Ai(A{l00XAGOlW%d~&YCF=z1sODS+~JQ z?hMBWD(Dv`^N+Q_{n?2#+^zbRoRa0<)-?2G%Sx?vUzE0tJk=E&HSY<54sG(Alxq>f zDP2t#v2waz-n}lv{;7SobYJo#syX{69J#l^BMHhthi!WaMP&>@y`RU1+CR_+4zUNe z!kNaqsu47{d92S0s$$U31~qR}6jUg_+xnpNu8*tonWJCpcTHaXA#AWQ4VXEa(r zAM2ojmWwK(GINGc9#Bnu_9Cga?#kqf^UN-;^9pz_(hu_9B|in~(h^Q}EZSA8;>z$e z%RQp#v17teO3&g5l~oO zk~)?J$$sL{zPk2~6RPt28D)rG2*iYy*@=Rc=}kN%R2AgtSyd8hduIQ>&S0_dz3t}t zVVaOTTXfltF=NR+vs{H8(dAl-sHcda;K3jtei&1|Ac1 z1wHHI+ql#$S3pb6R(c`4_CVUAO%pLXY37T9=bHI4XRP&TQ6H_YG3QIu83vmkh(HQ` z{M*}|;SpV#+BTKB4RH6Z0xQ`5H9u&Rd=6NtB!`qZ%MfLKgRiKt7+Jq$k#~G#}tf$to#|TKJM#t8LOzUuNH;@O22A}XTsE)g9(tXJ#^QSC?6x#>UEyeu0U9pN!~81EW-(j+S4Xk<(NyeH?Z~?W$MTpl6i3lBc2t zHg^Rgg$~YpL<%H&m`h!S&ZZ;Q^rZY|uj|A|=Wj0RzI-G<-xf`6cyyfJU+y`#wz zA;S&n3CJtIr1jAm#iRdBoymysc6}6)!?79kuKNL zjI>%<*5_dW0u_)XRC~7oWN9vlWbd&J!fASQCP@ciwWNJ`F+Z;f9%y<$Kw&kDxOB~I z1Vr}o2rQSNcNTmzchvq0f27DMTP^1LBT|GAuQ?*%hnf!>A%%ufK3D17o98V!x*qzR z(rFTeJO2K%fF#iqS6$m#*oFu(bw(NkMqXJ=hc>k7_`$72I-k0!z`oZ>0%+KL`hp|( zU!|28Nj)9NJ#)M*&<<59#2ramJ#KqGubq~yarmM{2*x9Q7jn(ig027q4_JMrjtLOv z*S%BLn1Y6x${sT=w4Ppw zh|m*n=4Sg=aUSSmxONzk(YCT+Mf#cV0zW9|0HFBmRHa{DUP{+tEN;sqEzP7yBquiF zHRQa`WGndm35}{7WL*0*HKvQ(x$&%Z`OdZd`{k}j=^L7dv3it6;tH36u2PQq$X+bE zP~6ZP%S>`F;&eIX8`FWViL1rFI*TyD6dve6AmtdWJYm7e7}N6)QbM+cLkY{Xjh+nS zv!b57o)4H5^T`=~iumO(3xQd}36;mXXWnsM@K2XQRe};rtk4P5c zqzD*V+CPn|>btki-Got-Uk-I}XIEzDjKDA_o(F$&Bzs8BO2n^^MnFFyBoR7dR@_WG0soZN|Vf!KOA;| zYa6>MK)KniBagVGFUNNEX~FnVZExj5FX7Lcu3=?JY84)a>-VYxM>|B#z6R~%>MJ0tPz*1=lg=KO_?LpSWKlc)EMoqR zeR=+ZLtx%a4!(USBGIn|8jFe_=6ClLwS4g?+|;R71R4$EX_a!m?WPAV{Wa1J*8LIo zS7r}tGb{4bO9>YD<(TYl5jQmSk}i9?l6s)D=)IpRy6seoKp?%MHJZblmSFhD@88~& z=eRLT*{m#UQeMX+uT}Bg?K0{wO-)Ymld9>I4ZGvW`&ztPB#7A5O^iTnMIJrMG_>9N z^kvaYzRAFZDM{DNdx)NQUWI*r*RheJHlM?9IE`fHnPlX;(~NZgD?!Ah1sV1LLIjxN z))VKMxA-snw5$){pv?5p_;UcQe)fu5m&nqW)(v}_w$bRr^-_GYDoDX_T?;T{7~IB_ zZl?ZfW(Ut*^s1Sa@+l}Nd~vwxp6*>rcLy&!UyOMgM6B#1SN`62mA%*?uBvb#j_sDh`U z)1O%uALC1W}$10dGe*5 zu21pZt0hUB7pvTk&Jm(jWd@#!y)U_y0zwrgFc=&dGi%?IpXGeBA{K7QFF~mgS>LnN zF0trV>ltX6;OWVm_BjX8_xd>gOiIRWz|q2wP<=O~W0JRm5p)%yDhC(E)ESS2V8z>+ zZh?NMcbDgx7zA5@}>E+LGHQJ_09rxeBLcQzf~GQX?&ylv5tDA&dhOHIz(+{!rT z)3cGbCC}Wq+KjIo+pMxM^TnmLuo$urg5&r-C-BA|N~Ih}Ud`j-%p+49GggCeY{ z2gRzeQ9v8w1P2qqVc81y8vPg+zG**H)bMV8Ip#vg?+Ry}G$}Iig?nodZIa4`h;`1# z715$Iqt-@Vf(n?LiCc-F`-bb=BOSS?7AN8@td>h2fjlkA$t;WkolJCDTucQU-y&6} z)|4VF@nP6O&{a>70VsZaEs@wGKP8((Kp*J0Wpi1TD8&hnKaMdYQTJj#_S&nf;i~8W@Ev zt?Z4}Bn?u5^ClQVqFAb1Cp-A8iDR!ub5D#Yj|KbHRYEEBRlf=>=7Z5!x z4q>rH49}?Q#~5og?6!<}7=`dvtzP?g7BN>MhFrXVxob2I+c*$ng`N4!VM1NXI9sHI zI8=M>!*Q6{SdlVC<2Sy(QRC++MVsp%*OVoB0VFc+9A4`PRU2 zCgAR#x}{~2)CsDmCTUny}zzv60-Q6tk*v{dk2?v<<^e9mr(|H!HR`Ww| z-(Zk@`AxLT$tb5?Bww+RCc@71i;sSb9#4}x%)!M$H*&jRkE0UK*%vr@lX?U_2e#06 zN>a?GrVjVwCE6X7k011xvidf^&Tu&$R8?ExCw7X3h$Rn|X_NibmHs z7ujnLCRf!C)NUpltsZxp295;!yu&AJ48(jTrd%(=_74x-&FNxUiM@>KcS$`}rWezc z4w*hB4}LnWejR#e{hCvdGckRzsaJJHsn-h+>8RL)bo;o%xo%7$voR>#+lAi?bM01& zMbKynTf!&rP-&rzSy@U9Q?jbi);8>yU7z5ualSS#PjA9;yh_j3UbFgFbhrs^uS>5o z(ai=!heq-)%Q6}}7_^2BTmsO|Zyiops%{wGF0wf@CaFscUN#yD{nA`JeDqN$>|ZaQ zHYaK9keJlxa=h@keLe5WwWZd*c$x3+>Cu~F`%9MUn2Z0cxwG`P$Q#D)tjuxN&u+b8 zl{NUH>CR8?1azKzF>us(x_Ex*+U7V^dB{X^k+(~=@l9ynxad)w>Y`@Uq5SHRc4g{j zUdgO1#(-sZS?wf)ac*I~`dVj($NI$Ts;b$&sXi~L@`9@Dtw{#_*@D5ddo7G7J69KriVrO?J6u}5m_6(B4>5bZ236#rYA^5J>P03}6gjYdjwai{!T~&VT zKbmV+2^afs_OI>cE^{rbYMpo`W!ia1@_iDj5UZ(G;o|L;`#E-A7!bWP0hZBYb7l*h zG=Hj^9+_aU(->Pg)S@lFi9B5DEK4r$qd&a-`eSpSbw}$?U5<5gG&1&Cl9|AH`$2Jg zyjv0Jyq@{!adgIWg%sU1A=~RQ1TwPHnCraKwT}VJSAv*QNRMewQ@taAGy0j{0 z^z%$Qme%i9cZ{0e_$Unl9h4_gKgT9q2#Wa<Lr}z%~;^vM$ z`6N#j$f?brFG}4zy7@Hc_OW!*-dZP})LQO;yNovPbRtH*xtII$ea~Trxs0%kL(g<; zVUtMUEq|sfj0`atWJL~g4yA$i#&_Ewa6qu7NeTQHcGCWSS3@FE*7MIFO8)2W@H3bUC1BPWX~N6rch#E~vc?hz>JbSz zYEbvBc*j9S@3U3!?qS@d7wNSK&vIUCqbZEP-Z>JfBf#}DO(_(GvB)l-R{wZk)S1hG z&FW!7g%a9Tl-_?rOeE)$rrV!$jLl#SW5OPmQT&BV?e-%}0Z2 ze%xdyf`!A+gMmLozC@R@3e91T? zBBS^Dcm>E@JY0sZ#|i4CUVP>F`aJQX`OdrC!WhP}A0yJ{n>q|fl}#ZtQ*G>7h$Q29 zmdGzeJ>hxFLy<{O-KE=*Y*uN_PBAL0X;6|!caR-yJIi>&tEWsHUeVYhD?76eq2w0B zGjMuq8gSwxt+2BDIDn-K6x3?tpT197%; zJBKcs2J{vqpWqdx`A!*ce-rkoN&BLXY{v4f*QdkhBeAh|Y6kLYGYF-!(%t3(B!&LP zvI)g}vNR<98hw??uR1AgxH59n1`+s7R_t;p}Dqy&`|`h07;EC)7a;C|#8*bM|nLc79z zm@lBXmvJd6jUzcg;@m4n5^IwaMUw6jN%jgN!JmVj|X{fX_-gzZYrjUdOgkMH?J zKf(P>Se3W|A(gUL!HQs-+$-B+#xTnx=tO>Gl~wE<#Sia9^U4|4B@Xn&i6z8soKk?Z z!)uqL2)#9~-!PV5^74^PhA(wJ#LFn}%yMylIRFj^!whf=wHPFSM!c0s#@TTViH zrhI$lAn4rU?3lbzyz2|wn%HrF;B<2QMeQKg!ZejJfyEfJFz8q+D-kyURPipEwIfa~1C(*NdMQue3}udRb@L9p}`oln4#(RO8FBH#-+B#R*<~W)Ys;LO^fgEly%Tts>Hw+G?Kw z*2H`Bn0nVPA|01J!ycHRG_+wg8C@HeSa31F-v7Mc&>Dihsx!MkX;ahoef>TkD1$|? zUWyq1K=QMEq&l8Dc|Ui(uFkhhA_xdVTh3mJzcg7WH1L~{Fn$I&c z-^4p4B`9{(q%`}9gbX!O@Eft9yRpVcR&jNi0C%17+u-V0!tKsVvK}d@J>*X|4B8|@ z8XmI0`-VoOM4OC`YI&YJzV!X{_R{xEc%k&y*^RJ6FXTVNo&40-&Gl67~$t4N2cybvL>!ui^w7@Ud+l$R7vREM_F%Phw;4G zSl~_n5J5M<<3&8dh;kHhZp|xrh@6JRN4EfZ~jS=srqyB6j29=4rF#gKmFTR?V z?6s12Mo`G`xcbha2)*DWXa7FnDsn;)etptX!qIY1KPsewd1&rl1wcMKKc|P+&+w}u zqi;Us=dZ-L*;&t3{jX-bUSVrrc(rledl&QDVBL>Bg)rQ$%gq;?>%J(QkKpw36p|S% zjWJ0@XndyOTC|!K+OogCt$3>#)d5Kjk+l%}#E+wjsy6aX# zG-S+zCF7R@@MZG#!RxFM8z^5sNA*|^dh0x1>TRPGiQI6djN6g@;({+CsZZy+m2lM+ zqznj{x2BEN!pa)2Jtov3%KY~D5m~k8wlZIV$2;DO$i9$)*17;LqYoogWIvNEkQckz zY#HzBo863FQATn?2GPmm=f_?1A7hU?o21WfgpM8kMvgN)?rO>|{Dh9d-}U@o$^5+V zzx0Cz*dF-*%hmt*!WDD%e>DosX7#M_e>~WK-+{*sGNSM1F6pDfiEIYTqr$)==KWB& z_lfu9*HRq|I?jl;$)0kqFTAzy(GWHTC75$+QMDhF0+ ztjE+sZX|2;x9+rNdyD*Dq$E00&2*{{=;+%`A1@`ibchz3Uui$Y1$9}xI%^hVt_q4o z>H^Za4d^G#6HoqfJWNSIgY7~)#s5*G54sreO`e$wxx<_&+a*h9SJN=gqdy!N>plJa zvg^2XV)oWI6h06>VYAC;`>ys}*A2HkTV17VrLJ56na$P6c_bBWwY-%tR9+3PB7}<< z^wD^&IyK_(?#r2{_kdd1PtO~VcLM1Q+P%=G)XkbNnj;LEjQ9(Z$oX}OfiFYB^& zUhfw=S=V74G*^igcGJ2}XXv8lwcByyb#g9#fFo``hQz&HL81Fvl9d`5G;!1atb^p>vnZ#BzEcoo%UI0yb zPK-IevhhlF{V`W69UIwB<1N6_o{O%9Su9>*qXThs&t#cg-R{FIZ=9UtB73FSRBLMj z!YBKK&SPv~NR<8Fr-8&(cq#Lp60wL`xEq*y_sZ_nlK{=}&fS@Ntiqnj&~46Cee7?Q zFk-QEqMaMDRdVZhNg|C?%Y!dB8#YcmRZMRQeZn8hB`+c*WSa*aP^|Oh$*gU!x;uB^ z1)b7G3t;Z@k4x9gW_w0?ki_PHMG_U*KPN-^G$Y*9B5r{7fzCH`PF^d6mkS1Yn#N(+ z6)4Q!jfO1pXu3-h8!v$8inq1s;KG^fHORMnE#~yWiSnhHq6hnA15_XbP2-4t&sT5# z`u86)=Jqd8UvL-cefmDaav}dniF`}%HcrG7d09JbC$|-H+t zsF6p(j1d+jHCjmnqujw;wYP6=NeI7sE*-7X2ndXChv_vnMBaWm zlF3grx2Q0C?BCF;-n!%ho@9h^y6V zzU!Z^VW(A$d9o7Q=<{?LQ4nM0(hIU{~-cbJIz^Og%w#q^4Y%s7n1U>1^dix+OLh|p zr7i{7`5E04&v!R77#|iDnp%Kb`D&$-GL!y9(q$3o6d`o4`bgtmf;-=N(g{}R^v7)gy8Leu2Db&E zJ0(xl4LbRQe_AnjDKO-Ky*ZR2=#4iE`VP4Zu+~0p0iXQI-rxER!8Sn*D|-zEs?Ocm zxNHKC0D!N`w6z0IT?{O9{AR-#Ei>k%y7gR5L$R<+ZY@dmHOKBg?I8T=fh(?kQ3 zxUD9TQq-!-(7f@f4tCb~vifm>!*bJ&X$Y$aZs6Gk0IRsQd1C&i@2EJiP_17gA!Pqr)rPe_jW|3Zm-PfyRnWf3hc zFUX{ah74?3Nwj!xhQkdLRE)-!n7bf8;3Av2dcM11J!|Gpb{F6|$Wow+P@t%s_Qx#^E5 zI0Shg|Gr~7yQ3rcarX2K>KJ@`_!yGJL2K@{$oG6t9 zT5UR1#L$9nh^_3D7jHZrVv)+dm`s(TzTl46u><$aAe_}nhm3lt$W+H43-*?|I#6*! z9dA0=F`(e080+|7n?T;{cx-HHb#qaF5>>p8*Jjh9*DNbX%&e{3zbd8W=mL8IS%Hpi zvx9pcww%Xix?X3>tikmC?Y%-%*~Sm|hp;nCzm!4WtP6@N;+1Iv&+j0-6olRIq3eR; zH%2<$VKcl9(`SfLStuwdXc`q21gJsc8ylNPqgNhx;4kn%q^HEp%*>brEKfgpdy0W3 z{a|SXM<%c&v1^~-7!*vO3+90xzX{UCRn8jR{C7-b5{3gm@_%fXW01c6o&aY^OTgIJ z*wg2U2RJHXJjFz7?a6e&KRSRr&!wHeK8FNFuLXZ_YbeOUA0ZdO3znxi2Y`VD)W1d5 z-SMaR=?HiVSBVfE2(-V23~~am<5Hpn>IWZrrEk_9P?&=w+QKP2SXmSJ4M8UIWaeLX z1J-)UlaO0~F>W#&2$r@#l!HTa^-M}i3OG-Ih2U_pvA=&l7GeYk6S!N;i%qb#Nl)ib zk8`N3tgN_yXh-l+@Ot<0$D|*^BkHZtx6?-jS#_sCt<_IOxB)%#!&Y0?+KFVzodYt7 zA<_@04KV05=K~%odJUvj?(5n2y>J`dzG+#pOkY{vEYPO#AXB>|oFf)SXp%D1e6RU# z^TA|h^*xf2J_*cKF+F{tzXUgPa3qlJp%MYVY>(9V zS*N;xK^QG_)n6PvHFdp;gDM*0Apdp1VM?tYgT)grUE4d3jg4(ORCjcWpD>A$N$q3K zavh3cuHHBeWqp166H0AmP64!;IyXx|;IT3b#vzQIqpK}AA70j{rY?c45=NixU#qUH z<7KWfJNQc7(`N^sLqjwz9C!+72Qm=kRqr@Dy#t#A;J4$51$k^@>)4PnH0p%xH=d@X z7>8!>*MW`asbd105;I%SGYD@I&APJ4KC1&d7>WRu@wZRRA|jXJt++zMzZ-=-GB^ei z&Ba6FBxr1H6NX)i!s!*%tOqZ1eb**-$i6>Afh|q-f=7RKs~Rn6ToI4=6;B1YcTwj#^;f!)iu(`Qf!@2?NzhKFzS7=aZYZrz%GHUV!I0`_500-psiJ_Rug3-|j zo*hJDsmlyNJFY$o1rVGHgPRbaLjuPiG$#13fL^#sf=Hw=x<9xjfbefjN&)FbwI$19rpdKUfO{+Ps7IkjsB`exfiwo=>!lMVR0TaFE-8wLj$P zq|dgh2vp|}W@S!bmeekL4uQUz4ybyv^eNh0);-zzmtBc*0>|zXt_Oe{by#&7Tz|g|P!id;< zjKP)!g$x{L#8`GPaT%QVar#n6_yD1xh;M5I?#Z);$TfJg9-IGU?+1zvjLAc!YGW z-x=O|2w0_F)xh7ct@zXbzw4DXaWQ3X!6kmL_8ft==m{U?yxr5{|U&HO

t^?vg>AY@fH+D(-koXTD>%=CM`Gt_d_D^`jS0WAvenpi z2#|@}f7aw2n>A9wi>N{(ng3=b<>lc#HB-f_s`yTgPIQ5kONQaDMQT+)AX9f-taJL? zH}?49YV^H)&w-sr6E<^KczsCp9r2k2l^^#fcNz}%yuK7qRWvd*-MOchXp;Hbs? zlu;AqW&4xv9!sx=z&?)900DgCziW&XWA;A;Glbo`7#1f`N%f^!`pjvg9OxUIP8g{1 zH0dvSx8-pFsXplih)-ZIZ|$}s@k{C0rHl$Z#-55WGx__IIU&NRSxzjuHK`+o=B}w* zgZkT&fhQTN&#)6h!G!o1z~mI{U@xCQ1vz~NuH>(e{+WYcJp5aPmILtq;mKgHUQ_{( z$MJFt>M4;T#CH2bh86e^k}o7i488x&!CTP5Jj*F{2|NVUX2jo<_w&i3C;YBKJ!LyK zhzg?L@h2E^3`VWBHW-cn83#Y@-+^OeUFD&)NbYxi=NOfzSQH4d8viVrJ&Mc%^WYxAr694q*}eCLUOPG9F(;f0pkT zKE~uB)IX+Weqenk?dYg7WWUR{8?KfAF(Wa8kg{)<(vnoP)K5Bxa~9qMA;mHFFxD3Q zi@S`qGSxv;`#@97>GHD&x5Lhz$^z2z90L9L`6~Tej|<0p(L7;A3JpcB`;2R7HsH*) zuC=#&F|TLYa=eVJ_=8YdblXyPgT2}?ENog1cyTYGccw?bOd}Dp6o-6@$AHrjVSWdp zvwun?FjVlTVbs+fa#GCCu{Ob!%X>Pn~OJec7Gs;hpOk%fKAZ@f_RUAO1F4&^G$ycfN2hzgLnT`9<84JhfbMPVI5Gj z&E~(-gbLgJKN51#T_gvMh{y&6&YS-fVYGN$FMm_oHb3nC9yo*)2#!<#ieu*LHU9+% zlj^C0+CJQ+}EvhC;p7i`IAVne14tF@_k}4_jW3-B|%}8xRDSJMdo)J4*c;iq|s!O`1o`( z0hqz$QK*n!Pn7^v-ILV%L)7s!?P~u<-S2$c^EWy@_FIn^WCDMc&7|oPH*mAR#3#_} z69}N+{*mkT-NCO=ZeZZh3!;R8tGK^^U{U|y0UEygZxRk>i6dXdDIFf0#_ckg} zCoN%RUOIIpxsx}x)6=Ze4(9v$Fv}`uzU5_9{8FhJa{nnVlH>(2Ektu4A%qIxY7dx zuhz}@5}eHbPA=f3vA?7=D5Jk4SP}(PGXVD6p;vW(i3HF+4K;O4PqiV@$>Guv0ccA= zr>DSo{!;>lGMS&9`}~yD zf&Nb+rMmGnq@)_=A55AlrNJwv$F@U`$MSY{z2?vKwNzfm&Q=R#g6z)%fp`3D_NZ${ zZ{FYVHh3%u$(x~nt7t31F-I3tP4Fw6XE1}QimT=RQ)LZMtF4|Le5yF$AllX;#674T zymYVup~|WMNWUO(*^BpE7q~Jex-W~&IPtxIh)!h|nxqc3^3`a)`lxQ!|>L%zMS4+ez9w1|2_#7wY*+9g@zQch+TN zC?%)4XXZ?>oXF3P?^SWnOoNg!D!w3h850R0nohj`X}{~uEF}MPQtV2o_F?`Z;Uert zo^|=5CZp!wZsH#7aI+U+j}r=Au?G1w%>Av%{kzM79sKKG9UNSrxjhk0D6@lwwK@=Z zy=l4NPf)UfQeT<1d%d9zu9g2jj8o8MBEu}m@&7RQ6;M%ieZMGzfrxa6AR4o8?BQA+a_&F zNV8WT7>L>kw0$Z2@ixoi+vlPt9(b@PQuc2Esck=bF8SF^-VB`Z>xPQJt)y=chsdcr0jGIL`K{r^hX!fYIwtPbA#43Ld z;CLU~kgckw(}0E)XHZKgp>j5W|JFpYFsk>;X+8R*x`3?_ZWEiTzIhRU%`bSX^US=y z58)$$m5x9^rB6Mf(nNy~#n{z49s%)`O!3Cd(J#FM1^h4-A+J6mNNsQe+-V0<=S1um zqcS5Fmr(L-Eq{>cC1-*~h>nuIvyk2+#2W5}a;unr$m#6t$mxjCB}15SW?`WwYrbzB ztz}tlSs62if+nPe;G!ZctEHRh&%dcx0*I9sf`MLrpta-8PA4OQUS|^0p2Qn3oG< zZgVYJkB)p59aQO*WyR!p$D<5(L^tlyK6)ZP3SXT^h~&um6VVgK6Y0}QD#UQ3paT$G z@gB)eNGlBGDsqqb@DYN{OulJ}M~0m`jgl~4egvg&x*@H_TOg$A_-$e4pSN~UOj3VL zK&dV={D@EsA|O5lBkZ7P{Z6M=&V#!bdnuc)$cW!e+KV0U7KO zf5<*`$c9`nPyNO}GR==C0MsN2IVt^J4pfXG7jY4+gDkb|`9SMzL3NEg&j(a`upwpS4) zL%f8W_N-_1^JW^BM(!P)H(K%nr%+bh^&tfbV-2z#fiJw0yA`r>h3;YP=LxfNQRhu3 zDTz>E=wV|Nf{TiNO!jK+AZ8U`zAJ{hibTY8x~>A!`%`aRgyK3Q-W0dggJX4$S}qaY z+(l6EcS0;js)Fx)wt4?P(aoO=yjGWgV~U;6csB`l`+lhb6(lbaAar6r@=cEm&PD8A z1X&^&CeIS(aw<}}n0K4LO8uR4sjN0OR;*T#sVGGeFMA-&SM48c_A`|+WrC*xy z5~{Pav(mQ)Am2=3jDg4_mRIvJEXxruk`y+1$1WTP5j3D4bp7q{r!}<{67b1hXN? z$;ttl*`=JE?2CJ3XXHKUxJQOPi2Yo|>5OF0i$x2>?CE=AGdX0^i#3&7NK*W678>i3 zA$DYczAkdXJiEv#Xx}Jq2ZFX^AeNf|r0hqQgZgdG8u?A#B`%g1ep|#bJW5eo@sLK4 zK74l(xfwG4>DJ;8zWI4K*E~RaBCW7K#InlwnpGAfmPxUPk<|t)B85u~#7Y{m8+vfD z)PiX))Qi+}n-}xIPsnu`C>x^obr%zARK)I6Ln)iUz?N zw)8eGxi(|5oSpE-hERjy`Q+EuK1U2M_&=6>tt;f*%hqRjUdO#Tr=jG6VJlP&J~3ix z+;S2jByN?thkHvT71mYc2EkxY`FcG)bMDpiqFAx1F; z0y(b=y-eyLiDg4Mhh+Ow+}bP*?R}EkI|R#D7>~fU4C?oQcxN8HSm`SKX8`%6Axn}H z&DF`a6xR&@&x>e?@lV9!$xM|`%KIF~Z&76WQ>xpNWKTsARdlvL^ zQ0@xDr^KlHNkf_U@yf*TMs>tiZJXxx+Fm~UOd3!ZUjwYVSKJl}%p_e*oXF`qsCG%& z@xg7!YrIsTc(eaSBkk==)2cDKV=?9UL#!gN#|d|>wbsl;Su!bCSMhz?62|=A=tI1YosO@SjYtvxe%!0nJluS-TC0cW^o_{7~H1iwm%QS-c3Rc~G zj?B_?%Q!#?SjB0Wsqvuad`P z?e+VlL1*{Qw4U?F!i&~(Pa}O(=w%7tdFnOMpRgqQ)1-+X64zz=%_)YItMub50V|3eC+W4QGtC%+saT=x zsdQxqqmTxL-etRWE0)mrmq*!hh_fHNc@VmkqKbcw3k6y!vyMB5;=QtVh@3-bcR$DL z^WnADq3GJ8#r`;F<@@~Oz|nbX3l2)0zQ2|^0Mh!jb7o}g2zFB|7l1X=TwH!$*qEHs zBl4=ST7M!Dniyv1>6FdQcS~k{$9Nec@ED$4k(cb*SnLhmP`*Ru%;5bJo5J_(QDpZ7vc3( z@~MMP_SKEd;i!3pi(cv;;8h#iun=e?tBJ_a+sC>W4819Z9biKySuMkzq4?>!T(x2< z!R{m0(*>Un-kmTjTFWLjspJOF*k*-@@{X5EG<|OO_$R)sZcV`WlC`7yx)x_C_VR zV6xDXsVpU<#Gn!JW!23IdL|VhE2aG&OGuJ=y*98!H~5v|L+EK)?<5bk?hJg_i9UX^ zJzZ;eL(^otl%#ovzd86EkZ=k-=~6kYc*8eerM8$t?=n&}&e6A0Hx&(0%hH}?yn8yI z0GZ|PElSy?faw}8!~t#LMen)X_8(2zy*`uG5;P7BunND@5-nR9*8)g;^yxkM)ys31 z$vHyunPr?wjkttX6RVUfWz%BDDvg*s`RoFxbhf%<4Q{8akljZGhwT~o=HtF4GlPY| zr&Ak`4+#Vlp^bv>$LyV{UkiH#tq*?Ez`%c{%t(3~%=MzokbAA&@ zvT=RbhCc9&oc6l5fYXXeZD_>FOloae;_*14Wk_v6wm(eS(cN3ka${O|<-iH(XU?-A zF*j>PltOsAU1@QG_;y9$%2Q3%Ji?~9VT(GMsH^QEw;n`%ZGY+6;4W~Q`3Qqlpte#1A7&EZmsJDdnJ{*lY~I0@CqEtizDYY;8hoQfN*euXi}TFQ4Usxv3?m z%U)yVL)W|o#)N~+GCAK*Nque*5$=WytCVQHZ=jJLGV>v}oVVjugz&5x*-LE<3+yqK z7Hrt{5vi9K+Ge`YobJ7=}U$U&M1Uu*#$!$Yu3IA2`} zsDR*H49YE@Dj+4mij@dMR8Mg$;?_s;@k=@IIr+AENY;g z)$BY`!2JOYv(Em>d0WvG-;k!bR#pz&rG)67<>*V2&6!CZEdzSn$USooxP;c#llpX{ zSJr;%CyXg^>L+hS1V96$A4j~rG*;YIT#vmGJA*LBNI8Aj&Q}4Y zJ9W%kYK2am+{ZKSJEwxj=0OuGUib~h34rjRkM&0^9#S&!78#SMk75gjB@QDi74Zo% zOQVaWLh)Y-n$~wI?9YnaErix{`KFWRQUW1zFA};ucA0kD(vlm|#0O*hU%Gfq63g=v zQoSC(UBQK6RVq(u6)rkxlOEe>S(eS8R^JX37xmsg|PpYHL4#clHOyTYwzrCNBbCe%`F0i0+X0~W7Rck9qd z$_uL_Q#Z>F7C756Zs1hJ%gtiVl5u-p8I2r{Qct2a;4wNiEYG5_XLWQL=Ogn{Zd#@+ z(Q?(T2F=cQH#}+oxJ!_y{(uFt=C8p7iTgB{%y`b_(v8np41H8*KK>$) z8$T9L#QLZyJGhBtOT2)KZY?GIFomZuI&P&~GPVK4>mhglwqg@;GH6fIn1zr}$SuyP zBdDZabP}Yl=JdK;@H3isFY#rm?7fhod;EcyD!p#-kzd_WWa6@c4S*glO(}#y>J;hD zQR$@l7c0C}zfRcV>7zCU@x+mh*6=RYR==6Vb_b$Got}Vv?|;^)d>Y-&Op=H0>j&>k z(oj+=Q@y6BqS6L)04!KlO(vet+0IWqbG!W2Uhi;+gd=a=WCv_eem<;)*-d8OP>D4u zxhtC5J;&sJn2q6BL$t$(KtqSfq+bW8k54>xrJAJP5G?#WHwTT4Fr6MfbaRI^dJc!f z93N^@@Ta&uaV^#1ePGNBd-+*D)kB9Lx+;1!8crKo$b3E<9ZWM3*t^J?d3_)Bd|hwp zfLu7f(uB?x$9X-OA%Eri7iz8~bY_Aa5#hA*Po>1tI#AdSSkZb;)LAE?H<9Uv{oUacyUx$gD(D#If{h0EM&Ap{Rj2fmZ|#jz z3bcP%$nUUrSF_YjYp};;vN(A*$3&3;G=mX-S#zE+cjVvT3CPLaBWO~o%C}t~OJ7l@ zI8r2CGl}*2?Cv-+m~{vti)raUDsW#qJ~+!HL=6B}2T0ai(`-@LQn?)*k~pFEi`QD7 zk32gk(4`VJE!6!8&@);6!g7O={j(#VZl?y7M~U_bA(Ud`#P#7A$x5z>~c-V4iW z84p@n)N*(@Xr_`#Um=*`#dg?adaGQ!``LB?q+HIu00b_SJAqDjt~gt8*92Vj}nL=G%GDppu=@nequY+}f>nq#@m> zWOk#^+t_OLNu%lg^BY6Vqqn&L2>Z|x@3B;EGs+H2U(I(Z-(RlG=C9yT z^+vOI&334FVQCETja`;BP$+$s1UR~k*0zu0TWa$coKmm?3p16$i zeUhk9i8m3quNaoaSj$!CtxoP&kP9Do@hKvZGY zVcy;_Qk+A;ma~~EZ(3G{%xsAQ%hsd5$SHo2jIJsX4r^1_06vrCrWg5%{0&*c2Wi{M z<|&mX22-&)PPtZozCqE=xqwmxQZht3Q*hZNyRMco8oJ{Oth~cOOOrCobDBum=l5=2 zS<)-aTHk!+>5$_UOr>Y21Kr*6#nF^G5hmWlCcIH=q3xKiAgG%c^Pc9h&gh*>=3sF< z0oA&`cRmTAC03CA&;wn&H{@oHPVpdHxwnytX^$!!3!UAFlBb4w zPwkz0XF}N1Bj%@9tKxLSAgF=6%BNcEF#BDH?a@Trtq4DFIGy+A^u0xangRi6LUQuh z<2-{f){wJs*!LmqVcMNvx*CYx=RD` zC9wwD;D`3-eZ-8q94Gf69?CuCn^04*)D(wZ)1Jj7Umalb#d|FCe0u4pgO%;kLPc|S z84bA2iBI0HnlCM{G1px0+Tt`j-Oi-)9b`U#9WdpeCNNT{8wRw&KCKH%FZXXN>XCeE z=9GMoE#|W{8#Xxw%Nrf8H`dSgR^KNMJrUWxdupQnSx)&;X-69AqmX89A=M<&qjtH9 z%f>36`ZVLg>1iD`j(lH+wSsj8Ln6(xi? z-5zrlFcU3G3(sumzpw2WF7%Whjh?lMDnK6r<*+d)d`C4!F>_l7Hz(uE^fR$cJD#ss z0Wn!xS|b_!74K*(Wx7^dM6P3``V|DYtXYIc-$eW?<M4)6ndb_OjKrLp!4 z5#tUw;WzZJr**#`7`XlFTEuL5f@|p8Lepc5-g_dmTFq0|u#%Dy3;sZyO`*iZwMANb^kYWIsfjyV#)oTH;?ldGm!2Cq1`kZz z$LOSEfc^I2gYA~ZKGboIkl9ttGpkKmH*VBR@%6h8ttdJq?IqxlTQ%o^Jvz%Sg6H=nvcU(vOPdqoQeDYt}S=lKW>LMe-7_pJEOOyV%p797*cID;m1*_@MF z*r(=JmI5U?^yB)Yw??bl5|7-?n_OWgfe<2?8+!>?A)$VyS=+ji zwMU8?&+fvm13;zM0c>CH+oNzw_x3Q^QsGimp~kuqS6*V0Tym@O!Hh+sr`#?3^ zkkc3I>Ei{f6#?hXiO2+FeYJ!1fR61&ss}$#Zo`&y80%-iAUN!6!9w%UqLh6kIePvY87WjVp*0KskWg@0 zrN>P$#Um#9MnGwor5(P&)D(t1nCv9|6hk{CXk=~ubZQf(aH~ozzp+bfSUX@HfN>Ui z@Z8nsB%m|Zk>s2aI`biCV$>7bh~QYoy5*ICthw{elPoM$FJ!@o7Fkhp${ZY+#JrzATgZ(+#c(%w zTaQ&@X!CHNHgzGX?$L%02=?2a8R09A@|U9u*Cy5V2ecW%t#x`^oFDj2!&ioRPw0a> zwC|peVX4LwD&@7ErfHtrR3@2rC*&RuRAQ+ojf;`M!yGD)c)FL5CKEUvwfwGj3(U?9 zy!6|}*#`(9ou+Pm8kxKlOK zH>b)ll}gVY5J>kn4_}QU)B4)PbhdI~#Z*B&w0TewBhz{_1ZvE#OWp=#oKXnn6m3^M zo|}B>a%%&YT6#JOYQ&cDOj7F50kLfG5v;tMkccqtYL&+sc7Hn$p3JQ|LB9x~Hdwhr9^r;^szB*3w8PPCjT12!J3Wnrg+ zG~gWXB_<$?pT6qRsK*K3d_9!wDM;LaM)(;;aSIk^w7s~R``JEtoK_HPphJ;O;TCp{ zwkYks{dE4ZXHSL`>^kxq(4Ij2~HITSbDI7B1>i)1nBYYabv z>1OjECML>dN)>aS#3ne^>Z~wTaq*nNGg3TeK5Es}KS=kzCH_n)#ff*23#4=JF4%_D zrH7R~C+Gm9bpP$ACDJK}gd0hVEj}|2#0R5`JSt`;1m#Qp6%dADi#|5a#QtUdwb2`1 z-UcDrW7h>>$MdT@e%n->>Qe=2-&*Mm(#kB(b(j&P7rO3x^WL!wB+NTmGE0&s z8;{-`40k*cvZap)zRA4HL3%(aV?8(ZGMSB|F!{rjg)rRL^Dg2zWMcGL9?Y{O&4Rn-YJZ6k#$^>g=Qlqit|Jf}XtS6+D@!_d=hJv+ zs9h7^0~L(Dt{0evo*mawt?er2cU29*mlSFZ6d3Wc)OhzgF3rkuSX~2)`POkeU#CJS zHsagJIgb+dCQ!KBa-F1a>K7^ftuK+5?J-*YQ8B>}GOtQ#_4b9Fh;GGsRGV~8?u}Do z&MmZEsiSzF!)+rdFl6O__gVfY!Ux`}CTi7>o^r&w8Sg%Q9Kw#Sd`LuNYbyraYC2)wU+If)qS3Ql_@w-EYTnm5=tKka~QQI+2h zueNx<8~Uz+BYr2IO##Oqb2oLbYeoL%;1hm*RpA^~;sXZmbO$GgN0F=56TuPzYh@6V z;KOPa$>5_!`f5Q9?6j>V3YqQB)e)==%bFVxYw?<5nfQGlch8$96->O&-5*yL-~C!y z(@h!W?J(4E12>Mxf|aF)({Dw9NNQ@qd~TUwdW$AVc#l~sKGr2sW2V~4RHdQA`0g!Z z7-Z~Zm4|V+-7t}`{-IS|=@_YcopwOI4~lW!GjZ)UVQhr@$@Kw?e&?O3U0MXq=1ij@v9Ci#S)SuOji zHsH3rph$L*^!-wBQ)%||D?Um(8lWaA#_ID!3p_c$xQyntm+DSC7-|#=WUk8Os08~N zN)K(5WEB+0XHnQ|zhr_va~k7e8gd2R z3+@i>^dW}!$X{?UHgNsT&F3-d7IRJw zrb&f12kKLtq=uX7tgwYA#{}4oi^Zfz;GVBf^FPgbfz*77?|dK@YYkV;tf+dk&U3X? z325k3@Prcon(&^g%%iZG5;?!4!!Rksq8!5y)>YayWb-tC%|j9SDF`1%%44yz#5dI3cH7}Tuq5zxWtlJ9a0L+V~Ou#<+B zy4lfg2&y;b$QX-RPpQBs9}tEWeO>Vpv8q$M4#xe`Lf#Ikhs;}iu*Oce&)3#iw?EgI$< zv9nkvV-xLQ8>_5j3ujn*B*T)zV}nH+X4!-r5Ii2$IY1b7YT;s;9v>x`7TppP%FbA= zcSWuk?t~X0ZDl36RDRmko}C)G;1y9I(fr=OkQzv77_EyRC#ZwbMqOF@zZuaJ_S_a-5U0K+m^k?Cw{edyiS6)5-&@%8t&aFpRS?FeJEL`S4 z)@t9QHD2(Yv3J(1EbZlW2A@BG4=GSo*X9tp6|eYCb91MOL~YE=LcZ8s`uwSdNkhbc zJ3nJzOo7k2Q~?NnR7hm8zB5r>s(eh_3P#KZB{BDGp%maGNSX<;Wj`@q7-@lhU3+Lm zl#tPG_qGDd<2aZN7OoYYI=ux$QF8;g#aoeRZQdBK19YYLHiSpH4T{Z(J zfD?6VA(plZ$E!K^h4M9p6Gi<|!5andSzj8dk1Ud*8<3bp8(+=1fpz)gnmbltD|Wj; zFU45J(+u{QETUF-zH7K4t?|)sO&n?6C~~lG#I=SntefnPqwiZm#f&T`GXgvxB{e>H zmUj^K*?lGBRAlv7MH&N~fRU_?{uqs`WPXH4J6CB_tsoF=9ECY-B%Pz)^N=pQg3(B| zvm2j@rN-4g%P}?cX$aMaEcnFxmD?`4#lXT-(B^Z6FO;SJ@y|U6i2!SFsQZ>~MDe}> zTMnJ{7bP|?Q?SnA-lbL>zh|PXD;q^gCYtPN(dQ^<8KF|kt~N<+Jhne*nACdTN7vre5P+XAYMJQ> z8e_?nZdcbJl=Q=E_~7co^j>L&-Qf#<$748n2cHZz=vKsl@O^&MWcw?^j*c+uE_Ez*r6Nevt z=cD4wB5=DY!>-DXg{IXRhenP2dsQ6tM?#CJ21jYyo<7gT=ZU6|>mX1Pqs}<;5XIrv zp{%@C)(opt#&&vPIXrUe$w2rNmiJ7TCFArBsphI* zEN1e__`O@Gev^3VIZ}BEJ0&ZL^U-3RSJ^si81*{dDOV~Ez;L?S7qd=DRfJbX$JnM+ zR}@>X-$=87o{(&gNNdhd=UHyeH`lL&aJgB)n+N$Au`jdF0Pp6vJbV9ceJcs$wh!gQs^N-}?bs9Q$g8j;mlZNH&KD3o12c^lU3+hM{4Khc(Orm$tdh*r7@vyWXJ zQlmhOA@oiygZ}7AvlIFumm=nmn0t26)?QRmnX}Kybm)$jHa-hx?`HUh;J%A*jne$7v1R znjqSVs^MK~AE^FAuPiYx$(hXILQ*O1i449kCtF{^TnulWaZH%RMI(tz;OF(-zlP#p9KdP_xRLpv^=-%;i2_abbh!DeRY#s>YH##ZY}b#ElqdT6Z%;eEj)> z$m=6NKRSu05q7tGTzx~`jJ9<_O1|sBjiY2RD)URp$N9l@Un5ve;^GRdk z6Vp50^tQBRhI(-tBfmtE9Ny@%F788*Zj29{!AHyECaGC7HP^~mC-Bm|X;+fXpu|vI z*7iZXlS$0;Woi?Fi67XiadB!~j%5JyRPrh5hs)kwOl?Hw;FPb(Silq4g zUwt0(8AK7L47|p9t8Fw5f6yD+*8*_>DbsqfVllb~v2zpLz?!1^WTczU-Lew8o$MtB&nJrhLH>Ae$}9l~rLhBsxwqU4m*e0@&MH;>~G z(1}x*{n|QHLb0OkeoycGoMF~wXk>Gqbz@QfxNn~Cszf+X|utztl*S>fOaDo&=oxg{AcvOBD!$!&@AE<-RwuF5n>gGqV; zFI~L%B~cW|zDpRn9b=}%oyD3F&YTbWRyXS1t*z&?vPFB^N5up9aWIzvIgZ8TYO~f} z9Kht-Rn#VqRepmV|xsG-B->7CBveId#f1EX{uMeR%2}v@i-)-w~Awk(#dQC*g!U zIfGp@`MP@fp0GUsQ2fz0znFMAU%4(p?#FD+JCYH~6Cw?Y-Dl~TG}kHnm5g6LXZO+$ zG)vUp$~zyGv@yKBa`0Ac^7D(A{KP`y;d6A_PG4C0bBani88>$bJYKvgVIp=LR}D5N zsDC%;=}0fZqQYk9C&>YPac8nm#5Yd`+jrhS1$RQ<0XB7){t8u=GdII_o`LE0G%Qq; z`>SE0V|TW@;_}FWR*w@fx1MKE;JRgEbI?e*doD5L66n#X%7#?j% z{dL?~ElA1=fhNV(km|-d$~-m0R*Xd(`kpah($;HGx`}I3c))$c*CRvDm&s9*A-Fl= zzT#RwHHGNqu8+W?pac!EGh4Ty1J_SB@LDa#`~ILC77Ku&OG<5Db%KL^F*G_?8Ws30%^+PJX=j(G9@MA}xav{BWJJm814V}5Qv~%my1}Ev11|~)nda~z1b=KaS%|-&)yxmgM zr)mOilJ%Hq6A7zWpV8iDB-+#PjOxy?i61Y0$c#E$%^pNqwFPhm&o72DR+VoM0}r zQooPZV|zXPJUNnS94SNhG~NoMlpVNA_I(Cji%kDezt%%1CvI(sb(Lnu)FCXzQ3l_C zKFzYGK9TcMySyfwS0Rm`vsG#A{zj2=IVz>xrR&VtcMWsWl6s6aT{F81+x=U8R?;=UdGa!5v#uQ)r4C3I*|0E-r>hc#gk#g3ro(} zxC2G%rH|+Ix@Maf*Uv}39&4X$ZS^xlfPotZew{LVvb!W=m(6lF>qUbUF9kP0UA~zf zjh;r-sB~*R*M3#gX0qS;f&rcZJkP=WamxD|)J(#qmh-yG0q%?@{j zyl5{~3E#jh-yH5e9oP!+UDc_kxFV-yZsPO`xD*ZbL|y5Xb}xHnyBoK!wG7WHq6)hw zbmCDhXz4wq*@507FlUxm#r@D%Mw}!vu_Z49@q5;er?LXhq~o!M8<|SoPVXt*<2)^@ z=3h|cDAg})wfK}v;=OKD>j8)^HPlQ}X0bk=sXzZBnv(v)v3$ra;jLJpbWh)lw{~}L zpUX(7-upO8ph@*Gq^Hf#icR{_YltEnVeY`ab0(7qHLy0dD~lb(4F{z}0bDN=tAZ|> zS5r%E%UIuY2;68quxaEv`Qp+$cNW@xwtquD19>Woin^^vAYZ5_n@4;{BYyF|{I?6z z6T1)LDJp6ofplNQKtw^h|5X6zl3@1^;g?$1UV+xE{|YyYYT~-K*%Y3*LZ@pP)NEDt@B>e%6gZ1lO|!BUd(cj& zfP1V4+;BfM{T=LOA9$!>Th#HlOwSfvKQy=kW!+*6G#ga5Rq7@y-_^sM?91K%=Xw5l zMJopn_{2ThlR!>x^K>6!IQz#?#xMbB3h{yf`~Q7eE`kfgI2qO^A>dRwR%_GVjK3Er$Q(UvK0@WLWaUv|0&H4ouyjW{zV=i z76u{F$-Wq{B$sR=@BO}=*djE$hmg5NT5*8yQq*+T6nlHu2v`9VGYi zA~R%L_m^`$o8LinWXip6|Z>)i#GGXX|rNLdi zo!zum%w9EE+2lVvo&|isKdQLK#g~g+y>VK5_5gIgY@M zJe9G@hYjpz>j;AL!kr&1Rf0~A3h;xv;B1k7EAA&(JEtPOzMcWtI*n8Kt(AtMx|Az> zNbw<_J%4e#kQ~Sla=uF$q~{;X7G%=f2{LVUM=1jD1R~?h^wYWX#2DEA)`5^Hh1}za zZd=Nm%>(Q`Xqnh1|AQX;-m9MQTdm_TljJ9EatbUSZPv9b*2yk~GBtnS>E>3MlP-mB z^Sp%i;AFt{bp-z$5I1aQJN|7?{~V1YaKnR8_wEACdSumC>iDOyhyU|9|5h!vi-zwq zmz(Va0}G(JTL%?ak_T0JkAUP!M?SdE>CumyF z&SYqon}PFG{lmnOcp}u?$Zz06f6=%d(vv0;;py)WMY_Mm@DKf00dL1FITBEoQ zmeBsUMyo-j8lF7RBQ<54uyT|DO@0t*mG<`{jrluwvIOrTxs&fRk~@Jp4gUp; zASN&aDGaNsK(iGjcM4ehul}tx`YEwsdG4C_3)*Jjb!`=Eel@?||4J$$2irXlDHw8* ztPO9VDKz^N-sleu0pWw({fxGgBSClgv);!x%sROx(;j-BUH4dMbmoY4#^l_yS{iPw z8!xEfcv+r##!PNp$D~!=hI`zB&0 zgm4&rs!fopMYu_8*ta+nqAS{ZIm)J5TURcGE-3lIM+WO4I|X6e{Knyb9i@Mj%RY_EU#02Kf#qWiHj(%kcyT5#ump92q8$;F zO$0~4l$|@i2Ye3(v*gYH(xJJ$^NLUJ{yOE%V!=DVMRe!3i-{S&f< zzkMpNKj59?&uVz$w#&ia8`%ouodBu2&eO%s zT7RTv`6m2aKJdKFxUiiy>lDbJD1Nhub?%3z(SI|ZfrJ4K#vrBfZSn&OXa3R6@Z`ur z}S3}tFlHbg6L>7az-z;}9!idND zX5^b8O!MC|_E|Z?Qv<0igX~~=YF!sL%g#}I@9vT=oq!6`&KICAll{-u{hwRn|9Z&) zymud%oOyFttb#_<_a=HKs{AMY&LGXG&v|B|lA`zxg6*%X6J2^M0uPFmoyY^{0<0q~ zdK2WxLA1f7B9|54%5Dmqoy&hV<(fFP=#ge9ef5uu!;H}G zt?k)n;5oN^!|H=E3AB4lYCWQx8H#qPde96|B)YvjcfFnU%X zo);^>!b=F5>dX|Al!;26oRZ#nXQ6pbhx~T$?EB{7HIHeV-YIQKN}(0Y-2Hhf4pYr~ z=~p?3|Ke;Ar`+%~E9so!!l2R0im;NAh1i@|)@F09VdoruEsd4aR^?o?cdqop>5LSp z7MygHu&J!UV&0?)JKk6k4q8{iEG_#Q{p`+-)T*AQIMz5(Bql%dNMVN+%F4*f<#bke z9={b8MPR>N0KkX}A;BLI*e?ec7%vwKM*oA$w@m(vD=G-(pwHeP0UX-<>Sr>nL?jl` zujiiuf9kl}=z02E@^46MsXtR+;0QO6AZ9_b$f?jD9YQE*2fK?6?v@)!Fh&mqxakG7 z@6?5ABKU!HQ85KXC@-rE1$J?v`dR3oIsh{pZU<|8$m!Yz%FOq4|J7jd!8<#15cuoW z9lw=|!0bR`_5A5UQUP%WAWeS+6v+=9j}?d~68}T*XCA1C_IubT9uy+`BQnzd0yX4k z=>Nd+4I=Wl5&5t4_y*ekTMmEY6hf4M#Jkc%`Gx_D{7VjEz2no*M+^jB8Y0g31JN0& zKiq$CXSb}_SVd%q#Bzu2A{>Yw{n?)t6UQYWyKctuQe=SLt3Sx^-$X%x$s=LHz8_cq zrpf;hEc7GQ1)?@L;`l{sKZE|%fjZUb`3VX6d4Y9y!5auj8D#5hP*A=GJ z?!7=(`^7i^-bIC~%gxQs%`G4+EA+ius7PG!-zsDg%(xU(GJ{RM2 z`*$BCBlJ=I?t@H}D1^Y0Jv~NZaUG41BQTZ6#}Ph1j=v1=5AT0_ub4PRKrL(4ZYDwn zs{7P0CjEbSd4+_?L$vD?$_08P7YfQnqh9@dM9Af`e<~PRU&QgNLxuh+%HMe`mU;Y& za?x#Ah%18j;l0}kS&8hVgoR%*{*CKt>_KYc|2dxWn~Sbb|XXZ?}`ijh^^GDU;3$7?)ku&~O+M z9W&YEVT5dPGwaG{*AfozRNr70h%A$%lzOHPzpJ08NgAR#rcH11u--{Rg-Yx;dT-Ra z#1h`SfQnvizKC(I_Bk0J{SUN}K-6n?HaePw<(1g);s#?1&7)ySpefoL0Ts+5&4dq{ zX?cp35+ZL3*B5Q&>I+s4zC2ppHy=vWS&JtiFLO^)z#2$Z?k8$7;U!_QbbB0eVi%zM zwo;MiX;KlT!ebMYglE(CSoXHzE%=5MuKNc4Eh}3OdNF4{*xmKF>T_&lw{T=Vjrf{r z2os?$cRe%6b*YHhx_w={fG0P6a4P;XNlkaTN!#MX224@Ono!Cu0T6r z27GRvTP%l061?G|f5z>utKfcgNOc9})$PkDLf0;#P+ht9)Ah~sHNx+EFoFRuqx?Ls zAe!{!xJ~uLg@PjV>Zki>^mi!FE?s{2?Cy{AFArCmaQbIHvP{Yx*PsKdE@ue~siz!e?_I3pyu$-H}inI);%_sdTQ!t2W~Z_EEf+*?Q0)iaHwP$*D}yA*eKcR0AaLveTaV#T4j zySuv;_u>b4cXzw=vG=*(`~B9sf84t+?6vnKlgv(baxycSWP&a5XCnT81`-gE4_3UN z-$m)t)wAe4D3|ZsYCfu^_mda#gF_t~95hG2b(a5T{vaRzP2ykFSR2~uXr5Kh7&63W z>^Ml5tS#)LG?s1aX3DIXfh~(pMSc`$;>0yywBFwsNfRaxJ8gY9GpeWTSjKkIR?f0i zpyls^gnb^}Udf0Vowh5)$doo{EaH$1lgq{&J)%$cfs#4ogQr`qOK6&bFD3xd>rJ&A z5NolI5Ax~Vy>I;C>XX}<5^VdaID$A!ZyF+TNeGFOIAu4g0%e}ESSiAy>Bexq>HBB> zs~L>cDQ7Dr4eS9is0bnskecf%j^YKi z3r_-_;U0NBptlcveVU?RJ!H@vgvdi`jueU*u0IWlqa43G^EJ-+VEg%_-2) zu4S&H9Bz!uL`M%&SogA74y*M7f~CyujYFlmeS)o5EYK~$An2bp=P)|hLA3c83eBV{ zJ~KPwri16bLOR$>r-5Zcv|^1P0vw%+5aQv8>Ym6uxfLMi(h_S2slldi3>7+`&hxN# zi0URX{0VeNlG!F}Ld+108Wws`<6hv3PSuSc5N6>rI#nkmtddC5F@2}71=x$cr% zNWUil0Wg^(N(cx>g9|HUuP%{+9tBw)=zVa}inDrx4CVu^W*yJ3-ixsN@}a>KNgR2y z5pVM>N@pqZ%ah0_BskFXC!!l2?$y&|mVnO|vr3I0>_MW$%J@Lg?vkPha70OK+Mo-TEzK~2}J-X8A)5jJZ3j0nZL{IS2F?X4Xm!B-KM3SPgALUqgj$|V(gT3 zcE@HTn+JB*k1GDS+G<%ZDC~+xetY94m)zu3* zf~wH4{9=?mg=h(33K#>7eunUWZo(0DR=HN#rb{A=tdswS8-54=ivI)tBN6N5f8h5F z4dK5ti~Nh;e<=S8(v50=qNB2k?Grz*Gdj8v4ZM}fEXG*m#=B${M+h3dZyy%>W-cem z!-0ppi_&P+SR<r_Lp>4b6VPO8J;DZR^NG zi(YlNea@nqw{a4|)1Dm;5Q2CQRH6KsFcL-rV#J)Zf+RNcg1I34AD@2V64fm4KqDfE zRO@1CF&Ogmg9rWKzbY~(f<%7z^MDN*nY}M|0u~|VNAdu6zegvOyyaw8PHGN&c^jO> z$%`08F9fMDe_IMPUXC=W-9XL_8nEchOn(dW6tQ)f?VS*x=en+wpyT$N5Yn&}ikka) z0LXPceEKyI2!_HW)auTBrV}CIri5W3f$m|&eqkhw_{b-B%eII?Oc_S0RIuA3Z9G*m z>>0!M8KQ9h?$<-6iA}(*E~r(<;v#EAuIc-YU;ll8$0&d2hgAF^prI2WAOMh$zgfsl z>2pf!l^}j-ksn`k4|5FPZh(KHn{o^RJg7Xlx}iA1E`&xYF9%Kuyq^x>4hgLMfpQyP-uW=_DC-wzs+ygy>Johtr7h}BQk#pjiin6W1A zTY2M{)@jh7g$d_Onw_-F{R#b!;wrbF{2Mj>CnCGzkCcU@8qd&w;e>$Y?tfDo?sT$v zl6)s*x*##75;OBxDQHOYFI+SZ{wc=)xnO@4HBsB5Bs3p}0#&R;Sb=K$x#H}UJBnQb zjuqbD_^8Gp-@c$HW3DfZP6HS5kyxv~dwv`VUZ1e=<~y)5(}%VCMKI0;(?(C;BfDNM zqDV9*@C!hcg|NL@TV1~MmT}7~H1xa5x4YEJ0~WG@jE1Y(TsGAU`QagzG0w;(YTjc0 zZtSH4jnK-^7|~SXQ-_|(eAPEAYnKk>k8lBZ76e2$cz0(Z?@G*9eaHYp{BIN>Jh!vt z5^_|wShfo3t)f@CSdU8kRi~t|@^SZibncMwQ5ZSO%r3~;-xv*a>t38kzR5A-Hr9!@ zB)cTbsEpVM0Erh+E#BOa+&+>f$j9Yb|e#!Hj5`6TcYf6A zW;39^01#irP$c88P2teS!~S zzRY<}ToxW*td)#E<=37p`(0wOT#DFhP=<2mH+N=9UXLh?*_?j_*s#5;sk8>oD%7kjyOX%8s3m2#%564f7_{bj(TU8J;!0*B$|vrkNR03bPa+2xM4+L~s| zGDJqf0w&6TPOq}vrHqkDQSe7E%7i+i*jHLQ9^~S*W7(H`N}^ZqvR~p*3E*CW;r!iY zF3AkQVn`pXpN*g~Ji)&Bj(C{&+8QhZMDv$6xXEs!e#rMLBp&5Pm$JTlNBTm0oT^Di zzuIyU#BOW=SJHcNcDl;-j6G}Ya+^@sd-nG-z7t4dX?ei{lhmgQ&-GQ#vT>fXxfVvj z4SKZg!-C}ra3&?OLeGSvKwYVFgRdo_B+i=1a-(|}RY5cJn?SM001;}%~vlZR4=hj+m4Q?uX0H~#HSZ!InAdD32_MgIn|5B24zDN9&=;8Zuq~F z*1x&WC|K?Ep!<~L<9`so#y+rZiSi;$<0jN>GA5|u+(?m_?uXkb>o-3F7`rVOdopvu zpohIouDD)HY-XvftR2_sIxiqkB|yCakk(S;siz5}xlThJ_fcvqbgFU(U#U$lRIN3B z%zZISH@el0ZxhmIEGyHRC-7TH`;}Zo@e9kG!`M?@&uooiR`0U{x9(_QVsXR6t-Px8 zHwV4$EsVixQZAJ4R#?{*Ch}-TfkOq5kcK(lK~C>zf`$)dF&koZinket6XXe*O&*2~ z0$W-6pn_!31c=Q&L)8bG-JE;jW9Nz>_F3$)9gZa$B@jiy;}hIvcf!`(2_`dl!Jf4w zeU6<~OxTg~-;-SkBv7H=EX@i9We6!h=3USPjYKyxWb^Q7D6a%Mbnzi!fPy0kLkM<< zjNl2u<13@E;Q2Ft{u~=0;$Z+wAwxsB9~_*4>T=`GwL+91)GF!#8~nh6mNTOfCU34@ zML>LaT6v}5PD@dI#+%8vp~G;{qIzskBfpWOS5O>}r$DOlzMEMA&wCYX4aZdgW6vd? zl4=DAKlp}e-Gg`i$nSCg4^AR;r1dvPF|9lO zUlFMZ@BC)^iM2iO&rGOnRextOJJz^S`OS^~AeK?7vGX@w;XZAjheJTo5zT-Q&fogeE(C5-<)4nX-@8 zdX2an$_;gGc>0x?>iBI#pF4?Kmp9zTbV|)g%baIl{dzkmPaDdSHCSFk6?{kXXy;QG z00~F_mt6cuXq2BQd9(?sSu=!?*c(n&zC5yTjfTNOG-m719`51o-TDE_G zajoX}kB?x#4gxyz4Fr<*eG0z`1mxer_Qi7$5ZC_^`M(4&TtPs%wcf2vwjdxoKS96} z-wk>HOVG9Led_CfdCmWz*rxM-{0RDQ5}?*c$u^sZk$*l-*QNROy?=m)zsi>vfzPsX z8kAxp#rBMEiDD(GU>id}t`X(l439n@aa4Moc=Spf;&*0*ltkMulUNiEU#HuMuv8M$ z>AT;0YCHglE_l;;fhMvL3Lt`A z^H~kNPo{DfHXLq>1*azz=nFu4S&N@oKFi9^v?Qq$BJ8JZ5%P(M6NTFj^#BD4xu-|C za0T#%G)KR3Kx|`0njXDyl94WCG1+@X%7p}8Ix7kqw04HlO*qB|@ac!`{mDmty+VbMy8s0cipbW1?8z^ws3y>`Y@oc0y&GFkXHSXkWcCm^DM z#wP0UCNt4;Z7l3aV(u!l2oguyMKfyea-D{+nW);zW23L?X)EP#J$2e$td1OQ5)eOT z$Or7Zdd6_C2p84=9xX#tS0w*zDe`)BGFNjKn=u!tv6!9(Hl$DNjN-(BG5jmt_~C$v zNU9T`^T**C@Wh1PQ#7u(g&46+hnbWN#wKmW=#AXcKgsqq%8cm zdZD9j@pszaf_(nm|4HWmLt&wUA%gxbs}|pGr-?>mdsN z8Tj9si&E)7mE@eGL8aXJoe_#c5L(VJ4!-~&{~$Eolv7N*se0GAzNni5=|z3+^H;bC z)X%qPSCR00F_+bOA|PaGr<0`=?8|{nA5o;?8Uc&p3*kP#=^P5L12&vpfhY4zfa^OmGBeJonK#o;`R+ZkNXssd1O}IZ9g>9QYaF z7+uUSTD=h~+GN@fO>wr6wh}q-c+{y{am=oK|jkm zg(=Z)-Z3t&fsG$z7ai~S7?<+Nr@&mV?5~)ef|Gzb`JT99)ronAZk3t%dW zQ|}KJy!%`Z^u9NT5&ph%hDda+lLB8ejhbqNaCFt_v|b4~T9@BGap@Q-Ffd>>$zVlYL_X;6%FW;FlofwHV^^_%aU^&kGoy?8nI z&M@2lVZK)uBRcObcJVts%5TYsRv!NR#lUu$Pn&a#uV|UOgqWCuM>YPT^T&iJ_9JT} zaSX2`9$&{v>ZLepS6VUNxdLUjGOGOF*D{L{rjoMoe=ZgEVaB48(EnUzwFzziD581w z%BTtU;U$!D3<*&0$|YsAMifW_F;6$d^mA)x!y8`QJ>Jei8)}9_qScWd?2JeKpT9U%w)VdTISmgkUFi28*xu*&q2iB5&Gf2&i2plX$8X^#`wyXY%LNhK zVaj`z%O_&-0dG1J_9UH;<3(E^8;y8d^v<;gM-!;%G}CcrdRM9+=J?3-{7V!ldFId(hrKm^_=yV`RZ>oJtdbqH|&W}6wiL4*ReJ_L!v@1 zXu;8*R?;K2-t^?(DRik1`n$iZ@L;L94b^{F3BC*q|M*4X%q}Q%cdmX-?z-Wru;p&N z;9qm>{|$ovu@=Usy^#0vADiNDljDc~VR9t+$K(hNN*qHD^&g|y|LXyMf+E%>t8f2Z z)_=u7Msc$=Y3}{Lx=WVp#1&! zf&cyXzx~0#c{5OE;8Er^;-pAw#A_z; zT0HnC-R-#%S@VjD*R6DZ}wc*zwVWDIx~RuqzjZY&eda-*NAL9M&_6S=cJ}?CEEP#)gh_kbtcru`nC(qBe1#y*HYp z2z8uB3hQci1FeQL3Y{uL)dqdI)C56X*BsYT3Kk-6%?^jhXseXnN=#3Q+?>LvFT{^c zp7KbN;SFS^k%b;3jL)g%w6B&Z!JaY~MjN+R1KWrtOAyV7IQ_1u>`#YY{cSR8oHB|Ejl6}aj-UDQ+f z<3f5SM$Fv;{~Lj!qQ@dN$qCFM^sfqn+>SJAaA(1-^`^%XRs!Tv+|Z`)c$bPLOGs~0 z2j5>zSPxxM+(%usWCj{pwQ-?4brU)U-$FhaKT^hY_(b#RO=aH|{1NOUs*sipeZ?lj~!Wz{wEPk3f_h<4u)D zU2WL`6;~_vg=TqB1ekpeUiBFr^BWrZ{8`t!Y9~jUywplhzvY;RhmQoPlRMH%ZoI^x zh^+TeWyr{)A$YE=hrQv3r(ueSiC7&hD^^%`BDJO$~2n5|yy>JlsSPACdC~Y^zNiiiZ!5phvBDI-fj|kKJyL*y#z=J*6_{ z6&pqftPx4LI@hbnFv{+TqG>a~AbQw?^CF__1X-s0ZS#J2qPi1aw(c?ec8U^?xl=LW z=yYYScYmuPk_G6{2%ldo-1W^jF-p&@2>wBvHcXSP8v><=#8dX!Q5Wv+4c}s`W=mv? z*2nu1#e);0s5RE}IdAjXMnrc_Gy;01+rQzb={c(O%=|BNR+bK1s|_gY-695M{v?o;o8azzz|Q76rDTm($c#4Al|f! z#nHl6Tx!P0&42}-?QGus{H^0s90>UPZrIlH>7Y@zMz~SU)-ip7(Uf^;9=s7XlZq~V zyDy3iUpE(7K!@iguXe_>$kvPHi{j+Bo{iX4;F`*#=(islAm(*^o-hMP!)VlqdJU~Afa_PC!K0M3kB97qQ&XfxYRXU z!s5M1=Ee@`-g8gb+Dt4CYaF?gliOpsNXoz)sPMJ!kX_+%;D~r^9JnyM6E<~bZTC60 zEOwZL1NO#K3OR5U`PzHC#Nb4FRIw$#hAhQcnLO&#uGE-7<=L~;?4UC&lgE`Eg?>e? z`*e$%UsaEG^NJI`NuaR*{RD~qjVoceS~P|jq?x|at6d^X^KdY{at&t}xh8>-_au418?6-XqE-}9!^;1VkEzJrF@?RL-p66T;?u~VpbX-tA z>~lC1C2th0)pkk>rx!xw;feILbiDTZ$EWE8S4F`jYR4P-vQoo z+K?|;@*2XJe`!z6-`rR1Bx65C@-Q4;0Y%^kOJy@`L|@8Gu_UK!`05BD z;HWKNs1FUnMNTX^OolsTJYyX~Jrtk+w(53rA~1g`*IZvy$GajN@MP}e#D+&&M}29R zjCmm5^J-gT(YvxwG*(&NOsgDZNw@uT*kcY2br#{ZtSA$kyv`smSFZW zP(e1cuQY1}hfZy0)^L*Bg?yVst5Eu^PyH7y+PVLT5!MdT8{^rRrt?gz&=9+RVS8H2C;w-AX{Aa0Gfg&O`o;815pQ zQh35wchqCsW*_&WX5I9ddG-?fbf^pYv1ro1s(vuE{j)R>V9OvBpiEPQU~~LMWM0vp zG05xm?F*nRIBnD{aJ$vZrG0C%aIANQrA-X{wWw&VhLsytAHw;zL{7hycqJ)k)N`N8 zR)D6H3kxm|nXSt9c$jipVp-2^3SxivGJzAfIOpoLkDvh;KJLJBMWKW1>a(y-nBZV` z;5NS!3Qm@`VM?6Bek#>ol{d3)WQMSXXx|8mCd1P*TaRNvL=&={{iv_TSI%{vlpj`37@8<=f;>}LJs-hO}*N|p5Z>; z9^U>+NyCZk-o&5Q;k!*{N%@ZUP_M^bnU#wE*yTIxIiS;A%?j&fuzWV(>6sMnS8g-e z6f3VHzi8rSs5)(bAkb|^Cp)GuGm1-6gDZJ3~^nR0i8 zS=gabr@ngYk^^dT+=<~qlQb9J2bFLW&0FH!MQMC!>H?+1-Kow=jcQ`hfQb8a<@u`s zP5&@JXtdH^eJ!7pQQ?P~j)iqK8AMr8u-A5%Y**{(*bBLaQrfiXiObhw-{L(Q?L{LWGR8*zx0>GRrC;&L zo4F}@4EL8M`gshrJs;r`zfY;8?4gD)ThXpkS>e?D^yL*DOvakX8*7w1e*Nti?fz}NvK}XgvzOJ}==91&G zta*#I`B*Zb-dmx_?BmHCju=#Do~fmKroo zpq;{^*IGGw(Vs)bCHDi}J~vQCX*UrH6w{%paAGsv9%7m2XRkU-`+sL#$DkICvv&JV z>#ggb&U}qxhooKCn_)nk=XVa%_d$Wl%VPTw;mnGt?-Q>aJ$iWY*XjhS4FStUD*r!zQ83 z;!6NK)EIX%0ECOZ>W(#xenrY4Mp2?Vz~_u38}v#+>~jJs)YnIV$xh3s&rX6d)wP$0 z+^jQ~#bDC?Pfubs!>jAFlvg0hR^9!?%e_$Xuw#$LgPWN%G7=Ivn{h_k-CL@9+M}*- z^*ujZl;TJYd)1rrYp^mqLHq3xNAm6vMfFZ(btiA@VZIZ;H79|=EtzkbT;5q4s1{Gn z8*+6D1z^QK^faNCglXqYu(63Bb2UhJ*hgo2>xAV|WN;10WpmJmyk;L_Hkh<0jmX#p z(V3|3&o-Px*U?h>=E;#0laEF5@5B}Ox!7%;jZ+euRpQRz@F}2pqFA_`2Qh}O$>;Rx zaa-3tJ_w$9bBYZ|M8))DZ$;0G7}h6XuaxVK9lH)Wr3iRWt)w_<@o9T8fLgybzMf;p zhm1@6{Ms5jJ!-SoESAu~0JO(GAb(K88&cJFNeoQy<2*T5^}-nkDt5F#8`_f*1)K74 zwgvby<-`UgYN`#bt@70?Bu_=fL_Hg1D}vQ9QmsWEosYtVY5BY^2Qx zh`OwGJ*ho+pN2ol1+OM|Fzyrhcb2`oCQp?YOGb@_YYLGMb4YTU^wMRFpD+1n0bC7v zz2dKrBg0hDC+kD3Hv}+=o3B;);p7uq&k*W9g0884dAud-wbt&7QB#6Eeq3`|B)Pfw zlNm^lQD=@x>*3|ev6R{~0hgn8^nH`EUa_Dd+Hjx0Z7#v&kG0d6&&Hq#Oof#mn<~I{ z)3un5+qyoazfu{jl;?%=NNv&*hEiK$CfBZAt%#yK+d5gb{Py*&_B3l7y#H^Cn8IyuIs4SE$wO^>ndij!z%Ra9NpaL z`0W@QnQ}wyO}cZ?9v6u%ByIFlf?V7@v=zi{ws8xn15VmR}t}{H& z$}KbuEfqKaaYisBrO_wN&g7&XTmTuCPk+uK!=(ey2Ler^#Y-jzJb#U?RD&$Irhj#pJ@NSW6ysEufYGm112(Y>l zsFZ${MvCC;4)-%eS}1TWW-1i1V`|$4au?D4PWGeaL8L=a4`FapYauG>jb>Oc&l6;7 z`M|PC=&$YTRt1NPAw-b1hQ2yx1%W zjCev6l;#^RQ>JViY0|Z7l(FN*7}3Zzg_C_E#AcF+O+XwxB-lFR@Nb=nUaUrWx=}Xn z;zVE^gbK2Y7EFhkGx2=$q*7NkvBzUAQvnwy>7oTmEv(H^i7@1vUB7_l7C?(L8RHL8 zCs7s?)p_i<#+e^XF1uKBJ90H>WCXOe`5F>>d?u5Ub*`|t5Z0~^REP~92y2}|tv^y# z0!1`zxa*&CZ0XIE?8;SACofY!?yu{7IVJg)HNA-!(fBM4KS%(s(R&OEW&nN|myz>bk!486wrQu7lJMVczw#bUS7885#fKNO7bauaAh%Q!Dbl>1ro z<%w)gXAC*&OqT2}_n?yN;K_)Y<)lWzgU69!!?6H}Kk=w`61y0o4U(J5slr6=poiV^ z-E@|krTG#DvLrMlqb#A^S_dLv+m>l{y9*kcP#z6&-2Fj7y>_jRE|a6(7!BXkf8F-n zG{fs@I{?|B>%%7)8BcVq))}JmPNSFy!a4HrPakz@)HVm7#@B0@&DqE%WkbzBS30~{K`5uMM z58T*`yxL5d;vc>gYQJp`jFn!DnKeY~N1Lz8sI69>_Q=$)ezR~&2P_ANdk6F+==fo= zd^)T3mO_3QtdH+V*!Te2j?8o%zL)k%<2vW38OMbPVBYQl)WV)R`YfsFP^A`b+xy;O zH|DFGdL0$(AXqVduVwcD9K{Ue_ryz$E5ujkJuX)xYOl?K`$kDQ0r-KY+|6N%JwC3N zsUt7?uvIxL1S#o}PTmfzEkyrcmXs=BbUzD<^}mTr3>;V(LVI3+e0bv#BsRh3W#Ofm z;KYk`ssAp$)mkz&5MuqgAy-@9lS+uPC*L3c1t9eB;MhJ(@W8Va zt_;5hCT%wSkwEqFg041!CQ~4m*wdS75E9-)I84Q33kvFT+&H2%YK_KtuXS)h5wl)cj4f zT~{Z&LOsmaNSt~9vS-(n2w18x2bpnOp=_Cz{ll#D@39Jceelr!@4m~M(`#g>n<>c1 zPk=7~hpSn|iEAuO$!DwU*T=h=#}?Az9?QrTimyp1@_h9ulIe?z#{+q}8@^c^UOWzA zHRYikm9N$^+SL&kw;D3r^9RVh?Vi>p+TPP7$!8LTkW0dih?oTYMeJ@KT~*p9W9h+QeQ%url*9s_}#jbZkI-*c z5iaX?)hU&6u9x{Fnrv9!+xjqm-a=|r>)shs7=lU68)b#$*8J=O$ZNxN;oah^^1Zg> zdft|Df4paU-J&hkgcKFZqwH$#QtD)Ad!!9$y_ed9!)-noqF(=#nR3xX2zsd4@jiFB zZkQK$11Z04Yy1T?tWRvbPisN_m7eo@b8{TlK{zbjnw&p;=29%JF5AGGaMs4XGSL2hOrJ5ajejQ#;2r6*# z$j1O9j_oLySsg&5oh01MAq8}-=h@|!9fM~^NaqR9i{g4ot%`#1?VR)aB>M|bGLsX_ z#m5$J@s7DKyNs-=xINe+(HDES>n;|fHETU>G0!Pc`T0Du|u~SF>^R&Sn-jgTo10Pgg3YurI`!oE6nl2Rf;;##41 z^D=3j7>%>c1P=09)a}$OJvelUs~~eAB1dBja5FUAr>$~Xwe1AolGa(`cX{_CMPjMS z9g$^cdfsXh$qRLr*DwrDT)eKdCB41?Hl%qU*LGu8tisfSFjYi#PJZ^D=IFidYmiO8 zdFdY3Xa$Fhi4V}dumnlBg>Ux$ZarN+E>B%y#!%V0! zYJX%~8^2L~ck*ExLP!E)v9s}o9Y`bVR3VSXRvVB)VPcGn^@Y%xev>$iP9iL#L#I!v zJQhhXfgHI_5kbXbX?=*-kvsj)ij&2qX36PFQc!A9Ch$YNZdDwQ;Pl71LrP z{Ue^ZUt31tYuP;_B|WA?C)y^C1{LZc6BuI{aA!g$OEk~r{-i)?U3)w!A5sVqKo2-ue>=o!sqV*FMB5gsU_NYmXUzLP>5qF2@D7{wVY;4aa zqk6pbKgM>_fA_ELJSHWJ91Dj=5u}PCn~rEK#12HTm&Q$5e1jI5=*BV?ByXAj;@^Tn zX;i4$mZ^&bsZTPv!^{i}{Mbm-4y9_LqJ=dEU(&NwTXHlSx_KtjCU5k9qL$rRMjY)m zmYO^TkgvYPYj$`Scv&p&QH9RRPn@wBMj2QHZa0)-k?9fs?BQCU!Zz&4{v)zF1xGB| zfNCh>G)ZG(*Mu+dKxT43=@2I4^l--HaYYx6&C*_9>$)HCqKGqeg>%Mq9jHKGAV^a> zwVjj6O|8LFRzMZXufpomm4=y;Pnt@OI+!ubeiNP9n^VMCy_QzNf_;tN30jL{U9mP* z=rM5RiN2tF>8Zv~_5|!AqZxH9_>Jn5@9#Tj@c{qTnPRH_Q2RO(v z^89IjA=Az_45NGymbBOxdS|XsNSiquI4Vl{BE<%FLTGk?w7co+0)}%U*XcgeR_`oB z`*g@ur3>XA(LC;#p>>n>t#6mwZD~+mofGV8N6dc z(PHO%jG*U~@9d>pg`Ap8SNV0|l!uy9Rk|SVQhT3~M7>}+Y#E@qLx%OyncE{6Ejhq^H1fTdWdm=NS>+Ng|TM{T&K5|hwsHQP|4Z8rO8oQ||kQnZGl zX5JgK68R>s(uTc7;WFNDA*i;V)Y0z&hVQ0dksKsI|)58LhDt%2ML;*2}(V`%LB zE*{tiS9M;NgC0=^DEFf^@7dxKPL&A0wvxn^HH8y29H~gz;*Z}i{V7O-`*cMrb#XZZMCvC! zi#&Ji$EI7NPO6JdWPXgjRvTq{V}swA|MKU$hzU2{4ta@Jn@~-*V$7@Ls?9j(T6glw zMjRa300>8dqat<2iFPNwG}E!w2)$-(&u_`khZnBC3Uhk7^?7odYQ234!K)zqWG~$z zveYBAGXQ-OZwPQLgTK2E?cOA7#m+?8y|P?-vEk*D!bvP<$^F#fZLHy(*;&_W(O|+- z=c~^Er2qc4m%;_>wi-5K4tV9HwqeGp&}8t+`gn>=ilQ?z7#RMugUtWac$^!T4C-Nu zi%r3HvmODbI-qFKu@=39ku03fJeV)e%-pCQ-?F}FI+YiC~PB-M2l9kx08voY;d>(vX`x6^DXv^9`I>NV!)Laa}5 zP!Fumu4{)`u?c14rR8%I^s3PCXHQzDsd(0z))p~rQAL?5)xL3SM$P@ySGdO6x$)(8 z_uxq8t2{k`k(9naAF076QqBviMK~fH$!>6YAp?k%zfQwaff9iSJ~X; zSaXWKk=u_Q_u=kcv!-6ZI{LMX7KSOAbovY|P=E{b$FwISTr*K@VR6+ba~TP3_pd z$oz$Ef^4$wr>A{SMA-%Iqr$Sxa_0@uhfb-7p%(|USOKcB!q}kQE zon7TvE*z3LT!laC*XWGm5m)sD=BuC5GOgp0$D>P<#-e@R9iEd~QhPii!9EfX9GZna zaI8qbkv2*f3oM>J>0baY1Wx#dFq_IHoj(t!)Tr`BS^WgusI2clY#{smKs2I~s=eF~ zDRN>m96D`$!_{np7&rpAH zyBWvsaD+hPUCCuXFoP=O`}@wvrt|u7Zo&MN0fc_ewzJ75h7T*IkF98QPtk<{iSMt0 z-J{#`q%La>?euRQDQ>xEZNj#FbBOFyO`oQ}hU6g9IB`inS>GnD33XI~1B*eAYa1MC zi?bDU3_7Njvym+T`)sf;NuW-IV`x?MSxTo4Q~cr4u9B_LE#ip;czmy2~~{;`YTU2#T=2 zp8;VzH8ZPPY-=uW#!6({!sn#ScwLtz^YKp!xzk{<9X19xcKt@P9z-UjFZ8WDbfdWT zu7kWgwU;Odv7Qejz$wpw%KgtR~IzEKTGsE576hTQifH@{MuEVw&6S?&#yh%&msHv)dn;sF)ExSgdfIG3}p_yxTVKYbS07IG(U_zR>pPTCWF6 zDqa^Iuhry6E7xtWJX+nP>8W7bU1!eF65&)F1fqfC!Khl(luFNX&_0644>(g!H9fR9ut$jK@X(a|KU*N9!n|n+)c_Ra{Z%oJwlZ%C!*2flOXc zClaa!JyQ_Z(I&jJ+VdH9Dl6?#PVq9+(?O3K z?PpBZ+aEqU(|NB`rboJdKOHK(tGUF&8q+sF4k8Kn)E2dXVV$pVO|&m$>$viQF~TurgLxomXJD>Z^;8@HZ!a;#`aRk(6Gw2_kxjcW(|m{K~Q zH(hbbBYMMx?oBBxMI(Q%yk3M_`RvhF8Av5M#)-X_FA7;$cG|sT0|r%ID8A-T0emMr$(n`v^^VkMqTCbZhUOJEwecq(riJh0{YXn>~``kq}B=2E01lIlT98fQwPh z+raV&N#~G*WlILq;gv>4P^SW(%$K|?5*k55WKSKRLu1l!wcu>hUgH%Th>_dJN|hJ@ z@qKpIqogb1amBdO$#zw9`?;a?_UaKkM?UuX)Itfu%b@Mny=p2Q6uEV()U#Tp*V_AS zC?#idv*vKib3YPNa*LT92);qYH$Bn(flfb6G*yOPuiy|hK4pG9)4 z?nft-J9sAU%MMO2rz1DQEp>7Ant3?eo&4C1laz0ip2?DR} zA87g*X|eb#__0JaPjmas*fr#2MU;tM%wcGi>9$pd#5&p)@Pj7*D3H?dYY4(*k*I=> zZl*p{NP*syYjUCfd7$qF^|f-S@+!W^BMP034a|lWK1iVzF+GO{cc^^E53Q{Tc-AXe@J-r1M<(c~(QP~o*ejdz(_ScbFPpll<|0@+R>d)No3ATdv zP&bj)$_}ENk4Y>!H&k@TnFXU5cQQVy?Q{z+Y;LRCp$JkGW!Tcc?K9}po`~!7=y_}f zZB%6$Y|djhpnh%R2ilc|L}VsX)#lx2mdhP~ZsTO21VpYx!}%(1y^~Stqf)3XZ)z;s zgm7Rm=4BSZl;3t|DWy3*{u6tegX``iC`P3dAKYT(`05)4W;t)SIaY8|=bMZ26FJW3 zQre?>@-`M@9ENP=<|0xpvw3-2g?$&1#hB?<5eVm!xVaFS*&2WSsn*@xUfu52?roGr zlj$voJlzODA;jZchxuhr-0i5N#T0CwoDt}R=*W+h&(z>3ieiRD=K`mN6_Od z@A@Avb_Sx#t0n!XCCsV)EypY3En_E-!nVlbI*z>73Pesm zR5mUDpCssr?I;wXkqrs^ z?IeCjd(9$k_XtW|wrVKpHaTRGOp&eATvtwV*%HXpFhNf&5g?o@P z#q$dYG;K*pffDsdWk{LBSNg7k1RfN`9b$~x{@NbQR}O>aQW#BTR)eI%>&+mJW+dtO zPE>s>+&b2wf)LJT`>*fzB)!m@9Z`pnMo!&giw8Achx-o zC@nw1@Z0b&+AV6c7f};^x4C2Bu98}wMfl$SqNbKj1tXc^qV2zkc(&oIQ081Z7Gdg7 zwaMXFtBr3(Da(+gGV5;(hT!?~rE4=D_qTD|n!{s-Ysu1_?H&e}u^3mb< zbS`&CW9COJ_!ef{?DnYnPKI?+Wxw~TsGUkRgc>fwKC3s=ep!qn;lm1Hz89swkf)l0YgKX85hdWI^bQo{|YaP zW&e*As-tAtNk!D_%qZ25SNe*ecIU`J+%FeShBXrOV`28}@G-mq^df1xx_$o{zPq~E zZQ{G!l|5&tAGRC*!(~~)vt)sbufnn_v|pcE&kkp=04((5kG~ry&Oc(ky5TLyMn>#V z3=h`vs()X%dHjKMW_mWz_Em(VIn$2K$MYd_q;j3aTWtQHF~zlwlr<|CMU>)COf+#fi<+xaWbxs+Z&pESjcKVP_-ilp^<*=#q^u@mAG|1fJ`4E>2 zeq{d9;7So~V(4$&@c#+A-`k$Gx&sL%5z87By(_c~qK@xrAFAfjOin4fLuowa(&r@;txKShiyfDV_oFP1u zYTc%(+-q5$EYG-CeVbJ>W2BHAK@ddRdnO$&Jqf>I{%y}#Irr)%N(~FRq`v*RG@ifb zxzrB)W6iT;)xP$qULg~;1oz!f)`)!<1{kI6r^l7prG2SJ^)Vj%0wsBie8*~?Hx)Lu z_{xJ<>Zn4tqYShgvM*Ba)YpQ*2$9)Co*;HA-+%YXEQ}itX%oXYY$_LsD}fp7Q!V4~ zHVS>I=(D8o?%(f4<(G4tx^Kk})+h(9G9&&z?ay3(b&>EB&?~8aoFK(zBmg{NuWNGc zI9n*Dv`CsB?&S_p68oppFWhjTJ$`beyTK<}z`nQ|O9QUitgKtVc5S!g;D=9dYA6R^ z;v#SNN`7^2d~>yU(JMTyLq2vG;9xVmQIBPDEH%sbVyx{4u>K3-F+zI8N2zS`cDRNO z&s0^%oeFOFBWXFJt%=&(yQmIx)r-TLe;BDsK5nzR9nhz$ z8~xK4OqA4}(-hoqx!ek3AqweVp6%mmt>`Uc%&1?rjXg6?oy;Yf()gp@3C65?wD#4o zDoeA3tZz;pQCVZjiAap367mZI%qz`L6R*!*=rByG}Itaq@^~EcIF} z?-w|GU+^hA`r%(6H(P@1SueE((k!Z)K1?KUmy469SCJkEgU+xf!j9q5m#?)kTl}-E z(_Pg>a@#<4g_aLx$zcEl~@GV}NV=$Mq37jJc?VVQMcUp_()$M5X@( zv~FN=d76J3sZiUZNEzV3J88MS9P#WR$`wuTPyNXhh-H+CMK`oD(jHw+#WG$?9RAzR zZrgg%Av3k^pg*I=xwcJ zDw*SRdyt!aF_#`1$e|2(Ks^#4hNYD);7+dTXjUI11zIoL@WA2Qj*1BVkem zPMe=fIQTJ*gZaIW<{y<7O8*4P(rqCwSCzjucjf+azW(Jr)dIl#R*Oej77AZ|LKuc~eKQ-bw1q>9NT47HiG{cn>Ilihyks2DtdA~hLrI$1X2hzhPNp`r^u`ddY3ekx zOTmtS`biF4SvT02)e zAmUw%`$H8LSqUp4K^@O---S;zzCW%jngvVQpm#3EH>`%sw&_rO&`_2Tz#RtI>P?jq zEnDiM9J)PK(klte{5s4cVRy{$k;sr{MB9;sJ+%FEM|15-#8tE#OWr+}c@sgFfU)23 zf>)sbSc-1u4u^I^S}&!chL>ylK=W_2Pw})bV+R5-1!Q@g2?IYyH4MU{lnjqs9u!J}ong4k4Eez8I;+o9* z6d2vbp;u1A!?sawk5(6}?A=54YE|tq??qq^DKX+ZW-MX}IRNqYp_Zx0b=3BEV(qjV zw6bVZr@flqIwT{2_;kla^v~1YZs9UT!?`}X#4MyHn)p>Kv*&ihn8b)R)x`~49`1PU zpkd62O!VO}-qE0tKa$n zTlhD>n~>dCqm@1&KoMOWu4JfOl4nnu`K)`@$?W{fnTmfun-2bIKx%JJuM|t2Oa9U7 z;x6EP)Q5}U?~LTr-6wQ@MuXEC=mtzycuER|06dij&TceoB{URTqb9mvP0T&>|7YaU z<{tZG);0x)?qY}5VJwQ$kUsqiA&ZcW@mMFS1i$f z$4N-49Y(Rx$0A#CwY{DxRG7~Kz4$$XBN0w&8LYar(@tE`barr`!WC2HAlz0})a^BC z@F(9L)022ILXjKPh5GHy)O>P~>IqbObMY7BcT&p39;y;B#?E1L{kwsBjpKH9upx|O zL$**}YKTPBZa|ruAVu;o#)Sj0iC|fjPKW%N%xzuLyN8Af}ZYR@@7SBd$3v=Z=zKKKTH7FYo#>P^!?a2QLD~`(Z zVa?lzbUT5rM9zGTwpg=t8ssziEoRnu%AuMoyp67nyQn_GJfiQ|isfs0NW3BG%_48! zY%V(h*Zju=RE7-_&?s`;Us+(at(z02y0@ZoZQ3H7el!!9R}|J+_x!^f-ylfe7}TqG zPj?D7<0mKTCGG1*f24=*Io;(9$b!{Fsx5mITzRn67Q`CViYU5xLgRdrL@AAKeqG(m zk*RF!*S+n7j=Z{2FB=lQt`m=f##b<3q~;2C+}NpdBZ*6YgnLpaQqo{~=2_zz(3!6U z_F-aKx?`Y;C^_={-J55va@U6U4UjMtG)`l%^oI*FeVrB~{gth4MB5fSUoWANx!YZM z7wEfIwTw*hq>5Q(O~TMkBONwXn{0MiWFu~gyNhSE6%Qeme@;H6M9EleIju~|Ybx`U z?#4HsNrlI)XK$7}9mMtW7CJ^k*V5`14%y8%e4xUs#H2WNdS#cB;%2Eim;Af2Z9;`^ z?xJYrx=mtSYlOq4^^~B{I$<<0WrWCvqW6q@5$ z^tueSG7N0bUaOw`+a8IAl5VTg=X_etSugz81+! zI)4tygqtfjmd|H#5XI9#q~)w_7zOhm{4OxNJ!Xb~Fz*|qs7nlD?yz?^^2}OmOe7m= zkf+3|TQ9^#@xUDUGv4}C-#SC3`9R&34OG1sAgy5GG zozN$?T%JRu8n;XPO3VHw`_t48XxC(KyXGom{MCLs4nyN(iehDD+yB^ecTgQNr5;KZ(%BpF%Ba5kB0CNjn!f9Ay*vYI zqhdJtxCxK3h~{@1hxfwpfpYTX8jV>O%w~#Gzs#3G8rn8mZ1%cV&v~bcs(Kmm4T1Z9G&EbYb!8FQg)AK=7Ea>8bSOxVehQ8*szazW6_1Y1CY1ku zKAdX->Mzbphq$_0qG-EBYp&r&%!^`8Dwu`_7Lx>`iWF5P#`6?*J+utd$33UPP^glY z-rcpTQjR!=wN&`OY1`C=XR85fI$%Z8#p)ZX?V7%_Y4EKZFrLy&ggiCJFc#KBGzvX% zdHwcl6X|J~M$mVhEKKoUXQ-Rz+J8sX%@2l25`8Hma;G+q5_@&9XrfnuyEcqopO4-# zNJ<~x7cU@I8Wd6%H*F)UkkpM%g0#}7C{k8068q9qfoxQ#;SL*ijZ}-^V4-c-p_OUB zz;1>GJB+hlA7M4YVo6P{cUDaj^Q*1}Fw9V@O}JVU6@&ZRptO0F7MhuNQyt|&*z5SD zHD0Ru(KC*M4R~|jH(=`B;(k@%Vz4NdXM45`aK~LR7W4N>wdMG1>qkBt=#>Wi{z13& zWYh(Bl179pt}ml%_$K+Ag({;|3;Lg=ZuWH0Rkkge0ToU0V~(LwH}89Y^eiCJg26^Q z+$Y*{>r!uhW*6Z|?L_$Q=|!k)=JpEk z=7`1=sN}F$!qSFmwvYMF$rJlv-1VD2d&QHa+dU3{oeQ(6XyzeHBd*sD`X_^bYCTR$ zE?;B6OCjUeDOs!u})ZN)EMeAz` z0oADWu#+ves)@p)=gI7c)`-lQfwShfWYpUdoUSn7ZVX>1YDywe%cRHcNpOa!s*K5k z$C2cgV)MIoi&xSQzS_{utz?4pl#_Hwoz-0bXqX+}`b*%2nNH2q{8c^oR0d$GxV;-~ z_DOT-6nIUY_Y@*?E}kwe6+nJ@p8l?!r?_7FpAvMizt!-fZ80$O>Q5sw!ojCcl?E^A z=<4K1^tTnXK5fwcM@^jPgy?zX)vBS570TlczOz_Biqy5ITg-1Ot5C(GwxG3ZhUpB-Zn zg+R(E#SL&ZnwhG@W7ILf(+-`P+NnY=vXt3&zv34uYZi?B|HTVXRex7m^D#%*2$n+xf-^{^M)hvby6SR8g} zY@U&QDax1Q>gVW)Qi>BNcdQ9hyGjt0Zy$wwR%ga)ST$43c)!)XJ1`zMTw<%T z5!{d1$lhIn>Q9|YShbcI?CmzW#CXtGWH8N-=GN7U-)JOhC(w(ZqESDal-Nn`q%})s zDsOn?HrxDAx+kZF((hKyw9HaXDnNev(8l8-VYbpS7YC2;oR9Er^(F@R)|uc^cQ%jsh?zMjozCrt5ST=DVR|6_dI&Rmj7C|jojG%_Suiczs>8Qe z2Bp!%IeUlPu?E;bUIQc z8pig9wv9*FL0ju94p-iTo{c&5bPWQD{r-F+cGVL@F&edzh5@>xN~5uK#iGc0vLiWG z{|3O+*HgTsxKf*aZWnFsSrb>05 zjr1w|%~+L0c~Ul*wR<5eczb6OBGvx$N*t&*<;+ce?_H*{kC0EUukCHQmrJk9z~fas z?L%b9iOuD9K=5YP)BkQ=)(wv^@{s=uW$4~Syr|<(n+DWQC%o+4V7gXzGt<1y`_J{Q zP;9`QM30qbx^tZ0dKE`@$BQLJ7lY?Jbc=M+?|vRZdX*7&8$5kFbb%w8K_K5Z*B_ZktD-*)0N zfeoW#gd#Su8E?XMGbmzvuJLG=n}Lxz>y`rb*4pDU@*VrUfm;dhBu0K5v=LO@f}5 zCjlM@i&jF@UacK;Mtsr_6z@QrWG<|aIWtcTck0*<>Thc>W{2)s2Bw+Rp)Ax>d&~cu zAIwH%*Zw8j4o{`Ua&2SVOQxK5LlUQ-9SiL~vddb>PS$H0{0Wibu^RQloSYG|yn_j^ z2ki@KZo`Ab=^$_%P9uUI2KU$<50~!NYtwL})9PuQi76K-O=zu8^n7h^HxW9S8+XFB zOa5n>^*cQM91ZxC$SF<~*)OQs%phQX&qo;VxoVlCXV^#W_4_dGO=jJMw~sF&M$_Y} zsCZoMZH6G2xu`qZmnk8coLo?n#4L$mVk;@j+npRT(?m-gqb4XjM8LfB&2 zadZegVvqW;>yc^IA>}fpF=uQU^?k)<$-R)}x9^&4yX$E~Hb$NEc(QI~4l7qAZ&(Rt zi$N8|zwBq04dZ&w7?h0UHc@OjrC$jTXU3EMr8JoPAfSgtUH#bhd38*68MZ`po>7s9 zS0`l*zv6P2&Jlz@ywlp)QQSWm-p_dxqeBW`3>|1e+5!kwW{XkG2x8>Ogq0herzSf=`a0rvuc>URm`PN6?4IF9>w;tJ-!)wGEvpcsX) zAFDX4^BfK;lD#Ps&K=5|1of!dsu^4)Ol3l`?Ts~z(Kb(l*mXC`sPr2@M|lurVAXjQ zqR=R`)($fr0Nl3^;h;NN0gG}aNcJ!3@Q!i0K~=R;{B>zojCqBZgufrrRJ$NUB4C|!o-(@AuRW8HDC}zdf!Ri9x{X^cCoAN_ zobNi(X(Q7`X1gXL>PpL7mRIBVsq6+&v~)vuo8#Ma`466g!{y?iT8n=b~51p$qc z>E_pdBI3s-|081Y%k3lGq8k{<0)_?VDxmp33OPv1y^mu~0O;1faYYKK>}geHQxcuW zFZD0H7Qe>+*Cs57>Yk)wrFQFLwQPUl>3TQ^g^sWIvS%r_CC7WhVpIsYc-5&?1go<2 zO#7d`HG-JssdwMtz>fX;h!C!m47PL2Ya{_|J}o)_b%{9i#IvQUsh?7Xwk}eiwj9*u z%H`J}`HZSK+D-P}F>3$45v2gmSR5oSo(~;v(qKyr>pZ2!bXSu^7Pq(=$lD`D+dH)KCUq zO`6M!JnJ{Y@cFEsZfjOm<|`Zp`Izp_GUUc^x?++8 zj03~mChJWS`24W%ygARYte+X&s~1RRag@gtg?aIFE9=`t(uVPqg-ES7%XZmo{e4x< zdNvprHVBEV&Z2EeO#adm_P&j!#QCRjO~u^fjh6-#kx?94k|vg-hVH#_`e-hF&%F*Z ze(%}HsVd935{q-&tYgOpUsnamV%&y+5(*XSY?m-Sb?o;d2Sq>T)Dq z#;U-p*$UYF!4L8wQCOggP8y%8k$h_;}m=}W)N@ab6@BzKtx1DJ;*3d zwxKHi4`&G(DNbUM3ydb!c8UV9ubJIg*)tY0jTVu5Sm~(aBC4N-(R(G;tVFzV*n*1B zw#UAdbX%ASb3M?MRhSUg)i5b#!NnMfS9Ft|q-m_iVtEmvyi-`DaxRU_ljIbPRoGQ! zNf6@f)WKv;woPPRf}mA{>nwlD4XD!aFH3@0>?m_kQmHEq|CS<}Fkqm2i{Y*_*I3lL zUZfa{sJYk|UNJ(`Z;6z|g9aj$icdx5bjg`Dz; zQZ?h&rwTxAQXLDGjeI$$qT(>@apOmP=JAj6NO>7MZ>4DK$WJq(t{VnkvdXdz?M_Q8 zFtrAIA#q?3IWcYdglI0ATg?ahk*rV899u7dSs1CJlQ+oPKO}qOB5@HzP_1gYN-C%+ zyl&txHm$~Wc8by9MpZeA{m}V5G~hN;&e&JcD&ok4bPt#Gyj{r=YtoI=d+LcPX+}X= zshByhYrrzAa+dbb-xuE%E8j;3+gjtbdXlOcc8zpV4l5SMCvfa(s}Fj-D$HiY)v7G$Yzupg=6z-;-#(u& zSFx@A`q92@Hdv_8wm`#Qc_{BV`t=_XWNEGaa!s!_?0T$-pL#SY>vd~$RCM|&QZ)sw zDk-BjhPNXSCKidT(KXq2I!bI)O0_e}-2t0}UbNt`UG8?p<_Fal_*oJqmsKK` z6;(u865Z%w9>L!XrMqMD*o^AlOa#rU;(yuM5NEpu4-$hD44 z6ulXAa`XG%m(3{p9m+Vnjc9svL9Akcf1(z~2&%%bki13l0HNxgCc2{gWmhmUw{-JE za|8tX!`|1%WI|bddL&J9kvk6n`x8X6q8Ll%?ZfV&Bu2JqI%XEIs>4Qi7X*2OUjC(@ zXVBfF_xfj4t-9dY0aO_a91C-PpMO=_v(0St0{7Ka`zv$KzV|o4styd?g?9zEd|OV{ zaAe~8vpwGXSW@dfMt{@~Rl#8z9Sq$anX*zKW#_8=u1!#>IA8q_iUmT|Ww?b(0gbVi zh}$ZPhz@wSeoCmyM&P!rcj*^6ymvblp~$$Q4@M9rRn=DL&7NL4Ot8NoJ7kr7l$qi> zfW=BU3V|cO$dWu3H*0EI3S;F$$y{oARPu^2tPWWb?th%Y+N5I|>YH6}IW0~kJ3cmN zkiRh$hUQyMg!r+qc<~I8J*`6~7uC5w@aB6rmyQm>sGds7lZ7L)!Twp0X9F$#U}{?U zLQyMSR42SJ{ZkzaGIekXfXrD!0HJR|=sO3n< zyTUEMr6XX?UlY<}fRpe)b24=lcQkxoiR78$QRe#mN&D|P8{D^njU@cbesgEbW`ZDq z2I&(liIKq}X(F3ZBi*Y4=>=%UpX5#jRgfGJNFTJ)DPpK#IRiM;w#)HsSrU_;6M<~b z2%8F)R(JDrdip66G`3MzZ(!E3K^(r$V#)DtxTuAzv5rs~V>zpDHp&2E0cDSiDrk^V z?6BRCn5?iw&SZmjE}I^>iEYg=L~K@o>o>NHFT#sbrIr4m(~%dELGNG&5d}6DHKAIR z)r+KRS8Dc)L8|X$kdpFAjG^tTd9_m~b%)6c=X(pmDvs2~sV3zkGnbkM#VlA`31^pemT$l1ZA+UjmUhRB zG;sD*8=CS&y!H!{#mGcbBl^5*CXp&_^J*AL)0@>tq^7aj&edVMh?^RTy`bJ|cIm34 z+i2RW9%#+mUzqOA-yQCWwy*}C`qyhWy0Lyhs@!3ILNXtrEn+mE3pU}jZ3;f-!H?at z)*8y~vj2h^NPSKvVy_*IZpY8D+n~zOw!J)kB8dI-UXK>>VG%Z&HIYGFlP{wrPBWrg zAKTFESF#K^UEAO4&yPfJMDBb^p?@aFxN1~NU*Bl{7PGR(lVG)>Jj=y$)Y3fcW-Kch z^hyFE)ty?s2AlH@ZUU_F>%IZP6#zlW|CswXD()^V2*J4B zwlWtMH_440PYA+;$j%G&Z_TN7jCK1>>9SmZ81f=lTzgg0aA>s{Yb>15b`<94q}u4U zEPtA&xRhey*|ITGiwPv69x3gLSbvC&P+zI?6Vsv=Kw)j%ETvJnXj$8k=OH#eD+>_% z(zh?kqMPUR_zE%DY;F>_v2IHWB|{0MkN>E0`|r?FG2TgCUO2Hc<$GlrOsI9BgnXk0 zzrzGP)RaZTd?I70Z_J9neX@URw{Y{6QjFP2*ozZgtuc9r6h=BTB5fxU^xLq?`=&Yu zuBJA?Q)*VJe9?74jhWeSoi#&oI#h1Oxr1i46HfuD>gVyY*Vz#g`tseH1l$pQ_8z$` zk|jAf1tny=={^~Q#-tw<+LdbIcaA&E%0LH}`<)kV;tO^3yVVxq|RmWZFs zk4VC51L^XAG1WKcXAfgAMHLp zpp>_zg_uxWNv2&77I8vV?LODv@ua*j?5b_9_ zj*@uAGjOruA2IbiuJ0R}gWnsOT6u+V{~L&i=vus|b>$>wF83dBH@-b~kCG#KiHH$7 z+X)>^{;8nQ`r8dRjKmAYg+C2__-vJoqT%@{bH*y^;|KpAdaO&ic$7@M&)@5q8a9by z479-;L#pY-^=iAl?gg*56liQ172YQn35r@0s7__eMj1 zB6}AO!R!0*8SQ))A96c_j;5_((>Oh4#7Z15je;bBwb3Dz8OT*;>xSyYqy8W}*66pS iPJN_7(GS)DRQ-Q>SXn9NnCXlE;_gVN3K9jym9=1BxbC3< diff --git a/cfaforecastrenewalww/inst/extdata/example_params.toml b/cfaforecastrenewalww/inst/extdata/example_params.toml index f2fb45f2..ebacf844 100644 --- a/cfaforecastrenewalww/inst/extdata/example_params.toml +++ b/cfaforecastrenewalww/inst/extdata/example_params.toml @@ -4,7 +4,7 @@ uot = 50 [infection_process] r_prior_mean = 1 r_prior_sd = 1 -sigma_rt_prior = 0.3 +sigma_rt_prior = 0.1 dur_inf = 7 sigma_i0_prior_mode = 0 @@ -18,6 +18,10 @@ initial_growth_prior_sd = 0.01 autoreg_rt_a = 2 # shape1 parameter of autoreg term on Rt trend autoreg_rt_b = 40 # shape2 parameter of autoreg on Rt trend # mean = a/(a+b) = 0.05, stdv = sqrt(a)/b = sqrt(2)/40 = 0.035 +autoreg_rt_site_a = 1 # shape1 parameter of autoreg term on difference between + # R(t) state and R(t) site +autoreg_rt_site_b = 4 # shape2 parameter of autoreg term on difference between +# R(t) state and R(t) site autoreg_p_hosp_a = 1 # shape1 parameter of autoreg term on IHR(t) trend autoreg_p_hosp_b = 100 # shape2 parameter of autoreg term on IHR(t) trend eta_sd_sd = 0.01 @@ -51,7 +55,7 @@ log10_g_prior_sd = 2 log_g_prior_mean = 27.63102 # 12 * log(10) log_g_prior_sd = 4.60517 # 2 * log(10) -sigma_ww_site_prior_mean_mean = 0.5 +sigma_ww_site_prior_mean_mean = 1 sigma_ww_site_prior_mean_sd = 1 sigma_ww_site_prior_sd_mean = 0 sigma_ww_site_prior_sd_sd = 1 diff --git a/cfaforecastrenewalww/inst/stan/renewal_ww_hosp.stan b/cfaforecastrenewalww/inst/stan/renewal_ww_hosp.stan index 32ee89e5..9ff0fb27 100644 --- a/cfaforecastrenewalww/inst/stan/renewal_ww_hosp.stan +++ b/cfaforecastrenewalww/inst/stan/renewal_ww_hosp.stan @@ -18,8 +18,8 @@ data { real mwpd; // mL of WW produced per person per day int if_l; // length of infection feedback pmf vector[if_l] infection_feedback_pmf; // infection feedback pmf - int ot; // total time span where we have hospital admissions - int oht; // number of days with observed hospital admissions + int ot; // maximum time index for the hospital admissions (max number of days we could have observations) + int oht; // number of days that we have hospital admissions observations int owt; // number of days of observed WW (should be roughly ot/7) int uot; // unobserved time before we observe hospital admissions/ WW int ht; // horizon time (nowcast + forecast time) diff --git a/cfaforecastrenewalww/inst/stan/renewal_ww_hosp_site_level_inf_dynamics.stan b/cfaforecastrenewalww/inst/stan/renewal_ww_hosp_site_level_inf_dynamics.stan index 4c368998..e07ef35a 100644 --- a/cfaforecastrenewalww/inst/stan/renewal_ww_hosp_site_level_inf_dynamics.stan +++ b/cfaforecastrenewalww/inst/stan/renewal_ww_hosp_site_level_inf_dynamics.stan @@ -6,6 +6,7 @@ functions { #include functions/infections.stan #include functions/observation_model.stan #include functions/utils.stan + } // The fixed input data @@ -16,8 +17,8 @@ data { real mwpd; // mL of ww produced per person per day int if_l; // length of infection feedback pmf vector[if_l] infection_feedback_pmf; // infection feedback pmf - int ot; // number of days of observed hospital admissions - int oht; // number of days with observed hospital admissions + int ot; // maximum time index for the hospital admissions (max number of days we could have observations) + int oht; // number of days that we have hospital admissions observations int n_subpops; // number of WW sites int n_ww_lab_sites; // number of unique ww-lab combos int n_censored; // numer of observed WW data points that are below the LOD @@ -54,6 +55,8 @@ data { vector[6] viral_shedding_pars;// tpeak, viral peak, shedding duration mean and sd real autoreg_rt_a; real autoreg_rt_b; + real autoreg_rt_site_a; + real autoreg_rt_site_b; real autoreg_p_hosp_a; real autoreg_p_hosp_b; real inv_sqrt_phi_prior_mean; @@ -273,7 +276,8 @@ model { vector[7] effect_mean = rep_vector(wday_effect_prior_mean, 7); w ~ std_normal(); eta_sd ~ normal(0, eta_sd_sd); - autoreg_rt_site ~ beta(autoreg_rt_a, autoreg_rt_b); + autoreg_rt_site ~ beta(autoreg_rt_site_a, autoreg_rt_site_b); + autoreg_rt ~ beta(autoreg_rt_a, autoreg_rt_b); autoreg_p_hosp ~ beta(autoreg_p_hosp_a, autoreg_p_hosp_b); log_r_mu_intercept ~ normal(r_logmean, r_logsd); diff --git a/cfaforecastrenewalww/man/cfaforecastrenewalww-package.Rd b/cfaforecastrenewalww/man/cfaforecastrenewalww-package.Rd index df1192eb..d344d32c 100644 --- a/cfaforecastrenewalww/man/cfaforecastrenewalww-package.Rd +++ b/cfaforecastrenewalww/man/cfaforecastrenewalww-package.Rd @@ -6,7 +6,7 @@ \alias{cfaforecastrenewalww-package} \title{cfaforecastrenewalww: Wastewater informed COVID-19 hospital admissions forecasting} \description{ -Semi-mechanistic renewal approach jointly calibrating to state-level hospital admissions and site-level wastewater concentrations +Semi-mechanistic renewal approach jointly calibrating to state-level hospital admissions and site-level wastewater concentrations. } \seealso{ Useful links: diff --git a/cfaforecastrenewalww/man/compile_model.Rd b/cfaforecastrenewalww/man/compile_model.Rd index 02fec0de..3ab2bda4 100644 --- a/cfaforecastrenewalww/man/compile_model.Rd +++ b/cfaforecastrenewalww/man/compile_model.Rd @@ -31,7 +31,7 @@ Default \code{FALSE} (use single-threaded compilation).} \item{target_dir}{Directory in which to save the compiled stan model binary. Passed as the \code{dir} keyword argument to \code{\link[cmdstanr:cmdstan_model]{cmdstanr::cmdstan_model()}}. Defaults to a temporary directory -for the R sessions (the output of \code{\link[=tempdir]{tempdir()}}).} +for the R session (the output of \code{\link[=tempdir]{tempdir()}}).} \item{stanc_options}{Options for the stan compiler passed to \code{\link[cmdstanr:cmdstan_model]{cmdstanr::cmdstan_model()}}, as a list. See that function's diff --git a/input/params.toml b/input/params.toml index ab37e68b..5e40e022 100644 --- a/input/params.toml +++ b/input/params.toml @@ -8,7 +8,7 @@ sigma_rt_prior = 0.3 dur_inf = 7 sigma_i0_prior_mode = 0 -sigma_i0_prior_sd = 0.5 +sigma_i0_prior_sd = 0.1 i0_certainty = 5 ## effective number of binomial trials ## in beta prior centered on estimated i0/n @@ -18,6 +18,10 @@ initial_growth_prior_sd = 0.01 autoreg_rt_a = 2 # shape1 parameter of autoreg term on Rt trend autoreg_rt_b = 40 # shape2 parameter of autoreg on Rt trend # mean = a/(a+b) = 0.05, stdv = sqrt(a)/b = sqrt(2)/40 = 0.035 +autoreg_rt_site_a = 1 # shape1 parameter of autoreg term on difference between + # R(t) state and R(t) site +autoreg_rt_site_b = 4 # shape2 parameter of autoreg term on difference between +# R(t) state and R(t) site autoreg_p_hosp_a = 1 # shape1 parameter of autoreg term on IHR(t) trend autoreg_p_hosp_b = 100 # shape2 parameter of autoreg term on IHR(t) trend eta_sd_sd = 0.01 @@ -55,7 +59,7 @@ log_g_prior_sd = 4.60517 # 2 * log(10) sigma_ww_prior_sd = 0.5 -sigma_ww_site_prior_mean_mean = 0.5 +sigma_ww_site_prior_mean_mean = 1 sigma_ww_site_prior_mean_sd = 1 sigma_ww_site_prior_sd_mean = 0 sigma_ww_site_prior_sd_sd = 1 diff --git a/command_line_eval_hosp.R b/pipeline/command_line_eval_fit_hosp.R similarity index 83% rename from command_line_eval_hosp.R rename to pipeline/command_line_eval_fit_hosp.R index 4eb79c62..cee4f83d 100644 --- a/command_line_eval_hosp.R +++ b/pipeline/command_line_eval_fit_hosp.R @@ -12,12 +12,10 @@ parsed_args <- arg_parser("Run eval pipeline for one config") |> add_argument("config_index", help = "index of entry in eval_config to use", type = "integer") |> add_argument("eval_config_path", help = "path to eval_config.yaml") |> add_argument("params_path", help = "path to params.toml") |> - add_argument("output_dir", help = "directory to store output") |> parse_args() eval_fit_hosp( config_index = parsed_args$config_index, eval_config_path = parsed_args$eval_config_path, - params_path = parsed_args$params_path, - output_dir = parsed_args$output_dir + params_path = parsed_args$params_path ) diff --git a/command_line_eval_ww.R b/pipeline/command_line_eval_fit_ww.R similarity index 83% rename from command_line_eval_ww.R rename to pipeline/command_line_eval_fit_ww.R index 2261dc73..c8bd16ae 100644 --- a/command_line_eval_ww.R +++ b/pipeline/command_line_eval_fit_ww.R @@ -12,12 +12,10 @@ parsed_args <- arg_parser("Run eval pipeline for one config") |> add_argument("config_index", help = "index of entry in eval_config to use", type = "integer") |> add_argument("eval_config_path", help = "path to eval_config.yaml") |> add_argument("params_path", help = "path to params.toml") |> - add_argument("output_dir", help = "directory to store output") |> parse_args() eval_fit_ww( config_index = parsed_args$config_index, eval_config_path = parsed_args$eval_config_path, - params_path = parsed_args$params_path, - output_dir = parsed_args$output_dir + params_path = parsed_args$params_path ) diff --git a/pipeline/command_line_eval_post_process_hosp.R b/pipeline/command_line_eval_post_process_hosp.R new file mode 100644 index 00000000..87e4d476 --- /dev/null +++ b/pipeline/command_line_eval_post_process_hosp.R @@ -0,0 +1,21 @@ +library(argparser, quietly = TRUE) +library(cfaforecastrenewalww) +library(wweval) +library(ggplot2) +## some functions from plots.R complain about aes() +## function not existing if we don't load ggplot2 + +options(mc.cores = 4) + +# Command Line Version ------------------------------------------------------------------------ +parsed_args <- arg_parser("Run eval pipeline for one config") |> + add_argument("config_index", help = "index of entry in eval_config to use", type = "integer") |> + add_argument("eval_config_path", help = "path to eval_config.yaml") |> + add_argument("params_path", help = "path to params.toml") |> + parse_args() + +eval_post_process_hosp( + config_index = parsed_args$config_index, + eval_config_path = parsed_args$eval_config_path, + params_path = parsed_args$params_path +) diff --git a/pipeline/command_line_eval_post_process_ww.R b/pipeline/command_line_eval_post_process_ww.R new file mode 100644 index 00000000..95bf0e43 --- /dev/null +++ b/pipeline/command_line_eval_post_process_ww.R @@ -0,0 +1,21 @@ +library(argparser, quietly = TRUE) +library(cfaforecastrenewalww) +library(wweval) +library(ggplot2) +## some functions from plots.R complain about aes() +## function not existing if we don't load ggplot2 + +options(mc.cores = 4) + +# Command Line Version ------------------------------------------------------------------------ +parsed_args <- arg_parser("Run eval pipeline for one config") |> + add_argument("config_index", help = "index of entry in eval_config to use", type = "integer") |> + add_argument("eval_config_path", help = "path to eval_config.yaml") |> + add_argument("params_path", help = "path to params.toml") |> + parse_args() + +eval_post_process_ww( + config_index = parsed_args$config_index, + eval_config_path = parsed_args$eval_config_path, + params_path = parsed_args$params_path +) diff --git a/src/setup_eval.R b/src/setup_eval.R index 0fc36702..a37deb0d 100644 --- a/src/setup_eval.R +++ b/src/setup_eval.R @@ -14,14 +14,19 @@ write_eval_config( "PA", "PR", "RI", "SC", "SD", "TN", "TX", "UT", "VA", "VT", "WA", "WI", "WV", "WY" ), - forecast_dates = c( - "2023-10-16", "2023-10-30", "2023-11-06", "2023-11-13", - "2023-11-27", "2023-12-04", "2023-12-11", "2023-12-25", - "2024-01-01", "2024-01-08", "2024-01-22", "2024-01-29", - "2024-02-05", "2024-02-19", "2024-02-26", "2024-03-04" + forecast_dates = as.character( + seq( + from = lubridate::ymd("2023-10-16"), + to = lubridate::ymd("2024-03-11"), + by = "week" + ) + ), + scenarios = c( + "status_quo" ), - scenarios = c("coe_proxy", "coe_regional", "max_40", "fed_funded"), # nolint config_dir = file.path("input", "config", "eval"), scenario_dir = file.path("input", "config", "eval", "scenarios"), - eval_date = "2024-04-29" + eval_date = "2024-04-29", + overwrite_summary_table = FALSE # Set as TRUE if trying to get a baseline + # score for all locations one forecast date ) diff --git a/src/write_eval_config.R b/src/write_eval_config.R index 0e77e511..715506b2 100644 --- a/src/write_eval_config.R +++ b/src/write_eval_config.R @@ -18,7 +18,8 @@ write_eval_config <- function(locations, forecast_dates, scenarios, config_dir, scenario_dir, - eval_date) { + eval_date, + overwrite_summary_table) { # Will need to load in the files corresponding to the input scenarios, so we # get the list of locations that are relevant for each scenario. We will bind # these all together to create the full eval config. @@ -61,11 +62,19 @@ write_eval_config <- function(locations, forecast_dates, scenario_dir <- file.path("input", "config", "eval", "scenarios") hosp_data_dir <- file.path("input", "hosp_data", "vintage_datasets") population_data_path <- file.path("input", "locations.csv") + baseline_score_table_dir <- file.path("output", "baseline_score") # stan_models_dir <- system.file("stan", package = "cfaforecastrenewalww") #nolint stan_models_dir <- file.path("cfaforecastrenewalww", "inst", "stan") init_dir <- file.path("input", "init_lists") output_dir <- file.path("output", "eval") - + figure_dir <- file.path("output", "eval", "plots") + hub_subdir <- file.path("output", "eval", "hub") + score_subdir <- file.path("output", "eval", "hub") + hub_model_names <- c( + "COVIDhub-4_week_ensemble", "UMass-trends_ensemble", + "UT-Osiris", "COVIDhub-baseline" + ) + raw_output_dir <- file.path(output_dir, "raw_output") ww_data_mapping <- "Monday: Monday, Wednesday: Monday" calibration_time <- 90 forecast_time <- 28 @@ -95,6 +104,15 @@ write_eval_config <- function(locations, forecast_dates, )) |> dplyr::pull(probability_mass) + # Table of hospital admissions outliers by location-forecast-date-admissions-date: + # This is currently fake/a test. We will replace with a load in to a path + # to a saved csv eventually. + table_of_exclusions <- data.frame( + location = c("TX", "TX", "TX", "TX"), + forecast_date = "2024-02-26", + dates_to_exclude = c("2024-01-30", "2024-01-31", "2024-02-01", "2024-02-02") + ) + config <- list( location_ww = df_ww |> dplyr::pull(location) |> as.vector(), forecast_date_ww = df_ww |> dplyr::pull(forecast_date) |> as.vector(), @@ -106,13 +124,21 @@ write_eval_config <- function(locations, forecast_dates, scenario_dir = scenario_dir, hosp_data_dir = hosp_data_dir, stan_models_dir = stan_models_dir, + baseline_score_table_dir = baseline_score_table_dir, output_dir = output_dir, + hub_subdir = hub_subdir, + score_subdir = score_subdir, + raw_output_dir = raw_output_dir, + figure_dir = figure_dir, + hub_model_names = hub_model_names, population_data_path = population_data_path, init_dir = init_dir, init_fps = init_fps, + overwrite_summary_table = overwrite_summary_table, calibration_time = calibration_time, forecast_time = forecast_time, ww_data_mapping = ww_data_mapping, + table_of_exclusions = table_of_exclusions, # MCMC settings iter_warmup = iter_warmup, iter_sampling = iter_sampling, diff --git a/wweval/DESCRIPTION b/wweval/DESCRIPTION index a3d6427d..2e1cea33 100644 --- a/wweval/DESCRIPTION +++ b/wweval/DESCRIPTION @@ -49,6 +49,7 @@ Imports: tidybayes, lubridate, ggplot2, + colorspace, stats, yaml, tidyr, diff --git a/wweval/NAMESPACE b/wweval/NAMESPACE index ade238ed..6645aa3b 100644 --- a/wweval/NAMESPACE +++ b/wweval/NAMESPACE @@ -3,13 +3,20 @@ export(add_time_indexing) export(clean_ww_data) export(combine_outputs) +export(create_hub_submissions) export(create_mock_submission_scores) export(date_of_ww_data) export(eval_fit_hosp) export(eval_fit_ww) +export(eval_post_process_hosp) +export(eval_post_process_ww) +export(exclude_hosp_outliers) +export(format_for_hub) +export(get_box_plot) export(get_diagnostic_flags) export(get_filepath) export(get_full_scores) +export(get_heatmap_relative_wis) export(get_heatmap_scores) export(get_hosp_data_sizes) export(get_hosp_indices) @@ -19,12 +26,16 @@ export(get_input_ww_data) export(get_last_hosp_data_date) export(get_model_draws_w_data) export(get_model_path) +export(get_n_states_improved_plot) export(get_plot_hosp_data_comparison) +export(get_plot_hub_performance) export(get_plot_quantile_comparison) export(get_plot_raw_scores) export(get_plot_scores_w_data) export(get_plot_summarized_scores) +export(get_plot_wis_over_time) export(get_plot_ww_data_comparison) +export(get_qq_plot) export(get_scenario_site_ids) export(get_scores_from_quantiles) export(get_stan_data_list) @@ -34,13 +45,16 @@ export(get_subpop_data) export(get_ww_data_indices) export(get_ww_data_sizes) export(get_ww_values) +export(make_baseline_score_table) export(make_df) export(sample_model) export(save_table) +export(score_hub_submissions) importFrom(arrow,read_ipc_stream) importFrom(arrow,read_parquet) importFrom(arrow,write_parquet) importFrom(cmdstanr,cmdstan_model) +importFrom(colorspace,scale_fill_continuous_diverging) importFrom(data.table,as.data.table) importFrom(dplyr,arrange) importFrom(dplyr,as_tibble) @@ -75,8 +89,11 @@ importFrom(jsonlite,fromJSON) importFrom(lubridate,ymd) importFrom(readr,read_csv) importFrom(readr,write_csv) +importFrom(rlang,arg_match) importFrom(rlang,sym) importFrom(tidybayes,spread_draws) +importFrom(tidybayes,stat_halfeye) +importFrom(tidybayes,stat_slab) importFrom(tidyr,pivot_longer) importFrom(tidyr,pivot_wider) importFrom(yaml,read_yaml) diff --git a/wweval/R/combine_outputs.R b/wweval/R/combine_outputs.R index 32a88db4..f8462046 100644 --- a/wweval/R/combine_outputs.R +++ b/wweval/R/combine_outputs.R @@ -1,7 +1,7 @@ #' Combine outputs #' @description #' This function is a helper function specific to the current nested file -#' stucture specified by the functions that save the quantiles and scores +#' structure specified by the functions that save the quantiles and scores #' for each of the model runs to disk. The function takes in the #' vectors of scenarios, forecast dates, and locations with output model runs, #' checks if they exist in the current file structure, and if they do @@ -9,8 +9,9 @@ #' all of the outputs row binded #' #' -#' @param output_type the type of output that is saved, either `"quantiles"` or -#' `"scores"` or `"ww_quantiles`. +#' @param output_type the type of output that is saved, one of `"quantiles"`, `"scores"`, +#' `"ww_quantiles"`, `"scores_quantiles"`, `"hosp_quantiles"`,`"errors"`, +#' or `"flags"`. #' @param scenarios The vector of character strings of all the scenarios #' @param forecast_dates The vector of character strings of all the forecast dates #' @param locations The vector of character strings of all the locations @@ -21,12 +22,17 @@ #' forecast_dates, locations, and scenarios #' @export #' -combine_outputs <- function(output_type, +combine_outputs <- function(output_type = + c( + "quantiles", "scores", "ww_quantiles", "scores_quantiles", + "hosp_quantiles", "flags", "errors" + ), scenarios, forecast_dates, locations, eval_output_subdir, model_type) { + output_type <- arg_match(output_type) df <- tibble( scenario = scenarios, forecast_date = forecast_dates, diff --git a/wweval/R/create_hub_submissions.R b/wweval/R/create_hub_submissions.R new file mode 100644 index 00000000..3bbdc81c --- /dev/null +++ b/wweval/R/create_hub_submissions.R @@ -0,0 +1,172 @@ +#' Create hub submission files +#' +#' @description +#' This function takes in the combined set of wastewater and hospital admissions +#' quantiles by forecast date, date, and location, creates a dataframe +#' formatted as it would have been for submission to the COVID forecast Hub, +#' and saves this to disk if specified. It returns the metadata on the number +#' of locations total and the number of locations using the wastewater model +#' for each forecast date submission saved. +#' It assumes the wastewater-informed model output quantiles passed in have already been +#' filtered to remove ones that we would wish to exclude (for convergence +#' diagnostics or other reasons) +#' +#' @param hosp_quantiles_ww a dataframe of the probabilistic estimates of +#' hospital admissions formatted as quantiles, from the wastewater model, +#' for multiple forecast dates if present for a particular location +#' @param hosp_quantiles_hosp a dataframe of the probabilistic estimates of +#' hospital admissions formatted as quantiles, from the hospital admissions +#' model, for multiple forecast dates +#' @param forecast_dates vector of forecast dates that we want to create +#' mock submissions for +#' @param hub_subdir character string of the outer directory specifying where to +#' write the submission files to (which will be named by forecast date) +#' @param model_name character string indicating the name of the model. This will +#' be used to determine both the name of the submission `.csv` file and the name of +#' its enclosing directory, per COVID-19 Forecast Hub formatting. +#' @param scenario string indicating which wastewater availability scenario to use when creating +#' the hub submission file. Default `"status_quo"` (all actually available wastewater data). +#' @param save_files Save the created submission data frame to disk as a `.csv` +#' file? Boolean, default `TRUE`. +#' +#' @return metadata_df a dataframe that has one row per forecast date indicating +#' thenumber of wastewater submissions in that date and the number of total +#' locations +#' @export +#' +create_hub_submissions <- function(hosp_quantiles_ww, + hosp_quantiles_hosp, + forecast_dates, + hub_subdir, + model_name, + scenario = "status_quo", + save_files = TRUE) { + hosp_quantiles_ww <- hosp_quantiles_ww |> + dplyr::filter(scenario == {{ scenario }}) + metadata_df <- data.frame() + for (i in seq_along(forecast_dates)) { + forecast_date <- forecast_dates[i] + first_target_date <- forecast_date + lubridate::days(1) + ww_quantiles <- hosp_quantiles_ww |> + dplyr::filter( + forecast_date == !!forecast_date, + date >= !!first_target_date + ) + hosp_quantiles <- hosp_quantiles_hosp |> + dplyr::filter( + forecast_date == !!forecast_date, + date >= !!first_target_date + ) + + all_locs <- unique(hosp_quantiles$location) + + # For each location in the hospital admissions data, use the + # wastewater model unless it is missing, then use the + # hospital admissions model. + full_quantiles <- data.frame() + for (j in seq_along(all_locs)) { + if (all_locs[j] %in% c(unique(ww_quantiles$location))) { + this_loc_quantiles <- ww_quantiles |> + dplyr::filter(location == all_locs[j]) + } else { # get from the hosp quantiles + this_loc_quantiles <- hosp_quantiles |> + dplyr::filter(location == all_locs[j]) + } + full_quantiles <- dplyr::bind_rows(full_quantiles, this_loc_quantiles) + } + + # Format for the hub + submission_df <- format_for_hub(full_quantiles) + + # A few quality checks + n_models_ww <- full_quantiles |> + dplyr::select(location, model_type) |> + unique() |> + dplyr::filter(model_type == "ww") |> + nrow() + message("Number of locations submitting wastewater model:", n_models_ww) + n_locs <- full_quantiles |> + dplyr::select(location) |> + unique() |> + nrow() + message("Number of locations in submission:", n_locs) + metadata_df <- dplyr::bind_rows( + metadata_df, + data.frame(forecast_date, n_models_ww, n_locs) + ) + + + if (isTRUE(save_files)) { + stopifnot( + "Don't have forecasts for all locations, not writing to disk" = + n_locs >= 51 # temporarily relax bc model convergence flags can + # lead to missing models for hosp model as well, in which case we wouldn't + # have submitted + ) + + cfaforecastrenewalww::create_dir(file.path(hub_subdir, model_name)) + + readr::write_csv(submission_df, file.path( + hub_subdir, model_name, + glue::glue("{forecast_date}-{model_name}.csv") + )) + } + } # end loop over forecast dates + + return(metadata_df) +} + +#' Format quantiles for the current COVID Forecast Hub +#' @description This function takes in a dataframe of quantiles and their +#' values and the names of the columns specifying the date, location, +#' value, quantile, and forecast date, and returns a hub formatted dataframe +#' +#' @param quantiles dataframe of quantiles containing columns with the specified +#' names for date, value, location, forecast date, and quantile +#' @param date_col_name string indicating the column name for date of the +#' forecasted quantity, default is `date` +#' @param value_col_name string indicating the column name for the value of the +#' prediction at the date, default is `value` +#' @param loc_col_name string indicating the column name for the state +#' abbreviation corresponding to the forecast location, default `location` +#' @param forecast_date_col_name string indicating the column name for the forecast +#' date, default is `forecast_date` +#' @param quantile_col_name string indicating the column name for the quantile +#' value, default is `quantile` +#' +#' @return a dataframe in Hub formatting +#' @export +#' +format_for_hub <- function(quantiles, + date_col_name = "date", + value_col_name = "value", + loc_col_name = "location", + forecast_date_col_name = "forecast_date", + quantile_col_name = "quantile") { + formatted_quantiles <- quantiles |> + dplyr::rename( + target_end_date = {{ date_col_name }}, + value = {{ value_col_name }}, + location = {{ loc_col_name }}, + forecast_date = {{ forecast_date_col_name }}, + quantile = {{ quantile_col_name }} + ) |> + dplyr::mutate( + location = cfaforecastrenewalww::loc_abbr_to_flusight_code(location), + quantile = round(quantile, 4), + ) |> + dplyr::filter( + target_end_date >= lubridate::ymd(forecast_date) + lubridate::days(1) + ) |> + dplyr::mutate(days_ahead = as.numeric(target_end_date - forecast_date)) |> + dplyr::mutate( + target = glue::glue("{days_ahead} day ahead inc hosp"), + type = "quantile" + ) |> + dplyr::select( + target, location, forecast_date, target_end_date, + quantile, value, type + ) + + return(formatted_quantiles) +} diff --git a/wweval/R/create_mock_submissions.R b/wweval/R/create_mock_submissions.R index 8d562d63..641230de 100644 --- a/wweval/R/create_mock_submissions.R +++ b/wweval/R/create_mock_submissions.R @@ -8,7 +8,7 @@ #' add it to the dataframe, and continue to concatenate until we get a full #' score dataframe with all combinations of forecast dates, locations, and #' scenarios, labeled according to scenario. We will still know which ones -#' are from which model with the `model`` colum +#' are from which model with the `model`` column #' #' @param all_scores This a dataframe of scores for all the successful model runs. #' Because there are locations in each scenario where the wastewater data is @@ -59,9 +59,7 @@ create_mock_submission_scores <- function(all_scores, location %in% needed_locs ) |> dplyr::mutate(scenario = scenarios[j]) - if (!sjmisc::is_empty(needed_locs)) { - stopifnot("Replacement scores unavailable" = nrow(replacement_scores) > 0) - } + submission_scores <- rbind( scores_from_model, replacement_scores @@ -76,7 +74,45 @@ create_mock_submission_scores <- function(all_scores, nrow() n_expected_combos <- length(forecast_dates) * length(locations) * length(scenarios) - stopifnot("Number of combinations not what is expected" = n_combos == n_expected_combos) + message("Number of expected combos:", n_expected_combos) + message("Number of actual combos:", n_combos) + + # If there are locations/forecast dates with less than the unique number of + # scenarios, this is because the hospital admissions only model didn't + # converge. If this is the case, we probably want to exclude that location + # forecast date bc we can't compare across scenarios. + + exclusions <- all_submission_scores |> + dplyr::distinct(location, forecast_date, scenario) |> + count(location, forecast_date) |> + arrange(n) |> + dplyr::filter(n < length(unique(all_submission_scores$scenario))) + + # Function that excludes rows based on one combination of exclusions + exclude_combination <- function(df, exclusion) { + filtered_df <- df |> dplyr::filter( + !(location == exclusion$location & forecast_date == exclusion$forecast_date) + ) + return(filtered_df) + } + + # Apply the exclusion function for each row in the exclusions dataframe + filtered_scores <- all_submission_scores + for (i in seq_len(nrow(exclusions))) { + filtered_scores <- exclude_combination(filtered_scores, exclusions[i, ]) + } + + # check that all ns are n_unique combos + test <- filtered_scores |> + dplyr::distinct(location, forecast_date, scenario) |> + count(location, forecast_date) |> + arrange(n) + + stopifnot( + "Check that all locations forecast dates have full set of scenarios" = + min(test$n) == length(unique(all_submission_scores$scenario)) + ) + - return(all_submission_scores) + return(filtered_scores) } diff --git a/wweval/R/eval_fit.R b/wweval/R/eval_fit.R index 94608b54..c3785588 100644 --- a/wweval/R/eval_fit.R +++ b/wweval/R/eval_fit.R @@ -1,9 +1,8 @@ -#' Evaluate Wastewater Fit +#' Fit Wastewater Model for Evaluation #' #' @param config_index Index of eval_config to evaluate #' @param eval_config_path Path to eval_config (created with `write_eval_config`) #' @param params_path Path to params.toml -#' @param output_dir Path to directory for storing output #' #' @return NULL #' @export @@ -11,18 +10,22 @@ eval_fit_ww <- function(config_index, eval_config_path, - params_path, - output_dir) { + params_path) { + eval_config <- yaml::read_yaml(eval_config_path) + output_dir <- eval_config$output_dir + raw_output_dir <- eval_config$raw_output_dir + save_object <- function(object_name, output_file_suffix) { saveRDS( object = get(object_name), file = file.path(raw_output_dir, paste0(object_name, output_file_suffix)) ) } - raw_output_dir <- file.path(output_dir, "raw_output") + + cfaforecastrenewalww::create_dir(output_dir) cfaforecastrenewalww::create_dir(raw_output_dir) - eval_config <- yaml::read_yaml(eval_config_path) + params <- cfaforecastrenewalww::get_params(params_path) location <- eval_config$location_ww[config_index] forecast_date <- eval_config$forecast_date_ww[config_index] @@ -58,24 +61,32 @@ eval_fit_ww <- function(config_index, save_object("eval_ww_data", output_file_suffix) + # Get the table of hospital admissions outliers ---------------------------- + table_of_exclusions <- tibble::as_tibble(eval_config$table_of_exclusions) + # Wastewater model fitting loop----------------------------------------------- stan_model_path_target <- get_model_path( model_type = "ww", stan_models_dir = eval_config$stan_models_dir ) - input_hosp_data <- get_input_hosp_data(forecast_date, location, + raw_input_hosp_data <- get_input_hosp_data(forecast_date, location, hosp_data_dir = eval_config$hosp_data_dir, calibration_time = eval_config$calibration_time ) - + input_hosp_data <- exclude_hosp_outliers( + raw_input_hosp_data = raw_input_hosp_data, + forecast_date = forecast_date, + table_of_exclusions = table_of_exclusions + ) save_object("input_hosp_data", output_file_suffix) last_hosp_data_date <- get_last_hosp_data_date(input_hosp_data) - input_ww_data <- get_input_ww_data(forecast_date, - location, - scenario, + input_ww_data <- get_input_ww_data( + forecast_date = forecast_date, + location = location, + scenario = scenario, scenario_dir = eval_config$scenario_dir, ww_data_dir = eval_config$ww_data_dir, calibration_time = eval_config$calibration_time, @@ -88,12 +99,15 @@ eval_fit_ww <- function(config_index, ## Get the stan data for this location, forecast_date, and scenario ---------- standata <- get_stan_data_list( model_type = "ww", - forecast_date, eval_config$forecast_time, - input_ww_data, input_hosp_data, + forecast_date = forecast_date, + forecast_time = eval_config$forecast_time, + calibration_time = eval_config$calibration_time, + input_ww_data = input_ww_data, + input_hosp_data = input_hosp_data, generation_interval = eval_config$generation_interval, inf_to_hosp = eval_config$inf_to_hosp, infection_feedback_pmf = eval_config$infection_feedback_pmf, - params + params = params ) save_object("standata", output_file_suffix) @@ -106,10 +120,10 @@ eval_fit_ww <- function(config_index, save_object("init_lists", output_file_suffix) ww_fit_obj <- sample_model( - standata, + standata = standata, stan_model_path = stan_model_path_target, stan_models_dir = eval_config$stan_models_dir, - init_lists, + init_lists = init_lists, iter_warmup = eval_config$iter_warmup, iter_sampling = eval_config$iter_sampling, adapt_delta = eval_config$adapt_delta, @@ -118,223 +132,34 @@ eval_fit_ww <- function(config_index, seed = eval_config$seed ) save_object("ww_fit_obj", output_file_suffix) - ## Post-processing--------------------------------------------------------- - ww_raw_draws <- ww_fit_obj$draws - save_object("ww_raw_draws", output_file_suffix) - ww_diagnostics <- ww_fit_obj$diagnostics - save_object("ww_diagnostics", output_file_suffix) - ww_diagnostic_summary <- ww_fit_obj$summary_diagnostics - save_object("ww_diagnostic_summary", output_file_suffix) - ww_summary <- ww_fit_obj$summary - save_object("ww_summary", output_file_suffix) - errors <- ww_fit_obj$error - save_object("errors", output_file_suffix) - flags <- ww_fit_obj$flags - save_object("flags", output_file_suffix) - # Save errors - save_table( - data_to_save = errors, - type_of_output = "errors", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - # Save flags - save_table( - data_to_save = flags, - type_of_output = "flags", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - # Get evaluation data from hospital admissions and wastewater - # Join draws with data - hosp_draws <- { - if (is.null(ww_raw_draws)) { - NULL - } else { - get_model_draws_w_data( - model_output = "hosp", - model_type = "ww", - draws = ww_raw_draws, - forecast_date = forecast_date, - scenario = scenario, - location = location, - input_data = input_hosp_data, - eval_data = eval_hosp_data, - last_hosp_data_date = last_hosp_data_date, - ot = eval_config$calibration_time, - forecast_time = eval_config$forecast_time - ) - } - } - save_object("hosp_draws", output_file_suffix) - ww_draws <- { - if (is.null(ww_raw_draws)) { - NULL - } else { - get_model_draws_w_data( - model_output = "ww", - model_type = "ww", - draws = ww_raw_draws, - forecast_date = forecast_date, - scenario = scenario, - location = location, - input_data = input_ww_data, - eval_data = eval_ww_data, - last_hosp_data_date = last_hosp_data_date, - ot = eval_config$calibration_time, - forecast_time = eval_config$forecast_time - ) - } - } - save_object("ww_draws", output_file_suffix) - - full_hosp_quantiles <- { - if (is.null(hosp_draws)) { - NULL - } else { - get_state_level_quantiles( - draws = hosp_draws - ) - } - } - save_object("full_hosp_quantiles", output_file_suffix) - full_ww_quantiles <- { - if (is.null(ww_draws)) { - NULL - } else { - get_state_level_ww_quantiles( - ww_draws = ww_draws - ) - } - } - save_object("full_ww_quantiles", output_file_suffix) - - hosp_quantiles <- { - if (is.null(full_hosp_quantiles)) { - NULL - } else { - full_hosp_quantiles |> - dplyr::filter(period != "calibration") - } - } - save_object("hosp_quantiles", output_file_suffix) - - ww_quantiles <- { - if (is.null(full_ww_quantiles)) { - NULL - } else { - full_ww_quantiles |> - dplyr::filter(period != "calibration") - } - } - save_object("ww_quantiles", output_file_suffix) - # Save forecasted quantiles locally as well as via - # targets caching just for backup - save_table( - data_to_save = full_hosp_quantiles, - type_of_output = "hosp_quantiles", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - - save_table( - data_to_save = full_ww_quantiles, - type_of_output = "ww_quantiles", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - - ### Plot the draw comparison------------------------------------- - plot_hosp_draws <- { - if (is.null(hosp_draws)) { - NULL - } else { - get_plot_hosp_data_comparison( - hosp_draws, - location, - model_type = "ww" - ) - } - } - - save_object("plot_hosp_draws", output_file_suffix) - - plot_ww_draws <- { - if (is.null(ww_draws)) { - NULL - } else { - get_plot_ww_data_comparison( - ww_draws, - location, - model_type = "ww" - ) - } - } - save_object("plot_ww_draws", output_file_suffix) - ## Score hospital admissions forecasts---------------------------------- - hosp_scores <- get_full_scores(hosp_draws, scenario) - save_object("hosp_scores", output_file_suffix) - save_table( - data_to_save = hosp_scores, - type_of_output = "scores", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - - hosp_scores_quantiles <- get_scores_from_quantiles(hosp_quantiles, scenario) - save_object("hosp_scores_quantiles", output_file_suffix) - save_table( - data_to_save = hosp_scores_quantiles, - type_of_output = "scores_quantiles", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "ww", - location = location - ) - # Get a subset of samples for plotting - # Get a subset of quantiles for plotting } -#' Evaluate Hospitalizations Fit +#' Fit Hospitalizations Model for Evaluation #' #' @param config_index Index of eval_config to evaluate #' @param eval_config_path Path to eval_config (created with `write_eval_config`) #' @param params_path Path to params.toml -#' @param output_dir Path to directory for storing output #' #' @return NULL #' @export #' eval_fit_hosp <- function(config_index, eval_config_path, - params_path, - output_dir) { + params_path) { + eval_config <- yaml::read_yaml(eval_config_path) + output_dir <- eval_config$output_dir + raw_output_dir <- eval_config$raw_output_dir + save_object <- function(object_name, output_file_suffix) { saveRDS( object = get(object_name), file = file.path(raw_output_dir, paste0(object_name, output_file_suffix)) ) } - raw_output_dir <- file.path(output_dir, "raw_output") + cfaforecastrenewalww::create_dir(output_dir) cfaforecastrenewalww::create_dir(raw_output_dir) - eval_config <- yaml::read_yaml(eval_config_path) + params <- cfaforecastrenewalww::get_params(params_path) location <- eval_config$location_hosp[config_index] forecast_date <- eval_config$forecast_date_hosp[config_index] @@ -357,29 +182,39 @@ eval_fit_hosp <- function(config_index, save_object("eval_hosp_data", output_file_suffix) + # Get the table of hospital admissions outliers ---------------------------- + table_of_exclusions <- tibble::as_tibble(eval_config$table_of_exclusions) + # Hospital admissions model fitting loop----------------------------------------------- stan_model_path_target <- get_model_path( model_type = "hosp", stan_models_dir = eval_config$stan_models_dir ) - input_hosp_data <- get_input_hosp_data(forecast_date, location, + raw_input_hosp_data <- get_input_hosp_data(forecast_date, location, hosp_data_dir = eval_config$hosp_data_dir, calibration_time = eval_config$calibration_time ) + input_hosp_data <- exclude_hosp_outliers( + raw_input_hosp_data = raw_input_hosp_data, + forecast_date = forecast_date, + table_of_exclusions = table_of_exclusions + ) save_object("input_hosp_data", output_file_suffix) last_hosp_data_date <- get_last_hosp_data_date(input_hosp_data) ## Get the stan data for this location, forecast_date, and scenario ---------- standata <- get_stan_data_list( model_type = "hosp", - forecast_date, eval_config$forecast_time, + forecast_date = forecast_date, + forecast_time = eval_config$forecast_time, + calibration_time = eval_config$calibration_time, input_ww_data = NA, input_hosp_data = input_hosp_data, generation_interval = eval_config$generation_interval, inf_to_hosp = eval_config$inf_to_hosp, infection_feedback_pmf = eval_config$infection_feedback_pmf, - params + params = params ) save_object("standata", output_file_suffix) ## Model fitting----------------------------------------------------------- @@ -390,10 +225,10 @@ eval_fit_hosp <- function(config_index, ) save_object("init_lists", output_file_suffix) hosp_fit_obj <- sample_model( - standata, + standata = standata, stan_model_path = stan_model_path_target, stan_models_dir = eval_config$stan_models_dir, - init_lists, + init_lists = init_lists, iter_warmup = eval_config$iter_warmup, iter_sampling = eval_config$iter_sampling, adapt_delta = eval_config$adapt_delta, @@ -402,111 +237,4 @@ eval_fit_hosp <- function(config_index, seed = eval_config$seed ) save_object("hosp_fit_obj", output_file_suffix) - ## Post-processing--------------------------------------------------------- - hosp_raw_draws <- hosp_fit_obj$draws - save_object("hosp_raw_draws", output_file_suffix) - hosp_diagnostics <- hosp_fit_obj$diagnostics - save_object("hosp_diagnostics", output_file_suffix) - hosp_diagnostic_summary <- hosp_fit_obj$summary_diagnostics - save_object("hosp_diagnostic_summary", output_file_suffix) - hosp_summary <- hosp_fit_obj$summary - save_object("hosp_summary", output_file_suffix) - errors <- hosp_fit_obj$error - save_object("errors", output_file_suffix) - flags <- hosp_fit_obj$flags - save_object("flags", output_file_suffix) - # Save errors - save_table( - data_to_save = errors, - type_of_output = "errors", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "hosp", - location = location - ) - # Save flags - save_table( - data_to_save = flags, - type_of_output = "flags", - output_dir = output_dir, - scenario = scenario, - forecast_date = forecast_date, - model_type = "hosp", - location = location - ) - - - # Get evaluation data from hospital admissions and wastewater - # Join draws with data - hosp_model_hosp_draws <- get_model_draws_w_data( - model_output = "hosp", - model_type = "hosp", - draws = hosp_raw_draws, - forecast_date = forecast_date, - scenario = "no_wastewater", - location = location, - input_data = input_hosp_data, - eval_data = eval_hosp_data, - last_hosp_data_date = last_hosp_data_date, - ot = eval_config$calibration_time, - forecast_time = eval_config$forecast_time - ) - save_object("hosp_model_hosp_draws", output_file_suffix) - full_hosp_model_quantiles <- get_state_level_quantiles( - draws = hosp_model_hosp_draws - ) - save_object("full_hosp_model_quantiles", output_file_suffix) - - hosp_model_quantiles <- full_hosp_model_quantiles |> - dplyr::filter(period != "calibration") - save_object("hosp_model_quantiles", output_file_suffix) - - # Save forecasted quantiles locally as well as via - # targets caching just for backup - save_table( - data_to_save = hosp_model_quantiles, - type_of_output = "quantiles", - output_dir = output_dir, - scenario = "no_wastewater", - forecast_date = forecast_date, - model_type = "hosp", - location = location - ) - - ### Plot the draw comparison------------------------------------- - plot_hosp_draws_hosp_model <- get_plot_hosp_data_comparison( - hosp_model_hosp_draws, - location, - model_type = "hosp" - ) - save_object("plot_hosp_draws_hosp_model", output_file_suffix) - ## Score the hospital admissions only model------------------------- - hosp_scores <- get_full_scores(hosp_model_hosp_draws, - scenario = "no_wastewater" - ) - save_object("hosp_scores", output_file_suffix) - save_table( - data_to_save = hosp_scores, - type_of_output = "scores", - output_dir = output_dir, - scenario = "no_wastewater", - forecast_date = forecast_date, - model_type = "hosp", - location = location - ) - - hosp_scores_quantiles <- get_scores_from_quantiles(hosp_model_quantiles, - scenario = "no_wastewater" - ) - save_object("hosp_scores_quantiles", output_file_suffix) - save_table( - data_to_save = hosp_scores_quantiles, - type_of_output = "scores_quantiles", - output_dir = output_dir, - scenario = "no_wastewater", - forecast_date = forecast_date, - model_type = "hosp", - location = location - ) } diff --git a/wweval/R/eval_post_process.R b/wweval/R/eval_post_process.R new file mode 100644 index 00000000..21070f1a --- /dev/null +++ b/wweval/R/eval_post_process.R @@ -0,0 +1,389 @@ +#' Post Process Wastewater Model for Evaluation +#' +#' @param config_index Index of eval_config to evaluate +#' @param eval_config_path Path to eval_config (created with `write_eval_config`) +#' @param params_path Path to params.toml +#' +#' @return NULL +#' @export +#' +eval_post_process_ww <- function(config_index, + eval_config_path, + params_path) { + eval_config <- yaml::read_yaml(eval_config_path) + output_dir <- eval_config$output_dir + raw_output_dir <- eval_config$raw_output_dir + + save_object <- function(object_name, output_file_suffix) { + saveRDS( + object = get(object_name), + file = file.path(raw_output_dir, paste0(object_name, output_file_suffix)) + ) + } + load_object <- function(object_name, output_file_suffix) { + readRDS(file.path(raw_output_dir, paste0(object_name, output_file_suffix))) + } + + cfaforecastrenewalww::create_dir(output_dir) + cfaforecastrenewalww::create_dir(raw_output_dir) + + + params <- cfaforecastrenewalww::get_params(params_path) + location <- eval_config$location_ww[config_index] + forecast_date <- eval_config$forecast_date_ww[config_index] + scenario <- eval_config$scenario[config_index] + + output_file_suffix <- paste("", location, format(as.Date(forecast_date), "%Y.%m.%d"), scenario, + sep = "_" + ) |> paste0(".rds") + + + + input_hosp_data <- load_object("input_hosp_data", output_file_suffix) + last_hosp_data_date <- get_last_hosp_data_date(input_hosp_data) + eval_hosp_data <- load_object("eval_hosp_data", output_file_suffix) + input_ww_data <- load_object("input_ww_data", output_file_suffix) + eval_ww_data <- load_object("eval_ww_data", output_file_suffix) + ww_fit_obj <- load_object("ww_fit_obj", output_file_suffix) + + ww_raw_draws <- ww_fit_obj$draws + save_object("ww_raw_draws", output_file_suffix) + ww_diagnostics <- ww_fit_obj$diagnostics + save_object("ww_diagnostics", output_file_suffix) + ww_diagnostic_summary <- ww_fit_obj$summary_diagnostics + save_object("ww_diagnostic_summary", output_file_suffix) + ww_summary <- ww_fit_obj$summary + save_object("ww_summary", output_file_suffix) + errors <- ww_fit_obj$error + save_object("errors", output_file_suffix) + flags <- ww_fit_obj$flags + save_object("flags", output_file_suffix) + # Save errors + save_table( + data_to_save = errors, + type_of_output = "errors", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + # Save flags + save_table( + data_to_save = flags, + type_of_output = "flags", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + # Get evaluation data from hospital admissions and wastewater + # Join draws with data + hosp_draws <- { + if (is.null(ww_raw_draws)) { + NULL + } else { + get_model_draws_w_data( + model_output = "hosp", + model_type = "ww", + draws = ww_raw_draws, + forecast_date = forecast_date, + scenario = scenario, + location = location, + input_data = input_hosp_data, + eval_data = eval_hosp_data, + last_hosp_data_date = last_hosp_data_date, + ot = eval_config$calibration_time, + forecast_time = eval_config$forecast_time + ) + } + } + save_object("hosp_draws", output_file_suffix) + ww_draws <- { + if (is.null(ww_raw_draws)) { + NULL + } else { + get_model_draws_w_data( + model_output = "ww", + model_type = "ww", + draws = ww_raw_draws, + forecast_date = forecast_date, + scenario = scenario, + location = location, + input_data = input_ww_data, + eval_data = eval_ww_data, + last_hosp_data_date = last_hosp_data_date, + ot = eval_config$calibration_time, + forecast_time = eval_config$forecast_time + ) + } + } + save_object("ww_draws", output_file_suffix) + + full_hosp_quantiles <- { + if (is.null(hosp_draws)) { + NULL + } else { + get_state_level_quantiles( + draws = hosp_draws + ) + } + } + save_object("full_hosp_quantiles", output_file_suffix) + full_ww_quantiles <- { + if (is.null(ww_draws)) { + NULL + } else { + get_state_level_ww_quantiles( + ww_draws = ww_draws + ) + } + } + save_object("full_ww_quantiles", output_file_suffix) + + hosp_quantiles <- { + if (is.null(full_hosp_quantiles)) { + NULL + } else { + full_hosp_quantiles |> + dplyr::filter(period != "calibration") + } + } + save_object("hosp_quantiles", output_file_suffix) + + ww_quantiles <- { + if (is.null(full_ww_quantiles)) { + NULL + } else { + full_ww_quantiles |> + dplyr::filter(period != "calibration") + } + } + save_object("ww_quantiles", output_file_suffix) + # Save forecasted quantiles locally as well as via + # targets caching just for backup + save_table( + data_to_save = full_hosp_quantiles, + type_of_output = "hosp_quantiles", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + + save_table( + data_to_save = full_ww_quantiles, + type_of_output = "ww_quantiles", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + + ### Plot the draw comparison------------------------------------- + plot_hosp_draws <- { + if (is.null(hosp_draws)) { + NULL + } else { + get_plot_hosp_data_comparison( + hosp_draws, + location, + model_type = "ww" + ) + } + } + + save_object("plot_hosp_draws", output_file_suffix) + + plot_ww_draws <- { + if (is.null(ww_draws)) { + NULL + } else { + get_plot_ww_data_comparison( + ww_draws, + location, + model_type = "ww" + ) + } + } + save_object("plot_ww_draws", output_file_suffix) + ## Score hospital admissions forecasts---------------------------------- + hosp_scores <- get_full_scores(hosp_draws, scenario) + save_object("hosp_scores", output_file_suffix) + save_table( + data_to_save = hosp_scores, + type_of_output = "scores", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + hosp_scores_quantiles <- get_scores_from_quantiles(hosp_quantiles, scenario) + save_object("hosp_scores_quantiles", output_file_suffix) + save_table( + data_to_save = hosp_scores_quantiles, + type_of_output = "scores_quantiles", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "ww", + location = location + ) + # Get a subset of samples for plotting + # Get a subset of quantiles for plotting +} +#' Post Process Hospitalizations Model for Evaluation +#' +#' @param config_index Index of eval_config to evaluate +#' @param eval_config_path Path to eval_config (created with `write_eval_config`) +#' @param params_path Path to params.toml +#' +#' @return NULL +#' @export +#' +eval_post_process_hosp <- function(config_index, + eval_config_path, + params_path) { + eval_config <- yaml::read_yaml(eval_config_path) + output_dir <- eval_config$output_dir + raw_output_dir <- eval_config$raw_output_dir + + save_object <- function(object_name, output_file_suffix) { + saveRDS( + object = get(object_name), + file = file.path(raw_output_dir, paste0(object_name, output_file_suffix)) + ) + } + load_object <- function(object_name, output_file_suffix) { + readRDS(file.path(raw_output_dir, paste0(object_name, output_file_suffix))) + } + + + cfaforecastrenewalww::create_dir(output_dir) + cfaforecastrenewalww::create_dir(raw_output_dir) + + + params <- cfaforecastrenewalww::get_params(params_path) + location <- eval_config$location_hosp[config_index] + forecast_date <- eval_config$forecast_date_hosp[config_index] + scenario <- "no_wastewater" + + output_file_suffix <- paste("", location, format(as.Date(forecast_date), "%Y.%m.%d"), scenario, + sep = "_" + ) |> paste0(".rds") + + input_hosp_data <- load_object("input_hosp_data", output_file_suffix) + last_hosp_data_date <- get_last_hosp_data_date(input_hosp_data) + eval_hosp_data <- load_object("eval_hosp_data", output_file_suffix) + hosp_fit_obj <- load_object("hosp_fit_obj", output_file_suffix) + + hosp_raw_draws <- hosp_fit_obj$draws + save_object("hosp_raw_draws", output_file_suffix) + hosp_diagnostics <- hosp_fit_obj$diagnostics + save_object("hosp_diagnostics", output_file_suffix) + hosp_diagnostic_summary <- hosp_fit_obj$summary_diagnostics + save_object("hosp_diagnostic_summary", output_file_suffix) + hosp_summary <- hosp_fit_obj$summary + save_object("hosp_summary", output_file_suffix) + errors <- hosp_fit_obj$error + save_object("errors", output_file_suffix) + flags <- hosp_fit_obj$flags + save_object("flags", output_file_suffix) + # Save errors + save_table( + data_to_save = errors, + type_of_output = "errors", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "hosp", + location = location + ) + # Save flags + save_table( + data_to_save = flags, + type_of_output = "flags", + output_dir = output_dir, + scenario = scenario, + forecast_date = forecast_date, + model_type = "hosp", + location = location + ) + + + # Get evaluation data from hospital admissions and wastewater + # Join draws with data + hosp_model_hosp_draws <- get_model_draws_w_data( + model_output = "hosp", + model_type = "hosp", + draws = hosp_raw_draws, + forecast_date = forecast_date, + scenario = "no_wastewater", + location = location, + input_data = input_hosp_data, + eval_data = eval_hosp_data, + last_hosp_data_date = last_hosp_data_date, + ot = eval_config$calibration_time, + forecast_time = eval_config$forecast_time + ) + save_object("hosp_model_hosp_draws", output_file_suffix) + full_hosp_model_quantiles <- get_state_level_quantiles( + draws = hosp_model_hosp_draws + ) + save_object("full_hosp_model_quantiles", output_file_suffix) + + hosp_model_quantiles <- full_hosp_model_quantiles |> + dplyr::filter(period != "calibration") + save_object("hosp_model_quantiles", output_file_suffix) + + # Save forecasted quantiles locally as well as via + # targets caching just for backup + save_table( + data_to_save = hosp_model_quantiles, + type_of_output = "quantiles", + output_dir = output_dir, + scenario = "no_wastewater", + forecast_date = forecast_date, + model_type = "hosp", + location = location + ) + + ### Plot the draw comparison------------------------------------- + plot_hosp_draws_hosp_model <- get_plot_hosp_data_comparison( + hosp_model_hosp_draws, + location, + model_type = "hosp" + ) + save_object("plot_hosp_draws_hosp_model", output_file_suffix) + ## Score the hospital admissions only model------------------------- + hosp_scores <- get_full_scores(hosp_model_hosp_draws, + scenario = "no_wastewater" + ) + save_object("hosp_scores", output_file_suffix) + save_table( + data_to_save = hosp_scores, + type_of_output = "scores", + output_dir = output_dir, + scenario = "no_wastewater", + forecast_date = forecast_date, + model_type = "hosp", + location = location + ) + hosp_scores_quantiles <- get_scores_from_quantiles(hosp_model_quantiles, + scenario = "no_wastewater" + ) + save_object("hosp_scores_quantiles", output_file_suffix) + save_table( + data_to_save = hosp_scores_quantiles, + type_of_output = "scores_quantiles", + output_dir = output_dir, + scenario = "no_wastewater", + forecast_date = forecast_date, + model_type = "hosp", + location = location + ) +} diff --git a/wweval/R/exclude_hosp_outliers.R b/wweval/R/exclude_hosp_outliers.R new file mode 100644 index 00000000..e4102d26 --- /dev/null +++ b/wweval/R/exclude_hosp_outliers.R @@ -0,0 +1,45 @@ +#' Exclude hospital outliers +#' +#' @param raw_input_hosp_data A dataframe containing the location, the date of +#' admission, and the number of hospital admissions, in that location +#' @param forecast_date The forecast date as a character string +#' @param table_of_exclusions A table containing the combinations of +#' locations, forecast dates, and dates to exclude. These will be treated +#' as missing by the model +#' @param col_name_dates_to_exclude The name of the column in the table of +#' exclusiosn that cooresponds to the date of hospital admissions to exclude, +#' default is `dates_to_exclude` +#' @return A dataframe in the same format as the `raw_input_hosp_data` but +#' with the location-forecast-date-date admissions rows excluded if indicated +#' in the `table_of_exclusions` +#' @export +#' +exclude_hosp_outliers <- function(raw_input_hosp_data, + forecast_date, + table_of_exclusions, + col_name_dates_to_exclude = "dates_to_exclude") { + # Filter table of exclusions to the relevant forecast date and location + + loc <- raw_input_hosp_data |> + dplyr::pull(location) |> + unique() + + stopifnot("Only one location passed in" = length(loc) == 1) + + exclusions <- table_of_exclusions |> + dplyr::filter( + location == loc, + forecast_date == forecast_date + ) + + if (nrow(exclusions) == 0) { + input_hosp_data <- raw_input_hosp_data + } else { + dates_to_exclude <- exclusions |> + dplyr::pull({{ col_name_dates_to_exclude }}) + input_hosp_data <- raw_input_hosp_data |> + dplyr::filter(!date %in% c(dates_to_exclude)) + } + + return(input_hosp_data) +} diff --git a/wweval/R/filepath_mapping.R b/wweval/R/filepath_mapping.R index ceb06c88..0aa5905b 100644 --- a/wweval/R/filepath_mapping.R +++ b/wweval/R/filepath_mapping.R @@ -2,7 +2,7 @@ #' #' @param model_type string specifying the model to be run, options are either #' 'hosp' for the hospital admissions only model or -#' 'ww' for the site-level infection dyanmics model using wastewater +#' 'ww' for the site-level infection dynamics model using wastewater #' @param stan_models_dir directory where stan files are located #' #' @return string indicating path to correct stan file @@ -30,7 +30,7 @@ get_model_path <- function(model_type, stan_models_dir) { #' @description #' A function to return a character string for each item to either be saved or #' read in, based on the nesting that is specified in this function. To change -#' the nesting, we will change this funciton +#' the nesting, we will change this function #' #' #' @param output_subdir Character string to upper level directory where outputs diff --git a/wweval/R/format_data_for_stan.R b/wweval/R/format_data_for_stan.R index efa331a6..19c5124c 100644 --- a/wweval/R/format_data_for_stan.R +++ b/wweval/R/format_data_for_stan.R @@ -5,6 +5,8 @@ #' @param forecast_date string indicating the forecast date #' @param forecast_time integer indicating the number of days to make a forecast #' for +#' @param calibration_time integer indicating the max duration in days that +#' the model is calibrated to hospital admissions for #' @param input_ww_data a dataframe with the input wastewater data #' @param input_hosp_data a dataframe with the input hospital admissions data #' @param generation_interval a vector with a zero-truncated normalized pmf of @@ -29,7 +31,7 @@ #' @param ww_value_lod_col_name A string representing the name of the column #' in the input_ww_data that indicates the value of the LOD in natural scale, #' default is `lod_sewage` -#' @param hosp_value_col_name A string represeting the name of the column in the +#' @param hosp_value_col_name A string representing the name of the column in the #' input_hosp-data that indicates the number of daily hospital admissions, #' default is `daily_hosp_admits` #' @@ -38,6 +40,7 @@ get_stan_data_list <- function(model_type, forecast_date, forecast_time, + calibration_time, input_ww_data, input_hosp_data, generation_interval, @@ -135,12 +138,13 @@ get_stan_data_list <- function(model_type, # Get the remaining things needed for both models hosp_data <- add_time_indexing(input_hosp_data) hosp_data_sizes <- get_hosp_data_sizes( - hosp_data, - forecast_date, - forecast_time, - last_hosp_data_date, - uot, - hosp_value_col_name + input_hosp_data = hosp_data, + forecast_date = forecast_date, + forecast_time = forecast_time, + calibration_time = calibration_time, + last_hosp_data_date = last_hosp_data_date, + uot = uot, + hosp_value_col_name = hosp_value_col_name ) hosp_indices <- get_hosp_indices(hosp_data) hosp_values <- get_hosp_values( @@ -220,6 +224,8 @@ get_stan_data_list <- function(model_type, viral_shedding_pars = viral_shedding_pars, # tpeak, viral peak, dur_shed autoreg_rt_a = autoreg_rt_a, autoreg_rt_b = autoreg_rt_b, + autoreg_rt_site_a = autoreg_rt_site_a, + autoreg_rt_site_b = autoreg_rt_site_b, autoreg_p_hosp_a = autoreg_p_hosp_a, autoreg_p_hosp_b = autoreg_p_hosp_b, inv_sqrt_phi_prior_mean = inv_sqrt_phi_prior_mean, @@ -607,7 +613,7 @@ get_ww_data_indices <- function(ww_data, #' the ww_data that indicates the number of people represented by that wastewater #' catchment #' @param one_pop_per_site a boolean variable indicating if there should only -#' be on catchment area population per site, default is `TRUE` bc this is what +#' be on catchment area population per site, default is `TRUE` because this is what #' the stan model expects #' #' @return A list containing the necessary vectors of values that @@ -748,6 +754,8 @@ get_subpop_data <- function(add_auxiliary_site, #' @param forecast_date string indicating the forecast date #' @param forecast_time integer indicating the number of days to make a forecast #' for +#' @param calibration_time integer indicating the max duration in days that +#' the model is calibrated to hospital admissions for #' @param last_hosp_data_date string indicating the date of the last observed #' hospital admission #' @param uot integer indicating the time of model initialization when there are @@ -770,6 +778,7 @@ get_subpop_data <- function(add_auxiliary_site, get_hosp_data_sizes <- function(input_hosp_data, forecast_date, forecast_time, + calibration_time, last_hosp_data_date, uot, hosp_value_col_name = "daily_hosp_admits") { @@ -783,7 +792,7 @@ get_hosp_data_sizes <- function(input_hosp_data, tot_weeks <- ceiling((ot + uot + ht) / 7) hosp_data_sizes <- list( ht = ht, - ot = ot, + ot = calibration_time, oht = oht, n_weeks = n_weeks, tot_weeks = tot_weeks @@ -816,7 +825,7 @@ get_hosp_indices <- function(input_hosp_data) { #' model has available calibration data in days #' @param ht integer indicating the number of days to produce hospital admissions #' outside the calibration period (forecast + nowcast time) in days -#' @param hosp_value_col_name A string represeting the name of the column in the +#' @param hosp_value_col_name A string representing the name of the column in the #' input_hosp-data that indicates the number of daily hospital admissions, #' default is `daily_hosp_admits` #' diff --git a/wweval/R/plots.R b/wweval/R/plots.R index 90930cd5..1c00ba52 100644 --- a/wweval/R/plots.R +++ b/wweval/R/plots.R @@ -4,7 +4,7 @@ #' the estimated wastewater concentrations in each site joined with both the data #' the model was calibrated to and the later observed data for evaluating the #' future predicted concentrations against. -#' @param location the jursidiction the data is from +#' @param location the jurisdiction the data is from #' @param model_type type of model the output is from, default is `ww` #' @param n_draws number of draws to plot, default = 100 #' @@ -34,7 +34,9 @@ get_plot_ww_data_comparison <- function(draws_w_data, color = "black", show.legend = FALSE ) + - geom_vline(aes(xintercept = ymd(forecast_date)), linetype = "dashed") + + geom_vline(aes(xintercept = lubridate::ymd(forecast_date)), + linetype = "dashed" + ) + scale_y_continuous(trans = "log10") + facet_wrap(~site_lab_name, scales = "free") + geom_point( @@ -89,8 +91,9 @@ get_plot_ww_data_comparison <- function(draws_w_data, #' @export get_plot_hosp_data_comparison <- function(draws_w_data, location, - model_type, + model_type = c("ww", "hosp"), n_draws = 100) { + model_type <- arg_match(model_type) sampled_draws <- sample(1:max(draws_w_data$draw), n_draws) draws_w_data_subsetted <- draws_w_data |> dplyr::filter( @@ -104,7 +107,7 @@ get_plot_hosp_data_comparison <- function(draws_w_data, p <- ggplot(draws_w_data_subsetted) + geom_line(aes(x = date, y = value, group = draw), color = plot_color, - linewidth = 0.1, alpha = 0.1, + linewidth = 0.2, alpha = 0.4, show.legend = FALSE ) + geom_point(aes(x = date, y = eval_data), @@ -115,8 +118,10 @@ get_plot_hosp_data_comparison <- function(draws_w_data, color = "black", show.legend = FALSE ) + - geom_vline(aes(xintercept = ymd(forecast_date)), linetype = "dashed") + - scale_y_continuous(trans = "log10") + + geom_vline(aes(xintercept = lubridate::ymd(forecast_date)), + linetype = "dashed" + ) + + # scale_y_continuous(trans = "log10") + xlab("") + ylab("Daily hospital admissions") + ggtitle(glue::glue( @@ -149,8 +154,11 @@ get_plot_hosp_data_comparison <- function(draws_w_data, #' #' @param hosp_quantiles dataframe of hospital admissions quantiles #' @param eval_data Hospital admissions data for visual comparison +#' @param figure_file_path Outer directory for plots to go in #' @param days_to_show_forecast Number of days to show the forecast, default #' vlaue is 28 +#' @param save_files Whether or not to write to a folder of plots, +#' default is `TRUE` #' #' @return a ggplot object with forecasts overlaid with evaluation data for #' each scenarion in a specific location for visual comparison @@ -158,7 +166,9 @@ get_plot_hosp_data_comparison <- function(draws_w_data, #' get_plot_quantile_comparison <- function(hosp_quantiles, eval_data, - days_to_show_forecast = 28) { + figure_file_path, + days_to_show_forecast = 28, + save_files = TRUE) { location <- hosp_quantiles |> pull(location) |> unique() @@ -171,7 +181,7 @@ get_plot_quantile_comparison <- function(hosp_quantiles, dplyr::filter( location == !!location, date <= max(hosp_quantiles$date), - date >= min(hosp_quantiles$date) + date >= min(hosp_quantiles$date[hosp_quantiles$period == "nowcast"]) ) @@ -193,12 +203,12 @@ get_plot_quantile_comparison <- function(hosp_quantiles, geom_point( data = eval_data_subsetted, aes(x = date, y = daily_hosp_admits), - color = "black" + color = "black", alpha = 0.3 ) + geom_line( data = eval_data_subsetted, aes(x = date, y = daily_hosp_admits), - color = "black" + color = "black", alpha = 0.3 ) + geom_line( data = quantiles_wide |> filter( @@ -252,13 +262,29 @@ get_plot_quantile_comparison <- function(hosp_quantiles, hjust = 1, angle = 45 ), axis.title.x = element_text(size = 12), - axis.title.y = element_text(size = 12), + axis.title.y = element_text(size = 10), plot.title = element_text( size = 10, vjust = 0.5, hjust = 0.5 ) ) + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "quantile_comparison") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("quantiles_{location}.png") + ), + plot = p, + width = 9, + height = 9, + units = "in", + bg = "white" + ) + } + return(p) } @@ -349,7 +375,7 @@ get_plot_ww_comparison <- function(ww_quantiles, hjust = 1, angle = 45 ), axis.title.x = element_text(size = 12), - axis.title.y = element_text(size = 12), + axis.title.y = element_text(size = 10), plot.title = element_text( size = 8, vjust = 0.5, hjust = 0.5 @@ -400,8 +426,11 @@ get_plot_raw_scores <- function(all_scores, #' and forecast date, for a single location #' @param eval_data a dataframe containing the hospital admissions data #' that the forecasts are evaluated against +#' @param figure_file_path Outer directory for plots to go in #' @param score_metric A string indicating the score metric to plot, #' default is "crps" +#' @param save_files Whether or not to write to a folder of plots, +#' default is `TRUE` #' #' @return a ggplot object plotting the scores summarized by forecast date #' and scenario over time with the data overlaid @@ -409,7 +438,9 @@ get_plot_raw_scores <- function(all_scores, #' get_plot_scores_w_data <- function(all_scores, eval_data, - score_metric = "crps") { + figure_file_path, + score_metric = "crps", + save_files = TRUE) { location <- all_scores |> pull(location) |> unique() @@ -419,7 +450,7 @@ get_plot_scores_w_data <- function(all_scores, dplyr::filter( location == !!location, date <= max(all_scores$date), - date >= min(all_scores$date) + date >= min(all_scores$forecast_date) - lubridate::days(10) ) coeff <- ( median(eval_data_subsetted$daily_hosp_admits, na.rm = TRUE) / @@ -437,23 +468,29 @@ get_plot_scores_w_data <- function(all_scores, mutate(forecast_date = lubridate::ymd(forecast_date)) |> data.table::as.data.table() |> scoringutils::summarize_scores( - by = c("scenario", "period", "forecast_date", "location") + by = c("scenario", "forecast_date", "location") ) - n_periods <- summarized_scores |> - dplyr::pull(period) |> - unique() |> - length() - location <- summarized_scores |> - pull(location) |> - unique() + baseline_scores <- summarized_scores |> + tibble::as_tibble() |> + dplyr::filter(scenario == "no_wastewater") |> + rename(baseline_score = {{ score_metric }}) |> + select(location, forecast_date, baseline_score) + summary_across_dates <- all_scores |> data.table::as.data.table() |> scoringutils::summarize_scores( - by = c("period", "location", "scenario") + by = c("location", "scenario") ) + coeff <- ( + median(eval_data_subsetted$daily_hosp_admits, na.rm = TRUE) / + median(summarized_scores |> + dplyr::select({{ score_metric }}) |> # nolint + dplyr::pull(), na.rm = TRUE) + ) + p <- ggplot(summarized_scores) + geom_bar(aes(x = forecast_date, y = .data[[score_metric]] * coeff, fill = scenario), stat = "identity", position = "dodge", alpha = 0.5 @@ -461,14 +498,13 @@ get_plot_scores_w_data <- function(all_scores, geom_point( data = eval_data_subsetted, aes(x = date, y = daily_hosp_admits), - color = "black" + color = "black", alpha = 0.3 ) + geom_line( data = eval_data_subsetted, aes(x = date, y = daily_hosp_admits), - color = "black" + color = "black", alpha = 0.3 ) + - facet_wrap(~period, nrow = n_periods) + theme_bw() + xlab("") + ggtitle(glue::glue( @@ -496,13 +532,28 @@ get_plot_scores_w_data <- function(all_scores, hjust = 1, angle = 45 ), axis.title.x = element_text(size = 12), - axis.title.y = element_text(size = 12), + axis.title.y = element_text(size = 10), plot.title = element_text( size = 10, vjust = 0.5, hjust = 0.5 ) ) + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "scores_w_data_overlaid") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("scores_{location}.png") + ), + plot = p, + width = 9, + height = 5, + units = "in", + bg = "white" + ) + } return(p) } @@ -630,3 +681,561 @@ get_heatmap_scores <- function(mock_submission_scores, return(p) } + +#' Get box plot by date and scenario +#' +#' @param all_scores the full set of scores for each location, forecast date, +#' scenario, and date +#' @param figure_file_path Outer directory for plots to go in +#' @param baseline_scenario the scenario to compute the difference in +#' crps scores relative to, default is `no_wastewater` +#' @param save_files Whether or not to write to a folder of plots, +#' default is `TRUE` +#' +#' @return p a ggplot object with a box plot showing the average improvement in +#' scores by forecast date and scenario for each location +#' @export +#' +get_box_plot <- function(all_scores, + figure_file_path, + baseline_scenario = "no_wastewater", + save_files = TRUE) { + scores_by_date_scen_loc <- scoringutils::summarize_scores(all_scores, + by = c( + "scenario", + "location", + "forecast_date" + ) + ) + + baseline_only <- scores_by_date_scen_loc |> + dplyr::filter(scenario == {{ baseline_scenario }}) |> + rename(baseline_score = crps) |> + select(location, forecast_date, baseline_score) + + overall_scores <- scores_by_date_scen_loc |> + dplyr::left_join(baseline_only, + by = c("location", "forecast_date") + ) |> + group_by(location, forecast_date, scenario) |> + dplyr::summarize( + relative_score = crps / baseline_score, + diff_in_score = baseline_score - crps, + mean_crps = mean(crps) + ) + + # Make a violin plot + + p <- ggplot( + overall_scores, + aes( + x = factor(forecast_date), y = diff_in_score, + fill = scenario, color = scenario + ) + ) + + geom_boxplot(position = "dodge", alpha = 0.5) + + xlab("") + + ylab(glue::glue( + "CRPS improvement (`{baseline_scenario}` - scenario score)" + )) + + theme_bw() + + theme( + axis.text.x = element_text( + size = 8, vjust = 1, + hjust = 1, angle = 45 + ), + axis.title.x = element_text(size = 12), + axis.title.y = element_text(size = 10), + plot.title = element_text( + size = 10, + vjust = 0.5, hjust = 0.5 + ) + ) + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path) + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("crps_improvement_over_time.png") + ), + plot = p, + width = 9, + height = 9, + units = "in", + bg = "white" + ) + } + + return(p) +} + +#' Get bar chart of the number of states improved compared to without wastewater +#' over each forecast date +#' +#' @param all_scores the full set of scores for each location, forecast date, +#' scenario, and date +#' @param figure_file_path Outer directory for plots to go in +#' @param baseline_scenario the scenario to compute the difference in +#' crps scores relative to, default is `no_wastewater` +#' @param threshold_for_improvement the relative reduction that we will consider +#' as an improvement over the baseline, +#' @param save_files Whether or not to write to a folder of plots, +#' default is `TRUE` +#' +#' @return p a ggplot object with a bar chart showing the number of +#' jurisdictions with an improvement compared to the forecast without +#' wastewater under each scenario. +#' @export +#' +get_n_states_improved_plot <- function(all_scores, + figure_file_path, + baseline_scenario = "no_wastewater", + threshold_for_improvement = 1, + save_files = TRUE) { + scores_by_date_scen_loc <- scoringutils::summarize_scores(all_scores, + by = c( + "scenario", + "location", + "forecast_date" + ) + ) + + baseline_only <- scores_by_date_scen_loc |> + dplyr::filter(scenario == {{ baseline_scenario }}) |> + rename(baseline_score = crps) |> + select(location, forecast_date, baseline_score) + + overall_scores <- scores_by_date_scen_loc |> + dplyr::left_join(baseline_only, + by = c("location", "forecast_date") + ) |> + group_by(location, forecast_date, scenario) |> + dplyr::summarize( + relative_score = crps / baseline_score, + diff_in_score = baseline_score - crps, + mean_crps = mean(crps) + ) |> + dplyr::mutate( + is_improved = ifelse(relative_score < threshold_for_improvement, 1, 0) + ) + + summarize_scenarios <- overall_scores |> + dplyr::group_by(scenario, forecast_date) |> + dplyr::summarize( + n_states_improved = sum(is_improved) + ) + + p <- ggplot(summarize_scenarios) + + geom_bar( + aes( + x = factor(forecast_date), + y = n_states_improved, + fill = scenario + ), + alpha = 0.5, + stat = "identity", + position = "dodge" + ) + + xlab("") + + ylab("Number of states with improved forecasts") + + theme_bw() + + theme( + axis.text.x = element_text( + size = 8, vjust = 1, + hjust = 1, angle = 45 + ), + axis.title.x = element_text(size = 12), + axis.title.y = element_text(size = 10), + plot.title = element_text( + size = 10, + vjust = 0.5, hjust = 0.5 + ) + ) + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path) + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("n_states_improved_over_time.png") + ), + plot = p, + width = 9, + height = 9, + units = "in", + bg = "white" + ) + } + + return(p) +} + + +#' Get plot of WIS over time +#' +#' @param all_scores Scores from entire time period of interest, including +#' the retrospective cfa model +#' @param cfa_real_time_scores Real-time scores from Feb - Mar for the cfa ww +#' model submitted to the hub +#' @param figure_file_path directory to save figures in +#' @param horizon_time_in_weeks horizon time in weeks to summarize over, default +#' is `NULL` which means that the scores are summarized over 4 weeks +#' @param save_files bolean indicating whether or not to save the files, default +#' is `TRUE` +#' +#' @return a ggplot object of WIS scores over time colored by model, for the +#' real-time cfa model from Feb - Mar and the retrospective CFA model over +#' all time points +#' @export +#' +get_plot_wis_over_time <- function(all_scores, + cfa_real_time_scores, + figure_file_path, + horizon_time_in_weeks = NULL, + save_files = TRUE) { + scores <- dplyr::bind_rows(all_scores, cfa_real_time_scores) + + if (!is.null(horizon_time_in_weeks)) { + scores_by_forecast_date <- scores |> + data.table::as.data.table() |> + scoringutils::summarise_scores(by = c( + "forecast_date", + "model", "horizon" + )) |> + dplyr::filter(horizon_weeks == { + horizon_time_in_weeks + }) + title <- glue::glue( + "Average {horizon_time_in_weeks}-week ahead weighted interval scores by model" + ) + } else { + scores_by_forecast_date <- scores |> + data.table::as.data.table() |> + scoringutils::summarise_scores(by = c( + "forecast_date", + "model" + )) + title <- glue::glue("Average weighted interval scores by model") + } + + p <- ggplot(scores_by_forecast_date) + + geom_line(aes( + x = forecast_date, y = interval_score, + color = model + )) + + geom_point(aes( + x = forecast_date, y = interval_score, + color = model + )) + + xlab("Forecast date") + + ylab("Average WIS score across locations") + + theme_bw() + + theme( + legend.position = "bottom", + axis.text.x = element_text( + size = 8, vjust = 1, + hjust = 1, angle = 45 + ), + axis.title.x = element_text(size = 12), + axis.title.y = element_text(size = 10), + plot.title = element_text( + size = 10, + vjust = 0.5, hjust = 0.5 + ) + ) + + scale_x_date( + date_breaks = "2 weeks", + labels = scales::date_format("%Y-%m-%d") + ) + + ggtitle(title) + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "hub_comparison") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("average_WIS_over_time_by_model.png") + ), + plot = p, + width = 8, + height = 4, + units = "in", + bg = "white" + ) + } + + return(p) +} + + +#' Get plot of overall hub performance +#' +#' @param all_scores df with granular (daily) scores from every model, +#' forecast_date, and location for the entire time period. Includes the +#' two retrospective models +#' @param cfa_real_time_scores df with granular (daily) scores from the +#' submitted cfa ww model for the time period when it was submitted. +#' @param figure_file_path path to directory to save figures +#' @param all_time_period string indicating the longer time frame we are +#' comparing, e.g. "Oct 2023-Mar 2024" +#' @param real_time_period string indicating the shorter time frame that +#' we submitted our model to the hub e.g. "Feb 2024-Mar 2024" +#' @param baseline_model which model to compute relative WIS compared to, default +#' is `COVIDhub-baseline` +#' @param save_files boolean indicating whether or not to save figures, default +#' is `TRUE` +#' +#' @return a ggplot object containing distributions of WIS scores grouped by +#' model and the comaprison time period, with the mean plotted alongside the +#' full distribution +#' @export +#' +get_plot_hub_performance <- function(all_scores, + cfa_real_time_scores, + figure_file_path, + all_time_period, + real_time_period, + baseline_model = "COVIDhub-baseline", + save_files = TRUE) { + scores_by_model_all_time <- all_scores |> + data.table::as.data.table() |> + scoringutils::summarise_scores( + by = c("model", "forecast_date", "location", "horizon") + ) |> + dplyr::mutate( + period = {{ all_time_period }} + ) + + scores_by_model_real_time <- all_scores |> + dplyr::filter(forecast_date >= lubridate::ymd("2024-02-05")) |> + dplyr::bind_rows(cfa_real_time_scores) |> + data.table::as.data.table() |> + scoringutils::summarise_scores( + by = c("model", "forecast_date", "location") + ) |> + dplyr::mutate( + period = {{ real_time_period }} + ) + + scores <- dplyr::bind_rows( + scores_by_model_all_time, + scores_by_model_real_time + ) + + # Want to get the mean across all forecast dates and locations for each + # model during each period + mean_scores <- scores |> + scoringutils::summarise_scores( + by = c("model", "period") + ) |> + dplyr::rename( + mean_score = interval_score + ) |> + dplyr::select( + model, period, mean_score + ) + + baseline_scores <- scores |> + dplyr::filter(model == {{ baseline_model }}) |> + dplyr::select(location, forecast_date, horizon, interval_score) |> + dplyr::rename(baseline_score = interval_score) + + scores_final <- scores |> + dplyr::left_join(mean_scores, by = c("model", "period")) |> + dplyr::left_join(baseline_scores, by = c( + "forecast_date", "horizon", + "location" + )) |> + dplyr::mutate(relative_wis = interval_score / baseline_score) |> + dplyr::filter(model != {{ baseline_model }}) + + p <- ggplot(scores_final) + + tidybayes::stat_halfeye( + aes( + x = period, y = relative_wis + 1e-8, + fill = model + ), + point_interval = "mean_qi", + alpha = 0.5, + position = position_dodge(width = 0.75) + ) + + theme_bw() + + coord_trans(ylim = c(0, 2)) + + theme( + legend.position = "bottom", + legend.text = element_text(size = 8), + axis.title.x = element_text(size = 12), + axis.title.y = element_text(size = 10), + plot.title = element_text( + size = 10, + vjust = 0.5, hjust = 0.5 + ) + ) + + xlab("Time period") + + ylab(glue::glue("Relative WIS compared to {baseline_model}")) + + ggtitle("Distribution of relative WIS across locations, horizons, and forecast dates") + + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "hub_comparison") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("overall_performance_WIS_distros.png") + ), + plot = p, + width = 8, + height = 4, + units = "in", + bg = "white" + ) + } + + + return(p) +} + + +#' Get heatmap of relative WIS +#' +#' @param scores df of granular (daily) score across models, locations, forecast +#' dates and horizons +#' @param figure_file_path path to save figure +#' @param time_period time period that scores are summarized over +#' @param baseline_model which model to compute relative WIS compared to, default +#' is `COVIDhub-baseline` +#' @param save_files save_files boolean indicating whether or not to save figures, default +#' is `TRUE` +#' +#' @return a ggplot with a heatmap with model on the x-axis, location on the y-axis +#' and fill by relative WIS score across forecast dates and horizons +#' @export +#' +get_heatmap_relative_wis <- function(scores, + figure_file_path, + time_period, + baseline_model = "COVIDhub-baseline", + save_files = TRUE) { + summarized_scores <- scores |> + data.table::as.data.table() |> + scoringutils::summarise_scores( + by = c("model", "location") + ) |> + dplyr::left_join(cfaforecastrenewalww::flusight_location_table, + by = c("location" = "location_code") + ) + + baseline_score <- summarized_scores |> + dplyr::filter(model == {{ baseline_model }}) |> + dplyr::rename( + baseline_wis = interval_score + ) |> + dplyr::select(location, baseline_wis) + + relative_scores <- summarized_scores |> + dplyr::left_join(baseline_score, + by = c("location") + ) |> + dplyr::mutate( + relative_interval_score = interval_score / baseline_wis + ) |> + dplyr::filter(model != {{ baseline_model }}) + + + p <- ggplot(relative_scores) + + geom_tile(aes(x = model, y = short_name, fill = relative_interval_score)) + + scale_fill_gradient2( + high = "red", mid = "white", low = "blue", transform = "log2", + midpoint = 1, guide = "colourbar", aesthetics = "fill" + ) + + geom_text(aes( + x = model, y = short_name, + label = round(relative_interval_score, 2) + ), size = 2.5) + + theme_bw() + + theme( + axis.text.x = element_text( + size = 8, vjust = 1, + hjust = 1, angle = 45 + ), + axis.title.x = element_text(size = 12), + axis.title.y = element_text(size = 10), + plot.title = element_text( + size = 10, + vjust = 0.5, hjust = 0.5 + ) + ) + + xlab("") + + ylab("") + + labs(fill = "Relative WIS") + + ggtitle(glue::glue("Relative WIS compared to {baseline_model} from {time_period}")) + + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "hub_comparison") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("heatmap_{baseline_model}_{time_period}.png") + ), + plot = p, + width = 6, + height = 10, + units = "in", + bg = "white" + ) + } + + + + return(p) +} + +#' Get quantile-quantile plot +#' +#' @param scores df of granular (daily) score across models, locations, forecast +#' dates and horizons +#' @param figure_file_path path to save figure +#' @param time_period time period that scores are summarized over +#' @param save_files save_files boolean indicating whether or not to save figures, default +#' is `TRUE` +#' +#' @return a ggplot object containing a plot of the proportion of data within +#' each interval for each model. +#' @export +#' +get_qq_plot <- function(scores, + figure_file_path, + time_period, + save_files = TRUE) { + p <- scores |> + data.table::as.data.table() |> + scoringutils::summarise_scores(by = c("model", "range")) |> + scoringutils::plot_interval_coverage() + + ggtitle(glue::glue("QQ plot for {time_period}")) + + + if (isTRUE(save_files)) { + full_file_path <- file.path(figure_file_path, "hub_comparison") + cfaforecastrenewalww::create_dir(full_file_path) + ggsave( + file.path( + full_file_path, + glue::glue("qq_plot_{time_period}.png") + ), + plot = p, + width = 6, + height = 6, + units = "in", + bg = "white" + ) + } + + return(p) +} diff --git a/wweval/R/post_processing.R b/wweval/R/post_processing.R index bdd01cd8..673dc180 100644 --- a/wweval/R/post_processing.R +++ b/wweval/R/post_processing.R @@ -21,7 +21,7 @@ #' type, joined with the evaluation data and the input calibration data #' @export get_model_draws_w_data <- function(model_output, - model_type, + model_type = c("ww", "hosp"), draws, forecast_date, scenario, @@ -31,6 +31,7 @@ get_model_draws_w_data <- function(model_output, last_hosp_data_date, ot, forecast_time = 28) { + model_type <- arg_match(model_type) nowcast_time <- as.integer(ymd(forecast_date) - ymd(last_hosp_data_date)) ht <- nowcast_time + forecast_time # Date spine for joining data @@ -172,7 +173,6 @@ get_state_level_quantiles <- function(draws) { return(quantiles) } - #' Get quantiles for site-lab level wastewater #' #' @param ww_draws a dataframe containing all the draws from the model estimated @@ -238,8 +238,9 @@ save_table <- function(data_to_save, output_dir, scenario, forecast_date, - model_type, + model_type = c("ww", "hosp"), location) { + model_type <- arg_match(model_type) if (!is.null(data_to_save)) { full_dir <- file.path( output_dir, diff --git a/wweval/R/sample_model.R b/wweval/R/sample_model.R index 270835c2..d038ac52 100644 --- a/wweval/R/sample_model.R +++ b/wweval/R/sample_model.R @@ -29,7 +29,9 @@ #' @param seed seed of random number generator default = 123 #' #' -#' @return a list containing draws, diagnostics, and summary_diagnostics +#' @return a list containing draws, diagnostics, and summary_diagnostics + +#' flags if the model failed to pass convergence tests. If the model errored, +#' just contains a list with errors #' @export sample_model <- function(standata, @@ -96,27 +98,17 @@ sample_model <- function(standata, seed ) - # If the model doesn't immediately error, get diagnostics - if (is.null(fit$error)) { - # Get the diagnostics using thresholds set in production pipeline - flag_df <- get_diagnostic_flags(fit$result, n_chains, iter_sampling) - any_flags <- any(flag_df) - } - - if (!is.null(fit$error)) { # If the model errors, return a list with the # error and everything else NULL out <- list( error = fit$error[[1]] ) - } else if (any_flags) { # If there are model convergence issues, pass dataframe of flags - out <- list( - flags = flag_df, - summary = fit$result$summary() - ) - message("Model convergence issues") } else { + # Get the diagnostics using thresholds set in production pipeline + flag_df <- get_diagnostic_flags(fit$result, n_chains, iter_sampling) + any_flags <- any(flag_df) + draws <- fit$result$draws() diagnostics <- fit$result$sampler_diagnostics(format = "df") summary_diagnostics <- fit$result$diagnostic_summary() @@ -128,6 +120,12 @@ sample_model <- function(standata, summary_diagnostics = summary_diagnostics, summary = summary ) + + if (any_flags) { # If there are model convergence issues, pass + # flags alongside the draws and summaries + out <- c(out, flags = list(flag_df)) + message("Model convergence issues") + } } return(out) } diff --git a/wweval/R/score.R b/wweval/R/score.R index be8fbd43..b3a01ee2 100644 --- a/wweval/R/score.R +++ b/wweval/R/score.R @@ -9,7 +9,8 @@ #' @param scenario a string indicating the wastewater data scenario we're #' running #' @param metrics Vector of scoring metrics to output, passed as the -#' `metrics` argument to [scoringutils::score()]. Default +#' `metrics` argument to [scoringutils::score()]. Default is NULL, +#' which returns all options for samples including: #' `c("crps", "dss", "bias", "mad", "ae_median", "se_mean")`. #' #' @return a dataframe containing a score for each day in the nowcast @@ -17,10 +18,7 @@ #' @export get_full_scores <- function(draws, scenario, - metrics = c( - "crps", "dss", "bias", - "mad", "ae_median", "se_mean" - )) { + metrics = NULL) { if (is.null(draws)) { scores <- NULL } else { @@ -71,18 +69,16 @@ get_full_scores <- function(draws, #' @param scenario a string indicating the wastewater data scenario we're #' running #' @param metrics Vector of scoring metrics to output, passed as the -#' `metrics` argument to [scoringutils::score()]. Default -#' `c("coverage", "dispersion", "bias", "range")`. +#' `metrics` argument to [scoringutils::score()]. Default is NULL which will +#' include all scoring metrics for quantiles by default, including +#' `c("interval_score", "coverage", "dispersion", "bias")`. #' #' @return a dataframe containing a score for each day in the nowcast #' and forecast period #' @export get_scores_from_quantiles <- function(quantiles, scenario, - metrics = c( - "coverage", "dispersion", "bias", - "range" - )) { + metrics = NULL) { if (is.null(quantiles)) { scores <- NULL } else { @@ -112,7 +108,199 @@ get_scores_from_quantiles <- function(quantiles, scores <- forecasted_quantiles |> scoringutils::score(metrics = metrics) } + return(scores) +} +#' Make baseline score table +#' +#' @description +#' This function makes a wide table with the average score of a pipeline run +#' summarized across all locations. The point of this is to get an approximate +#' estimate of the performance of the pipeline/model across locations when we +#' are iterating on model development. This way, we can use this score as a +#' baseline and aim to add new features to the model only if the overall score +#' of the forecast performance is improved +#' +#' +#' @param all_ww_scores the table of scores for all dates from all locations +#' from the wastewater informed model +#' @param baseline_score_table_dir character string indicating the directory +#' to save the baseline score tables +#' @param overwrite_table boolean indicating whether or not to overwrite the +#' current baseline table, default is FALSE. +#' +#' @return a table containing the summarized outputs from scoring utils for the +#' wastewater model across dates and locations +#' @export +#' +make_baseline_score_table <- function(all_ww_scores, + baseline_score_table_dir, + overwrite_table = FALSE) { + # Get metadata + locations <- all_ww_scores |> + dplyr::pull(location) |> + unique() + forecast_dates <- all_ww_scores |> + dplyr::pull(forecast_date) |> + unique() + scenario <- all_ww_scores |> + dplyr::pull(scenario) |> + unique() + + + # Score forecasts + scores <- scoringutils::summarize_scores(all_ww_scores, + by = c( + "scenario", + "forecast_date" + ) + ) |> + dplyr::mutate( + locations = paste(locations, collapse = ",") + ) + + if (isTRUE(overwrite_table)) { + model_type <- if (scenario == "status_quo") "ww" else "hosp" + # Check that is only one scenario + stopifnot("more than one scenario" = length(unique(scores$scenario)) == 1) + # Check that is only one forecast_date + stopifnot("more than one forecast_date" = length(unique(scores$forecast_date)) == 1) + + cfaforecastrenewalww::create_dir(baseline_score_table_dir) + + write.table(scores, file.path( + baseline_score_table_dir, + glue::glue("baseline_scores_{model_type}.tsv") + )) + } return(scores) } + +#' Score hub submissions +#' +#' @param model_name a vector of character strings indicating the names of the +#' models to scores +#' @param dates a vector of dates indicating the dates of the submissions to score +#' @param locations a vector of character strings indicating the locations +#' to score +#' @param hub_subdir path where the retrospective hub submissions are saved +#' locally since these are not on COVID hub github +#' @param pull_from_github boolean indicating whether or not to pull from github +#' @param submissions_path url pointing to the "data-processed" folder on +#' the COVIDhub github, which is where team's submissions are located +#' @param truth_data_path the path to the truth data used by the hub for +#' evaluation +#' +#' @return a dataframe containing all of the scores for all models, forecast dates +#' (indcated by dates), locations, target end dates, and quantiles +#' @export +#' +score_hub_submissions <- function(model_name, + dates, + locations = NULL, + hub_subdir = NA, + pull_from_github = TRUE, + submissions_path = "https://raw.githubusercontent.com/reichlab/covid19-forecast-hub/master/data-processed/", # nolint + truth_data_path = "https://media.githubusercontent.com/media/reichlab/covid19-forecast-hub/master/data-truth/truth-Incident%20Hospitalizations.csv") { # nolint + + truth_data <- truth_data <- readr::read_csv(truth_data_path) + + natural_scale_scores <- tibble::tibble() + log_scale_scores <- tibble::tibble() + missing_forecasts <- tibble::tibble() + for (i in seq_along(model_name)) { + for (j in seq_along(dates)) { + this_forecast_date <- dates[j] + this_model_name <- model_name[i] + if (isTRUE(pull_from_github)) { + quantiles <- tryCatch( + readr::read_csv(glue::glue( + "{submissions_path}{this_model_name}/{this_forecast_date}-{this_model_name}.csv" + )) |> + dplyr::filter(type == "quantile"), + error = function(e) NULL + ) + } else { + quantiles <- readr::read_csv( + file.path( + hub_subdir, this_model_name, + glue::glue("{this_forecast_date}-{this_model_name}.csv") + ) + ) + } + + if (!is.null(quantiles)) { + quantiles_w_truth <- quantiles |> + dplyr::left_join( + truth_data |> dplyr::rename( + true_value = value + ), + by = c( + "target_end_date" = "date", + "location" + ) + ) |> + dplyr::rename( + prediction = value + ) |> + dplyr::mutate( + model = this_model_name + ) + + # Filter locations if they are specified, otherwise leave them all in + if (!is.null(locations)) { + quantiles_w_truth <- quantiles_w_truth |> + dplyr::filter(location %in% cfaforecastrenewalww::loc_abbr_to_flusight_code(locations)) + } + + # Pass to scoring utils, no summaries just daily, quantiled scores + these_natural_scale_scores <- quantiles_w_truth |> + scoringutils::score(metrics = NULL) |> + dplyr::mutate(horizon_days = as.integer( + lubridate::ymd(target_end_date) - lubridate::ymd(forecast_date) + )) |> + dplyr::mutate( + horizon_weeks = + ceiling(horizon_days / 7) + ) |> + dplyr::mutate(horizon = glue::glue("{horizon_weeks} week ahead")) |> + dplyr::select(-horizon_weeks, -horizon_days) + + these_log_scores <- quantiles_w_truth |> + scoringutils::transform_forecasts( + fun = scoringutils::log_shift, + offset = 1, + append = FALSE + ) |> + scoringutils::score(metrics = NULL) |> + dplyr::mutate(horizon_days = as.integer( + lubridate::ymd(target_end_date) - lubridate::ymd(forecast_date) + )) |> + dplyr::mutate( + horizon_weeks = + ceiling(horizon_days / 7) + ) |> + dplyr::mutate(horizon = glue::glue("{horizon_weeks} week ahead")) |> + dplyr::select(-horizon_weeks, -horizon_days) + + + log_scale_scores <- dplyr::bind_rows(log_scale_scores, these_log_scores) + natural_scale_scores <- dplyr::bind_rows(natural_scale_scores, these_natural_scale_scores) + } else { # end if statement for quantiles empty + these_missing_forecasts <- tibble( + model = this_model_name, + forecast_date = this_forecast_date + ) + missing_forecasts <- dplyr::bind_rows(missing_forecasts, these_missing_forecasts) + } + } # end loop forecast dates + } # end loop model name + + scores_list <- list( + natural_scale_scores = natural_scale_scores, + log_scale_scores = log_scale_scores, + missing_forecasts = missing_forecasts + ) + return(scores_list) +} diff --git a/wweval/R/wweval-package.R b/wweval/R/wweval-package.R index fb56aade..d2258336 100644 --- a/wweval/R/wweval-package.R +++ b/wweval/R/wweval-package.R @@ -6,16 +6,17 @@ #' @importFrom jsonlite fromJSON #' @importFrom readr read_csv write_csv #' @importFrom glue glue -#' @importFrom rlang sym +#' @importFrom rlang sym arg_match #' @importFrom lubridate ymd #' @importFrom yaml read_yaml -#' @importFrom tidybayes spread_draws +#' @importFrom tidybayes spread_draws stat_halfeye stat_slab #' @importFrom dplyr filter left_join select pull distinct mutate as_tibble #' rename ungroup arrange row_number group_by #' @importFrom tidyr pivot_wider pivot_longer #' @importFrom ggplot2 ggplot facet_wrap geom_line geom_hline geom_point geom_bar #' theme scale_y_continuous scale_colour_discrete scale_fill_discrete geom_ribbon #' scale_x_date facet_grid geom_vline labs aes +#' @importFrom colorspace scale_fill_continuous_diverging #' @importFrom cmdstanr cmdstan_model #' @importFrom data.table as.data.table NULL diff --git a/wweval/man/combine_outputs.Rd b/wweval/man/combine_outputs.Rd index 59434bb9..388f6018 100644 --- a/wweval/man/combine_outputs.Rd +++ b/wweval/man/combine_outputs.Rd @@ -5,7 +5,8 @@ \title{Combine outputs} \usage{ combine_outputs( - output_type, + output_type = c("quantiles", "scores", "ww_quantiles", "scores_quantiles", + "hosp_quantiles", "flags", "errors"), scenarios, forecast_dates, locations, @@ -14,8 +15,9 @@ combine_outputs( ) } \arguments{ -\item{output_type}{the type of output that is saved, either \code{"quantiles"} or -\code{"scores"} or \verb{"ww_quantiles}.} +\item{output_type}{the type of output that is saved, one of \code{"quantiles"}, \code{"scores"}, +\code{"ww_quantiles"}, \code{"scores_quantiles"}, \code{"hosp_quantiles"},\code{"errors"}, +or \code{"flags"}.} \item{scenarios}{The vector of character strings of all the scenarios} @@ -33,7 +35,7 @@ forecast_dates, locations, and scenarios } \description{ This function is a helper function specific to the current nested file -stucture specified by the functions that save the quantiles and scores +structure specified by the functions that save the quantiles and scores for each of the model runs to disk. The function takes in the vectors of scenarios, forecast dates, and locations with output model runs, checks if they exist in the current file structure, and if they do diff --git a/wweval/man/create_hub_submissions.Rd b/wweval/man/create_hub_submissions.Rd new file mode 100644 index 00000000..d21d904b --- /dev/null +++ b/wweval/man/create_hub_submissions.Rd @@ -0,0 +1,57 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/create_hub_submissions.R +\name{create_hub_submissions} +\alias{create_hub_submissions} +\title{Create hub submission files} +\usage{ +create_hub_submissions( + hosp_quantiles_ww, + hosp_quantiles_hosp, + forecast_dates, + hub_subdir, + model_name, + scenario = "status_quo", + save_files = TRUE +) +} +\arguments{ +\item{hosp_quantiles_ww}{a dataframe of the probabilistic estimates of +hospital admissions formatted as quantiles, from the wastewater model, +for multiple forecast dates if present for a particular location} + +\item{hosp_quantiles_hosp}{a dataframe of the probabilistic estimates of +hospital admissions formatted as quantiles, from the hospital admissions +model, for multiple forecast dates} + +\item{forecast_dates}{vector of forecast dates that we want to create +mock submissions for} + +\item{hub_subdir}{character string of the outer directory specifying where to +write the submission files to (which will be named by forecast date)} + +\item{model_name}{character string indicating the name of the model. This will +be used to determine both the name of the submission \code{.csv} file and the name of +its enclosing directory, per COVID-19 Forecast Hub formatting.} + +\item{scenario}{string indicating which wastewater availability scenario to use when creating +the hub submission file. Default \code{"status_quo"} (all actually available wastewater data).} + +\item{save_files}{Save the created submission data frame to disk as a \code{.csv} +file? Boolean, default \code{TRUE}.} +} +\value{ +metadata_df a dataframe that has one row per forecast date indicating +thenumber of wastewater submissions in that date and the number of total +locations +} +\description{ +This function takes in the combined set of wastewater and hospital admissions +quantiles by forecast date, date, and location, creates a dataframe +formatted as it would have been for submission to the COVID forecast Hub, +and saves this to disk if specified. It returns the metadata on the number +of locations total and the number of locations using the wastewater model +for each forecast date submission saved. +It assumes the wastewater-informed model output quantiles passed in have already been +filtered to remove ones that we would wish to exclude (for convergence +diagnostics or other reasons) +} diff --git a/wweval/man/create_mock_submission_scores.Rd b/wweval/man/create_mock_submission_scores.Rd index fea52abe..bee26c0e 100644 --- a/wweval/man/create_mock_submission_scores.Rd +++ b/wweval/man/create_mock_submission_scores.Rd @@ -35,5 +35,5 @@ to get an overall score, we can just take the scores directly from the add it to the dataframe, and continue to concatenate until we get a full score dataframe with all combinations of forecast dates, locations, and scenarios, labeled according to scenario. We will still know which ones -are from which model with the `model`` colum +are from which model with the `model`` column } diff --git a/wweval/man/eval_fit_hosp.Rd b/wweval/man/eval_fit_hosp.Rd index 716c20f5..19415905 100644 --- a/wweval/man/eval_fit_hosp.Rd +++ b/wweval/man/eval_fit_hosp.Rd @@ -2,9 +2,9 @@ % Please edit documentation in R/eval_fit.R \name{eval_fit_hosp} \alias{eval_fit_hosp} -\title{Evaluate Hospitalizations Fit} +\title{Fit Hospitalizations Model for Evaluation} \usage{ -eval_fit_hosp(config_index, eval_config_path, params_path, output_dir) +eval_fit_hosp(config_index, eval_config_path, params_path) } \arguments{ \item{config_index}{Index of eval_config to evaluate} @@ -12,9 +12,7 @@ eval_fit_hosp(config_index, eval_config_path, params_path, output_dir) \item{eval_config_path}{Path to eval_config (created with \code{write_eval_config})} \item{params_path}{Path to params.toml} - -\item{output_dir}{Path to directory for storing output} } \description{ -Evaluate Hospitalizations Fit +Fit Hospitalizations Model for Evaluation } diff --git a/wweval/man/eval_fit_ww.Rd b/wweval/man/eval_fit_ww.Rd index c7c1c8a4..87ebc6c5 100644 --- a/wweval/man/eval_fit_ww.Rd +++ b/wweval/man/eval_fit_ww.Rd @@ -2,9 +2,9 @@ % Please edit documentation in R/eval_fit.R \name{eval_fit_ww} \alias{eval_fit_ww} -\title{Evaluate Wastewater Fit} +\title{Fit Wastewater Model for Evaluation} \usage{ -eval_fit_ww(config_index, eval_config_path, params_path, output_dir) +eval_fit_ww(config_index, eval_config_path, params_path) } \arguments{ \item{config_index}{Index of eval_config to evaluate} @@ -12,9 +12,7 @@ eval_fit_ww(config_index, eval_config_path, params_path, output_dir) \item{eval_config_path}{Path to eval_config (created with \code{write_eval_config})} \item{params_path}{Path to params.toml} - -\item{output_dir}{Path to directory for storing output} } \description{ -Evaluate Wastewater Fit +Fit Wastewater Model for Evaluation } diff --git a/wweval/man/eval_post_process_hosp.Rd b/wweval/man/eval_post_process_hosp.Rd new file mode 100644 index 00000000..540d0cd7 --- /dev/null +++ b/wweval/man/eval_post_process_hosp.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eval_post_process.R +\name{eval_post_process_hosp} +\alias{eval_post_process_hosp} +\title{Post Process Hospitalizations Model for Evaluation} +\usage{ +eval_post_process_hosp(config_index, eval_config_path, params_path) +} +\arguments{ +\item{config_index}{Index of eval_config to evaluate} + +\item{eval_config_path}{Path to eval_config (created with \code{write_eval_config})} + +\item{params_path}{Path to params.toml} +} +\description{ +Post Process Hospitalizations Model for Evaluation +} diff --git a/wweval/man/eval_post_process_ww.Rd b/wweval/man/eval_post_process_ww.Rd new file mode 100644 index 00000000..ad43f1ae --- /dev/null +++ b/wweval/man/eval_post_process_ww.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eval_post_process.R +\name{eval_post_process_ww} +\alias{eval_post_process_ww} +\title{Post Process Wastewater Model for Evaluation} +\usage{ +eval_post_process_ww(config_index, eval_config_path, params_path) +} +\arguments{ +\item{config_index}{Index of eval_config to evaluate} + +\item{eval_config_path}{Path to eval_config (created with \code{write_eval_config})} + +\item{params_path}{Path to params.toml} +} +\description{ +Post Process Wastewater Model for Evaluation +} diff --git a/wweval/man/exclude_hosp_outliers.Rd b/wweval/man/exclude_hosp_outliers.Rd new file mode 100644 index 00000000..9fcdb99d --- /dev/null +++ b/wweval/man/exclude_hosp_outliers.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/exclude_hosp_outliers.R +\name{exclude_hosp_outliers} +\alias{exclude_hosp_outliers} +\title{Exclude hospital outliers} +\usage{ +exclude_hosp_outliers( + raw_input_hosp_data, + forecast_date, + table_of_exclusions, + col_name_dates_to_exclude = "dates_to_exclude" +) +} +\arguments{ +\item{raw_input_hosp_data}{A dataframe containing the location, the date of +admission, and the number of hospital admissions, in that location} + +\item{forecast_date}{The forecast date as a character string} + +\item{table_of_exclusions}{A table containing the combinations of +locations, forecast dates, and dates to exclude. These will be treated +as missing by the model} + +\item{col_name_dates_to_exclude}{The name of the column in the table of +exclusiosn that cooresponds to the date of hospital admissions to exclude, +default is \code{dates_to_exclude}} +} +\value{ +A dataframe in the same format as the \code{raw_input_hosp_data} but +with the location-forecast-date-date admissions rows excluded if indicated +in the \code{table_of_exclusions} +} +\description{ +Exclude hospital outliers +} diff --git a/wweval/man/format_for_hub.Rd b/wweval/man/format_for_hub.Rd new file mode 100644 index 00000000..6e2ba00c --- /dev/null +++ b/wweval/man/format_for_hub.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/create_hub_submissions.R +\name{format_for_hub} +\alias{format_for_hub} +\title{Format quantiles for the current COVID Forecast Hub} +\usage{ +format_for_hub( + quantiles, + date_col_name = "date", + value_col_name = "value", + loc_col_name = "location", + forecast_date_col_name = "forecast_date", + quantile_col_name = "quantile" +) +} +\arguments{ +\item{quantiles}{dataframe of quantiles containing columns with the specified +names for date, value, location, forecast date, and quantile} + +\item{date_col_name}{string indicating the column name for date of the +forecasted quantity, default is \code{date}} + +\item{value_col_name}{string indicating the column name for the value of the +prediction at the date, default is \code{value}} + +\item{loc_col_name}{string indicating the column name for the state +abbreviation corresponding to the forecast location, default \code{location}} + +\item{forecast_date_col_name}{string indicating the column name for the forecast +date, default is \code{forecast_date}} + +\item{quantile_col_name}{string indicating the column name for the quantile +value, default is \code{quantile}} +} +\value{ +a dataframe in Hub formatting +} +\description{ +This function takes in a dataframe of quantiles and their +values and the names of the columns specifying the date, location, +value, quantile, and forecast date, and returns a hub formatted dataframe +} diff --git a/wweval/man/get_box_plot.Rd b/wweval/man/get_box_plot.Rd new file mode 100644 index 00000000..dd870584 --- /dev/null +++ b/wweval/man/get_box_plot.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_box_plot} +\alias{get_box_plot} +\title{Get box plot by date and scenario} +\usage{ +get_box_plot( + all_scores, + figure_file_path, + baseline_scenario = "no_wastewater", + save_files = TRUE +) +} +\arguments{ +\item{all_scores}{the full set of scores for each location, forecast date, +scenario, and date} + +\item{figure_file_path}{Outer directory for plots to go in} + +\item{baseline_scenario}{the scenario to compute the difference in +crps scores relative to, default is \code{no_wastewater}} + +\item{save_files}{Whether or not to write to a folder of plots, +default is \code{TRUE}} +} +\value{ +p a ggplot object with a box plot showing the average improvement in +scores by forecast date and scenario for each location +} +\description{ +Get box plot by date and scenario +} diff --git a/wweval/man/get_filepath.Rd b/wweval/man/get_filepath.Rd index b51ebf78..f3eaaa30 100644 --- a/wweval/man/get_filepath.Rd +++ b/wweval/man/get_filepath.Rd @@ -38,5 +38,5 @@ string of the full filepath \description{ A function to return a character string for each item to either be saved or read in, based on the nesting that is specified in this function. To change -the nesting, we will change this funciton +the nesting, we will change this function } diff --git a/wweval/man/get_full_scores.Rd b/wweval/man/get_full_scores.Rd index ef074129..c52a5e30 100644 --- a/wweval/man/get_full_scores.Rd +++ b/wweval/man/get_full_scores.Rd @@ -4,11 +4,7 @@ \alias{get_full_scores} \title{Get the scores for ever day for a particular location and forecast date} \usage{ -get_full_scores( - draws, - scenario, - metrics = c("crps", "dss", "bias", "mad", "ae_median", "se_mean") -) +get_full_scores(draws, scenario, metrics = NULL) } \arguments{ \item{draws}{a dataframe of the model estimated quantity you are evaluating @@ -18,7 +14,8 @@ alongside the evaluation data} running} \item{metrics}{Vector of scoring metrics to output, passed as the -\code{metrics} argument to \code{\link[scoringutils:score]{scoringutils::score()}}. Default +\code{metrics} argument to \code{\link[scoringutils:score]{scoringutils::score()}}. Default is NULL, +which returns all options for samples including: \code{c("crps", "dss", "bias", "mad", "ae_median", "se_mean")}.} } \value{ diff --git a/wweval/man/get_heatmap_relative_wis.Rd b/wweval/man/get_heatmap_relative_wis.Rd new file mode 100644 index 00000000..a719e6a9 --- /dev/null +++ b/wweval/man/get_heatmap_relative_wis.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_heatmap_relative_wis} +\alias{get_heatmap_relative_wis} +\title{Get heatmap of relative WIS} +\usage{ +get_heatmap_relative_wis( + scores, + figure_file_path, + time_period, + baseline_model = "COVIDhub-baseline", + save_files = TRUE +) +} +\arguments{ +\item{scores}{df of granular (daily) score across models, locations, forecast +dates and horizons} + +\item{figure_file_path}{path to save figure} + +\item{time_period}{time period that scores are summarized over} + +\item{baseline_model}{which model to compute relative WIS compared to, default +is \code{COVIDhub-baseline}} + +\item{save_files}{save_files boolean indicating whether or not to save figures, default +is \code{TRUE}} +} +\value{ +a ggplot with a heatmap with model on the x-axis, location on the y-axis +and fill by relative WIS score across forecast dates and horizons +} +\description{ +Get heatmap of relative WIS +} diff --git a/wweval/man/get_hosp_data_sizes.Rd b/wweval/man/get_hosp_data_sizes.Rd index d507182d..7aa8f786 100644 --- a/wweval/man/get_hosp_data_sizes.Rd +++ b/wweval/man/get_hosp_data_sizes.Rd @@ -8,6 +8,7 @@ get_hosp_data_sizes( input_hosp_data, forecast_date, forecast_time, + calibration_time, last_hosp_data_date, uot, hosp_value_col_name = "daily_hosp_admits" @@ -21,6 +22,9 @@ get_hosp_data_sizes( \item{forecast_time}{integer indicating the number of days to make a forecast for} +\item{calibration_time}{integer indicating the max duration in days that +the model is calibrated to hospital admissions for} + \item{last_hosp_data_date}{string indicating the date of the last observed hospital admission} diff --git a/wweval/man/get_hosp_values.Rd b/wweval/man/get_hosp_values.Rd index 610c6a36..3c708316 100644 --- a/wweval/man/get_hosp_values.Rd +++ b/wweval/man/get_hosp_values.Rd @@ -20,7 +20,7 @@ model has available calibration data in days} \item{ht}{integer indicating the number of days to produce hospital admissions outside the calibration period (forecast + nowcast time) in days} -\item{hosp_value_col_name}{A string represeting the name of the column in the +\item{hosp_value_col_name}{A string representing the name of the column in the input_hosp-data that indicates the number of daily hospital admissions, default is \code{daily_hosp_admits}} } diff --git a/wweval/man/get_model_draws_w_data.Rd b/wweval/man/get_model_draws_w_data.Rd index df963c2c..c1a262f3 100644 --- a/wweval/man/get_model_draws_w_data.Rd +++ b/wweval/man/get_model_draws_w_data.Rd @@ -6,7 +6,7 @@ \usage{ get_model_draws_w_data( model_output, - model_type, + model_type = c("ww", "hosp"), draws, forecast_date, scenario, diff --git a/wweval/man/get_model_path.Rd b/wweval/man/get_model_path.Rd index 72a8a6f2..89355b8e 100644 --- a/wweval/man/get_model_path.Rd +++ b/wweval/man/get_model_path.Rd @@ -9,7 +9,7 @@ get_model_path(model_type, stan_models_dir) \arguments{ \item{model_type}{string specifying the model to be run, options are either 'hosp' for the hospital admissions only model or -'ww' for the site-level infection dyanmics model using wastewater} +'ww' for the site-level infection dynamics model using wastewater} \item{stan_models_dir}{directory where stan files are located} } diff --git a/wweval/man/get_n_states_improved_plot.Rd b/wweval/man/get_n_states_improved_plot.Rd new file mode 100644 index 00000000..32c88135 --- /dev/null +++ b/wweval/man/get_n_states_improved_plot.Rd @@ -0,0 +1,39 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_n_states_improved_plot} +\alias{get_n_states_improved_plot} +\title{Get bar chart of the number of states improved compared to without wastewater +over each forecast date} +\usage{ +get_n_states_improved_plot( + all_scores, + figure_file_path, + baseline_scenario = "no_wastewater", + threshold_for_improvement = 1, + save_files = TRUE +) +} +\arguments{ +\item{all_scores}{the full set of scores for each location, forecast date, +scenario, and date} + +\item{figure_file_path}{Outer directory for plots to go in} + +\item{baseline_scenario}{the scenario to compute the difference in +crps scores relative to, default is \code{no_wastewater}} + +\item{threshold_for_improvement}{the relative reduction that we will consider +as an improvement over the baseline,} + +\item{save_files}{Whether or not to write to a folder of plots, +default is \code{TRUE}} +} +\value{ +p a ggplot object with a bar chart showing the number of +jurisdictions with an improvement compared to the forecast without +wastewater under each scenario. +} +\description{ +Get bar chart of the number of states improved compared to without wastewater +over each forecast date +} diff --git a/wweval/man/get_plot_hosp_data_comparison.Rd b/wweval/man/get_plot_hosp_data_comparison.Rd index 5120af2f..a0236239 100644 --- a/wweval/man/get_plot_hosp_data_comparison.Rd +++ b/wweval/man/get_plot_hosp_data_comparison.Rd @@ -7,7 +7,7 @@ get_plot_hosp_data_comparison( draws_w_data, location, - model_type, + model_type = c("ww", "hosp"), n_draws = 100 ) } diff --git a/wweval/man/get_plot_hub_performance.Rd b/wweval/man/get_plot_hub_performance.Rd new file mode 100644 index 00000000..b2c291df --- /dev/null +++ b/wweval/man/get_plot_hub_performance.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_plot_hub_performance} +\alias{get_plot_hub_performance} +\title{Get plot of overall hub performance} +\usage{ +get_plot_hub_performance( + all_scores, + cfa_real_time_scores, + figure_file_path, + all_time_period, + real_time_period, + baseline_model = "COVIDhub-baseline", + save_files = TRUE +) +} +\arguments{ +\item{all_scores}{df with granular (daily) scores from every model, +forecast_date, and location for the entire time period. Includes the +two retrospective models} + +\item{cfa_real_time_scores}{df with granular (daily) scores from the +submitted cfa ww model for the time period when it was submitted.} + +\item{figure_file_path}{path to directory to save figures} + +\item{all_time_period}{string indicating the longer time frame we are +comparing, e.g. "Oct 2023-Mar 2024"} + +\item{real_time_period}{string indicating the shorter time frame that +we submitted our model to the hub e.g. "Feb 2024-Mar 2024"} + +\item{baseline_model}{which model to compute relative WIS compared to, default +is \code{COVIDhub-baseline}} + +\item{save_files}{boolean indicating whether or not to save figures, default +is \code{TRUE}} +} +\value{ +a ggplot object containing distributions of WIS scores grouped by +model and the comaprison time period, with the mean plotted alongside the +full distribution +} +\description{ +Get plot of overall hub performance +} diff --git a/wweval/man/get_plot_quantile_comparison.Rd b/wweval/man/get_plot_quantile_comparison.Rd index 68039c85..f9bcae23 100644 --- a/wweval/man/get_plot_quantile_comparison.Rd +++ b/wweval/man/get_plot_quantile_comparison.Rd @@ -7,7 +7,9 @@ get_plot_quantile_comparison( hosp_quantiles, eval_data, - days_to_show_forecast = 28 + figure_file_path, + days_to_show_forecast = 28, + save_files = TRUE ) } \arguments{ @@ -15,8 +17,13 @@ get_plot_quantile_comparison( \item{eval_data}{Hospital admissions data for visual comparison} +\item{figure_file_path}{Outer directory for plots to go in} + \item{days_to_show_forecast}{Number of days to show the forecast, default vlaue is 28} + +\item{save_files}{Whether or not to write to a folder of plots, +default is \code{TRUE}} } \value{ a ggplot object with forecasts overlaid with evaluation data for diff --git a/wweval/man/get_plot_scores_w_data.Rd b/wweval/man/get_plot_scores_w_data.Rd index 26212390..307f8fe9 100644 --- a/wweval/man/get_plot_scores_w_data.Rd +++ b/wweval/man/get_plot_scores_w_data.Rd @@ -4,7 +4,13 @@ \alias{get_plot_scores_w_data} \title{Get plot scores with evaluation data overlaid} \usage{ -get_plot_scores_w_data(all_scores, eval_data, score_metric = "crps") +get_plot_scores_w_data( + all_scores, + eval_data, + figure_file_path, + score_metric = "crps", + save_files = TRUE +) } \arguments{ \item{all_scores}{a dataframe containing daily scores for each scenario, @@ -13,8 +19,13 @@ and forecast date, for a single location} \item{eval_data}{a dataframe containing the hospital admissions data that the forecasts are evaluated against} +\item{figure_file_path}{Outer directory for plots to go in} + \item{score_metric}{A string indicating the score metric to plot, default is "crps"} + +\item{save_files}{Whether or not to write to a folder of plots, +default is \code{TRUE}} } \value{ a ggplot object plotting the scores summarized by forecast date diff --git a/wweval/man/get_plot_wis_over_time.Rd b/wweval/man/get_plot_wis_over_time.Rd new file mode 100644 index 00000000..d4dcb72c --- /dev/null +++ b/wweval/man/get_plot_wis_over_time.Rd @@ -0,0 +1,37 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_plot_wis_over_time} +\alias{get_plot_wis_over_time} +\title{Get plot of WIS over time} +\usage{ +get_plot_wis_over_time( + all_scores, + cfa_real_time_scores, + figure_file_path, + horizon_time_in_weeks = NULL, + save_files = TRUE +) +} +\arguments{ +\item{all_scores}{Scores from entire time period of interest, including +the retrospective cfa model} + +\item{cfa_real_time_scores}{Real-time scores from Feb - Mar for the cfa ww +model submitted to the hub} + +\item{figure_file_path}{directory to save figures in} + +\item{horizon_time_in_weeks}{horizon time in weeks to summarize over, default +is \code{NULL} which means that the scores are summarized over 4 weeks} + +\item{save_files}{bolean indicating whether or not to save the files, default +is \code{TRUE}} +} +\value{ +a ggplot object of WIS scores over time colored by model, for the +real-time cfa model from Feb - Mar and the retrospective CFA model over +all time points +} +\description{ +Get plot of WIS over time +} diff --git a/wweval/man/get_plot_ww_data_comparison.Rd b/wweval/man/get_plot_ww_data_comparison.Rd index b17ed461..d46336a0 100644 --- a/wweval/man/get_plot_ww_data_comparison.Rd +++ b/wweval/man/get_plot_ww_data_comparison.Rd @@ -17,7 +17,7 @@ the estimated wastewater concentrations in each site joined with both the data the model was calibrated to and the later observed data for evaluating the future predicted concentrations against.} -\item{location}{the jursidiction the data is from} +\item{location}{the jurisdiction the data is from} \item{model_type}{type of model the output is from, default is \code{ww}} diff --git a/wweval/man/get_qq_plot.Rd b/wweval/man/get_qq_plot.Rd new file mode 100644 index 00000000..60a14c16 --- /dev/null +++ b/wweval/man/get_qq_plot.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/plots.R +\name{get_qq_plot} +\alias{get_qq_plot} +\title{Get quantile-quantile plot} +\usage{ +get_qq_plot(scores, figure_file_path, time_period, save_files = TRUE) +} +\arguments{ +\item{scores}{df of granular (daily) score across models, locations, forecast +dates and horizons} + +\item{figure_file_path}{path to save figure} + +\item{time_period}{time period that scores are summarized over} + +\item{save_files}{save_files boolean indicating whether or not to save figures, default +is \code{TRUE}} +} +\value{ +a ggplot object containing a plot of the proportion of data within +each interval for each model. +} +\description{ +Get quantile-quantile plot +} diff --git a/wweval/man/get_scores_from_quantiles.Rd b/wweval/man/get_scores_from_quantiles.Rd index d97754b1..5d4e114e 100644 --- a/wweval/man/get_scores_from_quantiles.Rd +++ b/wweval/man/get_scores_from_quantiles.Rd @@ -5,11 +5,7 @@ \title{Get the scores for every day for a location, forecast date, and scenario from the quantiles during the forecast period} \usage{ -get_scores_from_quantiles( - quantiles, - scenario, - metrics = c("coverage", "dispersion", "bias", "range") -) +get_scores_from_quantiles(quantiles, scenario, metrics = NULL) } \arguments{ \item{quantiles}{a dataframe of the model estimated quantiles alongside @@ -20,8 +16,9 @@ only} running} \item{metrics}{Vector of scoring metrics to output, passed as the -\code{metrics} argument to \code{\link[scoringutils:score]{scoringutils::score()}}. Default -\code{c("coverage", "dispersion", "bias", "range")}.} +\code{metrics} argument to \code{\link[scoringutils:score]{scoringutils::score()}}. Default is NULL which will +include all scoring metrics for quantiles by default, including +\code{c("interval_score", "coverage", "dispersion", "bias")}.} } \value{ a dataframe containing a score for each day in the nowcast diff --git a/wweval/man/get_stan_data_list.Rd b/wweval/man/get_stan_data_list.Rd index d0746dee..8f04ac8f 100644 --- a/wweval/man/get_stan_data_list.Rd +++ b/wweval/man/get_stan_data_list.Rd @@ -8,6 +8,7 @@ get_stan_data_list( model_type, forecast_date, forecast_time, + calibration_time, input_ww_data, input_hosp_data, generation_interval, @@ -31,6 +32,9 @@ Options are \code{ww} or \code{hosp}} \item{forecast_time}{integer indicating the number of days to make a forecast for} +\item{calibration_time}{integer indicating the max duration in days that +the model is calibrated to hospital admissions for} + \item{input_ww_data}{a dataframe with the input wastewater data} \item{input_hosp_data}{a dataframe with the input hospital admissions data} @@ -66,7 +70,7 @@ natural scale, default is \code{ww}} in the input_ww_data that indicates the value of the LOD in natural scale, default is \code{lod_sewage}} -\item{hosp_value_col_name}{A string represeting the name of the column in the +\item{hosp_value_col_name}{A string representing the name of the column in the input_hosp-data that indicates the number of daily hospital admissions, default is \code{daily_hosp_admits}} } diff --git a/wweval/man/get_ww_values.Rd b/wweval/man/get_ww_values.Rd index f8af5716..21eaa4ba 100644 --- a/wweval/man/get_ww_values.Rd +++ b/wweval/man/get_ww_values.Rd @@ -29,7 +29,7 @@ the ww_data that indicates the number of people represented by that wastewater catchment} \item{one_pop_per_site}{a boolean variable indicating if there should only -be on catchment area population per site, default is \code{TRUE} bc this is what +be on catchment area population per site, default is \code{TRUE} because this is what the stan model expects} } \value{ diff --git a/wweval/man/make_baseline_score_table.Rd b/wweval/man/make_baseline_score_table.Rd new file mode 100644 index 00000000..54dcf919 --- /dev/null +++ b/wweval/man/make_baseline_score_table.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/score.R +\name{make_baseline_score_table} +\alias{make_baseline_score_table} +\title{Make baseline score table} +\usage{ +make_baseline_score_table( + all_ww_scores, + baseline_score_table_dir, + overwrite_table = FALSE +) +} +\arguments{ +\item{all_ww_scores}{the table of scores for all dates from all locations +from the wastewater informed model} + +\item{baseline_score_table_dir}{character string indicating the directory +to save the baseline score tables} + +\item{overwrite_table}{boolean indicating whether or not to overwrite the +current baseline table, default is FALSE.} +} +\value{ +a table containing the summarized outputs from scoring utils for the +wastewater model across dates and locations +} +\description{ +This function makes a wide table with the average score of a pipeline run +summarized across all locations. The point of this is to get an approximate +estimate of the performance of the pipeline/model across locations when we +are iterating on model development. This way, we can use this score as a +baseline and aim to add new features to the model only if the overall score +of the forecast performance is improved +} diff --git a/wweval/man/sample_model.Rd b/wweval/man/sample_model.Rd index f106ee5e..56c53b0c 100644 --- a/wweval/man/sample_model.Rd +++ b/wweval/man/sample_model.Rd @@ -49,7 +49,9 @@ default = 250} \item{seed}{seed of random number generator default = 123} } \value{ -a list containing draws, diagnostics, and summary_diagnostics +a list containing draws, diagnostics, and summary_diagnostics + +flags if the model failed to pass convergence tests. If the model errored, +just contains a list with errors } \description{ This code was adapted from code written diff --git a/wweval/man/save_table.Rd b/wweval/man/save_table.Rd index 42b63888..d52962a5 100644 --- a/wweval/man/save_table.Rd +++ b/wweval/man/save_table.Rd @@ -10,7 +10,7 @@ save_table( output_dir, scenario, forecast_date, - model_type, + model_type = c("ww", "hosp"), location ) } diff --git a/wweval/man/score_hub_submissions.Rd b/wweval/man/score_hub_submissions.Rd new file mode 100644 index 00000000..335ed100 --- /dev/null +++ b/wweval/man/score_hub_submissions.Rd @@ -0,0 +1,45 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/score.R +\name{score_hub_submissions} +\alias{score_hub_submissions} +\title{Score hub submissions} +\usage{ +score_hub_submissions( + model_name, + dates, + locations = NULL, + hub_subdir = NA, + pull_from_github = TRUE, + submissions_path = + "https://raw.githubusercontent.com/reichlab/covid19-forecast-hub/master/data-processed/", + truth_data_path = + "https://media.githubusercontent.com/media/reichlab/covid19-forecast-hub/master/data-truth/truth-Incident\%20Hospitalizations.csv" +) +} +\arguments{ +\item{model_name}{a vector of character strings indicating the names of the +models to scores} + +\item{dates}{a vector of dates indicating the dates of the submissions to score} + +\item{locations}{a vector of character strings indicating the locations +to score} + +\item{hub_subdir}{path where the retrospective hub submissions are saved +locally since these are not on COVID hub github} + +\item{pull_from_github}{boolean indicating whether or not to pull from github} + +\item{submissions_path}{url pointing to the "data-processed" folder on +the COVIDhub github, which is where team's submissions are located} + +\item{truth_data_path}{the path to the truth data used by the hub for +evaluation} +} +\value{ +a dataframe containing all of the scores for all models, forecast dates +(indcated by dates), locations, target end dates, and quantiles +} +\description{ +Score hub submissions +}