Retrieve data for the residential sector

.env.template

@@ -1,24 +1,8 @@
-# This template comes from https://www.github.omc/kmax12/gridstatus
-
 # Register at https://www.eia.gov/opendata/register.php for an API key
 EIA_API_KEY=
 
-# Register at https://apiportal.pjm.com/ for an API key
-PJM_API_KEY=
-
-# Register at https://apiexplorer.ercot.com/ for username/password
-# and follow instructions at
-# https://developer.ercot.com/applications/pubapi/ERCOT%20Public%20API%20Registration%20and%20Authentication/
-# to get the subscription key
-ERCOT_API_USERNAME=
-ERCOT_API_PASSWORD=
-ERCOT_API_SUBSCRIPTION_KEY=
-
-# Request access at https://www.ncdc.noaa.gov/cdo-web/token
-NOAA_API_KEY=
-
 # Request access at https://api.census.gov/data/key_signup.html
-CENCUS_API_KEY=
+CENSUS_API_KEY=
 
 # Request access at https://www.epa.gov/power-sector/cam-api-portal#/api-key-signup 
 CEMS_API_KEY=

.gitignore

@@ -1,5 +1,9 @@
 cjest-data
-
+.snakemake/
+data
+01-energy-utility.ipynb
+02-census.ipynb
+puma_maps.ipynb
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,57 @@
 # 2024 Kansas City Analysis
-This repository holds analysis for the energy system in Kansas City, Kansas. Located in Wyandotte County, Kansas.
+This repository holds analysis for the energy system in Kansas City, Kansas. Located in Wyandotte County, Kansas.
+
+
+# Installation
+
+## Requirements
+
+* `git` - version control software
+    * [Windows Installation instructions](https://git-scm.com/download/win)
+    * [MacOS Installation instructions](https://git-scm.com/download/mac)
+    * [Linux Installation instructions](https://git-scm.com/download/linux)
+* Python installed with either `conda` or `mamba`(recommended)
+    * Download `mamba` installer [here](https://github.com/conda-forge/miniforge).
+    * 'anaconda' ('conda') installation instructions [here](https://docs.anaconda.com/anaconda/install/windows/).
+
+> [!NOTE]
+> Make sure you add Python to PATH during installation.
+
+## Installation Steps
+0. Open command prompt or terminal window. Copy and paste the following commands.
+
+1. Clone the repository
+
+```bash
+git clone https://github.com/ucsusa/2024-kansas-city-analysis.git
+```
+
+2. Set up the environment
+
+```bash
+cd 2024-kansas-city-analysis
+mamba env create  # mamba and conda may be used interchangeably, here
+mamba activate kansas-city
+```
+
+3. Creating the `.env` file
+
+Users should copy the `.env.template` file into a new file simply called `.env`.
+This file contains "secret" information, such as API keys, emails, and other data
+that should remain local. In order to run the current model, users must have API keys
+from the following organizations:
+
+* [U.S. Census API](https://api.census.gov/data/key_signup.html)
+
+These keys may be added directly to the `.env` file.    
+
+## Running the model 
+
+This project uses the workflow management tool, `snakemake`, to create a reproducible data pipeline.
+Running the command
+
+```bash
+snakemake --cores=1
+```
+
+will run the workflow illustrated in the directed acyclic graph (DAG) shown below.

Snakefile

@@ -0,0 +1,77 @@
+configfile: "config.yml"
+
+from pathlib import Path
+env_file = Path("./.env").resolve()
+from dotenv import load_dotenv
+load_dotenv(str(env_file))
+
+rule targets:
+    input: 
+        sfa = "data/timeseries/residential_load.csv",
+        res_structures = "data/residential_buildings.csv",
+        rates = "data/usrdb_rates.csv",
+        project_sunroof = f"data/spatial_data/project-sunroof-census_tract.csv",
+        dag = "dag.png"
+
+rule retrieve_spatial_lut:
+    output: 
+        spatial_lut = "data/spatial_data/spatial_lut.csv"
+    script: "scripts/retrieve_lut.py"
+
+rule retrieve_census_data:
+    output:
+        census_data = "data/spatial_data/county_census_data.gpkg",
+        state_blockgroups = f"data/spatial_data/{config['state'].lower()}_blockgroups.gpkg",
+        county_blockgroups = f"data/spatial_data/{config['county'].lower()}_blockgroups.gpkg"
+    script: "scripts/retrieve_census_data.py"
+
+rule retrieve_project_sunroof:
+    input: 
+        blockgroups = f"data/spatial_data/{config['state'].lower()}_blockgroups.gpkg"
+    output: 
+        project_sunroof = "data/spatial_data/project-sunroof-census_tract.csv",
+        local_potential = f"data/spatial_data/{config['state'].lower()}_rooftop_potential.gpkg"
+    script: "scripts/retrieve_project_sunroof.py"
+
+# a bespoke step to make this analysis specific to armourdale
+rule retrieve_armourdale_shape:
+    output: 
+        armourdale = "data/spatial_data/armourdale_shape.gpkg"
+    script: "scripts/retrieve_armourdale.py"
+
+rule retrieve_electric_utility:
+    input: 
+      cutout="data/spatial_data/armourdale_shape.gpkg"
+    output:
+      utility="data/spatial_data/electric_utility.gpkg"
+    script: "scripts/retrieve_electric_utility.py"
+
+rule retrieve_usrdb:
+    input: 
+        utility="data/spatial_data/electric_utility.gpkg"
+    output: 
+        rates="data/usrdb_rates.csv"
+    script: "scripts/retrieve_usrdb.py"
+
+rule calculate_res_structures:
+    input: 
+        census_data = "data/spatial_data/county_census_data.gpkg",
+        armourdale = "data/spatial_data/armourdale_shape.gpkg"
+    output: 
+        res_structures = "data/residential_buildings.csv"
+    script: "scripts/calculate_res_structures.py"
+
+rule retrieve_res_load:
+    input:
+        spatial_lut = "data/spatial_data/spatial_lut.csv",
+        res_structures = "data/residential_buildings.csv"
+    output: 
+        sfa = "data/timeseries/residential_load.csv"
+    script: "scripts/retrieve_res_load.py"
+
+rule build_dag:
+    input: "Snakefile"
+    output:
+        "dag.png"
+    shell:
+        "snakemake --dag | dot -Tpng > {output}"

config.yml

@@ -0,0 +1,33 @@
+# geographic data
+state: 'Kansas'
+county: 'Wyandotte'
+
+# historical data
+census_year: 2020
+census_level: 'tract'
+usrdb_start_date: "2024-07-23"  # today?
+usrdb_future_date: "2099-01-01"  # some date in the future, replaces NaT values
+
+# building data options
+building_data_options:
+  resstock_year: 2021  # DO NOT CHANGE
+  comstock_year: 2021  # DO NOT CHANGE
+  weather_version: "tmy3"  # or "amy2018"
+  release_version: 1
+  building_types:
+    residential:
+      - multi-family_with_2_-_4_units
+      - multi-family_with_5plus_units
+      - single-family_attached
+      - single-family_detached
+      - mobile_home
+
+energy_sectors:
+  - residential
+  # - commercial  # pending implementation
+  # - industrial  # pending data availability
+
+
+# geographic options
+geographic_crs: 4326  # for using lat/lon; EPSG code
+projected_crs: 5070  # for doing calculations; EPSG code