CR.SurfObs.DataIngest.Precip.xray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "####################################################################################################\n",
    "# CA.SurfObs.DataIngest.Precip.xray.ipynb\n",
    "# Karl Lapo August/2015\n",
    "####################################################################################################\n",
    "# Extract CIMIS precip data\n",
    "####################################################################################################\n",
    "\n",
    "## Import statements\n",
    "import numpy as np\n",
    "import xray\n",
    "import pandas as pd\n",
    "from netCDF4 import Dataset\n",
    "from netCDF4 import num2date, date2num\n",
    "from datetime import datetime, timedelta\n",
    "import pytz\n",
    "\n",
    "# OS interaction\n",
    "import sys\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Directory Lists\n",
    "# General directories\n",
    "dir_data_out = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/data/GroundObs'\n",
    "dir_print = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/Graphics'\n",
    "\n",
    "# Individual data directories\n",
    "dir_cv_cimis_ucipm = '/Users/karllapo/gdrive/GroundObs/CA_CentralValley.UCDavis/RAW/CIMIS/UCDavis_IPM.Daily'\n",
    "dir_cv_cimis = '/Users/karllapo/gdrive/GroundObs/CA_CentralValley.UCDavis/RAW/CIMIS/CIMIS.Hourly'\n",
    "dir_sio_cdwr = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/data/GroundObs/YOS.SIO.Obs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed Site: Alpaugh\n",
      "Processed Site: ALTURAS\n",
      "Processed Site: ARVIN\n",
      "Processed Site: ARYOSECO\n",
      "Processed Site: ATASCADERO\n",
      "Processed Site: Auburn\n",
      "Processed Site: BENNETT_VALLEY\n",
      "Processed Site: Big_Bear_Lake\n",
      "Processed Site: BISHOP\n",
      "Processed Site: Black_Point\n",
      "Processed Site: BLACKWLL\n",
      "Processed Site: BLYTHE_NE\n",
      "Processed Site: Borrego_Springs\n",
      "Processed Site: BRNTWOOD\n",
      "Processed Site: BRWNSVLY\n",
      "Processed Site: BRYTE\n",
      "Processed Site: BUNTNGVL\n",
      "Processed Site: CAMARILLO\n",
      "Processed Site: CAMINO\n",
      "Processed Site: Carmel\n",
      "Processed Site: CASTROVL\n",
      "Processed Site: Coalinga\n",
      "Processed Site: COLUSA\n",
      "Processed Site: CONCORD\n",
      "Processed Site: CUYAMA\n",
      "Processed Site: DAVIS\n",
      "Processed Site: Delano\n",
      "Processed Site: Denair_II\n",
      "Processed Site: Diamond_Springs\n",
      "Processed Site: DIXON\n",
      "Processed Site: DURHAM\n",
      "Processed Site: Esparto\n",
      "Processed Site: FAIR_OAKS\n",
      "Processed Site: FAMOSO\n",
      "Processed Site: FIREBAGH\n",
      "Processed Site: Five_Points_SW\n",
      "Processed Site: FIVE_PTS\n",
      "Processed Site: FRESNO\n",
      "Processed Site: GERBER\n",
      "Processed Site: Gilroy\n",
      "Processed Site: Hastings_Tract_East\n",
      "Processed Site: HOLLISTR\n",
      "Processed Site: HOPLAND\n",
      "Processed Site: HOPLAND2\n",
      "Processed Site: KESTERSN\n",
      "Processed Site: KETTLMAN\n",
      "Processed Site: KINGCTY2\n",
      "Processed Site: Laguna_Seca\n",
      "Processed Site: LINDCOVE\n",
      "Processed Site: LODI_WEST\n",
      "Processed Site: LOSBANOS\n",
      "Processed Site: LOST_HILLS\n",
      "Processed Site: MANTECA\n",
      "Processed Site: MCARTHUR\n",
      "Processed Site: MERCED\n",
      "Processed Site: MODESTO\n",
      "Processed Site: MORAGA\n",
      "Processed Site: NAPA\n",
      "Processed Site: Nipomo\n",
      "Processed Site: NSALINAS\n",
      "Processed Site: Oakdale\n",
      "Processed Site: OAKVILLE\n",
      "Processed Site: ORANGE_COVE\n",
      "Processed Site: Owens_Lake_North\n",
      "Processed Site: Owens_Lake_South\n",
      "Processed Site: Pacific_Grove\n",
      "Processed Site: PAJARO\n",
      "Processed Site: PANOCHE\n",
      "Processed Site: PARLIER\n",
      "Processed Site: PATTERSON\n",
      "Processed Site: PETALUMA_EAST\n",
      "Processed Site: Pleasanton\n",
      "Processed Site: Plymouth_II\n",
      "Processed Site: Point_San_Pedro\n",
      "Processed Site: PORTERVILLE\n",
      "Processed Site: SAN_LUIS_OBISPO_W\n",
      "Processed Site: Santa_Maria_II\n",
      "Processed Site: Shasta_College\n",
      "Processed Site: SISQUOC\n",
      "Processed Site: SJ_VALLEY\n",
      "Processed Site: SNLUIS_O\n",
      "Processed Site: SNTACRUZ\n",
      "Processed Site: SNTAROSA\n",
      "Processed Site: STRATFRD\n",
      "Processed Site: TRACY\n",
      "Processed Site: TRNQULTY\n",
      "Processed Site: TWITCHELL_ISLAND\n",
      "Processed Site: UNION_CITY\n",
      "Processed Site: Verona\n",
      "Processed Site: VICTRVIL\n",
      "Processed Site: Watsonville_West-2\n",
      "Processed Site: WINDSOR\n",
      "Processed Site: WINTERS\n",
      "Processed Site: Woodland\n"
     ]
    }
   ],
   "source": [
    "########################\n",
    "## Ground Obs - CIMIS ##\n",
    "########################\n",
    "# Empty lists and dictionary literals\n",
    "stations = []\n",
    "lat = []\n",
    "lon = []\n",
    "elev = []\n",
    "network = []\n",
    "grobs = {}\n",
    "\n",
    "# time zone variables\n",
    "tz_pst = pytz.timezone('US/Pacific')\n",
    "\n",
    "###########################\n",
    "##### READ CIMIS DATA #####\n",
    "###########################\n",
    "# Read supporting station information\n",
    "os.chdir(dir_data_out)\n",
    "stdat = pd.read_csv('All_StationSummary.v2.csv',sep= ',', \\\n",
    "                index_col=0,na_values=[-9999,'NaN']) # Read the supporting information\n",
    "stdat = stdat.groupby('Network').get_group('CIMIS')\n",
    "\n",
    "# Files to read\n",
    "os.chdir(dir_cv_cimis_ucipm)\n",
    "content = os.listdir(os.getcwd())\n",
    "num_files = len([name for name in os.listdir('.') if os.path.isfile(name)])\n",
    "\n",
    "for files in content:\n",
    "    # Only read .txt files\n",
    "    if files[-4:] == '.txt':\n",
    "        with open(files, 'r') as datafile:\n",
    "            # Skip the header of arbitrary size and read the column names\n",
    "            line = datafile.readline()\n",
    "            while not line.startswith('\"Station\"'):\n",
    "                line = datafile.readline()\n",
    "\n",
    "            ## format the header line for passing to 'read_csv'\n",
    "            line = line.replace('\\n','')\n",
    "            line = line.replace('\"', '')\n",
    "            col_names = line.split(',')\n",
    "            data = pd.read_csv(datafile, names=col_names, sep= ',', parse_dates={'Datetime' : [1,2]},\\\n",
    "                               index_col='Datetime',skipinitialspace=True,converters={'Time': lambda x: str('2359')})\n",
    "            data.index = data.index.tz_localize(pytz.timezone('US/Pacific'))\n",
    "\n",
    "            ## Read SW data, asign to PST, and get SW that passes QC\n",
    "            sitename = data['Station'][0][0:-2]\n",
    "            grobs[sitename] = data['Precip']\n",
    "            grobs[sitename].index = data.index\n",
    "            grobs[sitename] = pd.DataFrame(grobs[sitename])\n",
    "            grobs[sitename].columns =['Precip']\n",
    "\n",
    "            ## Fill in elevation/lat/lon\n",
    "            if sitename in stdat.index:\n",
    "                print((\"Processed Site: \"+sitename))\n",
    "                elev.append(stdat.loc[sitename]['elevation (m)'])\n",
    "                lat.append(stdat.loc[sitename]['lat'])\n",
    "                lon.append(stdat.loc[sitename]['lon'])\n",
    "                stations.append(sitename)\n",
    "                network.append('CIMIS_IPM')\n",
    "            else:\n",
    "                print((\"Site: \"+sitename+\" is missing from master list\"))\n",
    "\n",
    "########################\n",
    "##### COMBINE DATA #####\n",
    "########################\n",
    "# Concatenate \n",
    "grobs_daily = pd.concat(grobs,axis=0,keys=stations)\n",
    "grobs_daily = pd.DataFrame(grobs_daily)\n",
    "\n",
    "# Convert to xray\n",
    "ds = xray.Dataset.from_dataframe(grobs_daily)\n",
    "ds = ds.rename({'level_0':'station','Datetime':'time'})\n",
    "\n",
    "# Fill in descriptive variables\n",
    "ds.coords['lat'] = ('station',lat)\n",
    "ds.coords['lon'] = ('station',lon)\n",
    "ds.coords['elev'] = ('station',elev)\n",
    "ds.coords['network'] = ('station',network)\n",
    "\n",
    "## Output to netcdf\n",
    "os.chdir(dir_data_out)\n",
    "ds.to_netcdf('CA.CIMIS.Precip.daily.xray.nc')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}