CR.SurfObs.DataIngest.xray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Networks for import -- CIMIS and SIO/CDWR\n",
    "flag_xray_proc = 0\n",
    "\n",
    "## Import statements\n",
    "import numpy as np\n",
    "import xray\n",
    "import pandas as pd\n",
    "from netCDF4 import Dataset\n",
    "from netCDF4 import num2date, date2num\n",
    "from datetime import datetime, timedelta\n",
    "import pytz\n",
    "\n",
    "# OS interaction\n",
    "import sys\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Directory Lists\n",
    "# General directories\n",
    "dir_data_out = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/data/GroundObs'\n",
    "dir_print = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/Graphics'\n",
    "\n",
    "# Individual data directories\n",
    "dir_cv_cimis_ucipm = '/Users/karllapo/gdrive/GroundObs/CA_CentralValley.UCDavis/RAW/CIMIS/UCDavis_IPM.Daily'\n",
    "dir_cv_cimis = '/Users/karllapo/gdrive/GroundObs/CA_CentralValley.UCDavis/RAW/CIMIS/CIMIS.Hourly'\n",
    "dir_sio_cdwr = '/Users/karllapo/gdrive/SnowHydrology/proj/CloudClimatology/data/GroundObs/YOS.SIO.Obs'\n",
    "# dir_cv_pestcast = '/Users/karllapo/gdrive/GroundObs/CA_CentralValley.UCDavis/RAW/PestCast'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed Site: Alpaugh\n",
      "Processed Site: ALTURAS\n",
      "Processed Site: ARVIN\n",
      "Processed Site: ARYOSECO\n",
      "Processed Site: ATASCADERO\n",
      "Processed Site: Auburn\n",
      "Processed Site: BENNETT_VALLEY\n",
      "Processed Site: Big_Bear_Lake\n",
      "Processed Site: BISHOP\n",
      "Processed Site: Black_Point\n",
      "Processed Site: BLACKWLL\n",
      "Processed Site: BLYTHE_NE\n",
      "Processed Site: Borrego_Springs\n",
      "Processed Site: BRNTWOOD\n",
      "Processed Site: BRWNSVLY\n",
      "Processed Site: BRYTE\n",
      "Processed Site: BUNTNGVL\n",
      "Processed Site: CAMARILLO\n",
      "Processed Site: CAMINO\n",
      "Processed Site: Carmel\n",
      "Processed Site: CASTROVL\n",
      "Processed Site: Coalinga\n",
      "Processed Site: COLUSA\n",
      "Processed Site: CONCORD\n",
      "Processed Site: CUYAMA\n",
      "Processed Site: DAVIS\n",
      "Processed Site: Delano\n",
      "Processed Site: Denair_II\n",
      "Processed Site: Diamond_Springs\n",
      "Processed Site: DIXON\n",
      "Processed Site: DURHAM\n",
      "Processed Site: Esparto\n",
      "Processed Site: FAIR_OAKS\n",
      "Processed Site: FAMOSO\n",
      "Processed Site: FIREBAGH\n",
      "Processed Site: Five_Points_SW\n",
      "Processed Site: FIVE_PTS\n",
      "Processed Site: FRESNO\n",
      "Processed Site: GERBER\n",
      "Processed Site: Gilroy\n",
      "Processed Site: Hastings_Tract_East\n",
      "Processed Site: HOLLISTR\n",
      "Processed Site: HOPLAND\n",
      "Processed Site: HOPLAND2\n",
      "Processed Site: KESTERSN\n",
      "Processed Site: KETTLMAN\n",
      "Processed Site: KINGCTY2\n",
      "Processed Site: Laguna_Seca\n",
      "Processed Site: LINDCOVE\n",
      "Processed Site: LODI_WEST\n",
      "Processed Site: LOSBANOS\n",
      "Processed Site: LOST_HILLS\n",
      "Processed Site: MANTECA\n",
      "Processed Site: MCARTHUR\n",
      "Processed Site: MERCED\n",
      "Processed Site: MODESTO\n",
      "Processed Site: MORAGA\n",
      "Processed Site: NAPA\n",
      "Processed Site: Nipomo\n",
      "Processed Site: NSALINAS\n",
      "Processed Site: Oakdale\n",
      "Processed Site: OAKVILLE\n",
      "Processed Site: ORANGE_COVE\n",
      "Processed Site: Owens_Lake_North\n",
      "Processed Site: Owens_Lake_South\n",
      "Processed Site: Pacific_Grove\n",
      "Processed Site: PAJARO\n",
      "Processed Site: PANOCHE\n",
      "Processed Site: PARLIER\n",
      "Processed Site: PATTERSON\n",
      "Processed Site: PETALUMA_EAST\n",
      "Processed Site: Pleasanton\n",
      "Processed Site: Plymouth_II\n",
      "Processed Site: Point_San_Pedro\n",
      "Processed Site: PORTERVILLE\n",
      "Processed Site: SAN_LUIS_OBISPO_W\n",
      "Processed Site: Santa_Maria_II\n",
      "Processed Site: Shasta_College\n",
      "Processed Site: SISQUOC\n",
      "Processed Site: SJ_VALLEY\n",
      "Processed Site: SNLUIS_O\n",
      "Processed Site: SNTACRUZ\n",
      "Processed Site: SNTAROSA\n",
      "Processed Site: STRATFRD\n",
      "Processed Site: TRACY\n",
      "Processed Site: TRNQULTY\n",
      "Processed Site: TWITCHELL_ISLAND\n",
      "Processed Site: UNION_CITY\n",
      "Processed Site: Verona\n",
      "Processed Site: VICTRVIL\n",
      "Processed Site: Watsonville_West-2\n",
      "Processed Site: WINDSOR\n",
      "Processed Site: WINTERS\n",
      "Processed Site: Woodland\n",
      "Processed Site: bee\n",
      "Processed Site: dan\n",
      "Processed Site: dep\n",
      "Processed Site: for\n",
      "Processed Site: fsl\n",
      "Processed Site: gin\n",
      "Processed Site: gld\n",
      "Processed Site: hod\n",
      "Processed Site: lee\n",
      "Processed Site: mer\n",
      "Processed Site: olm\n",
      "Processed Site: oth\n",
      "Processed Site: piu\n",
      "Processed Site: pri\n",
      "Processed Site: smi\n",
      "Processed Site: smj\n",
      "Processed Site: stb\n",
      "Processed Site: sun\n",
      "Processed Site: vvw\n"
     ]
    }
   ],
   "source": [
    "###########################################\n",
    "## Ground Obs - CIMIS, UC IPM & SIO-CDWR ##\n",
    "###########################################\n",
    "if flag_xray_proc == 1:\n",
    "    # Load previously formatted data (in xray format)\n",
    "#     os.chdir(dir_data_out)\n",
    "#     with xray.open_dataset('cimis_ucipm.obs.daily.xray.nc') as cimis_ucipm_daily:\n",
    "#         print(cimis_ucipm_daily.keys())\n",
    "    print('to be updated!')\n",
    "elif flag_xray_proc == 0:\n",
    "    \n",
    "    # Empty lists and dictionary literals\n",
    "    stations = []\n",
    "    lat = []\n",
    "    lon = []\n",
    "    elev = []\n",
    "    network = []\n",
    "    grobs = {}\n",
    "    \n",
    "    # time zone variables\n",
    "    tz_pst = pytz.timezone('US/Pacific')\n",
    "\n",
    "    ###########################\n",
    "    ##### READ CIMIS DATA #####\n",
    "    ###########################\n",
    "    # Read supporting station information\n",
    "    os.chdir(dir_data_out)\n",
    "    stdat = pd.read_csv('All_StationSummary.v2.csv',sep= ',', \\\n",
    "                    index_col=0,na_values=[-9999,'NaN']) # Read the supporting information\n",
    "    stdat = stdat.groupby('Network').get_group('CIMIS')\n",
    "    \n",
    "    # Files to read\n",
    "    os.chdir(dir_cv_cimis_ucipm)\n",
    "    content = os.listdir(os.getcwd())\n",
    "    num_files = len([name for name in os.listdir('.') if os.path.isfile(name)])\n",
    "        \n",
    "    for files in content:\n",
    "        # Only read .txt files\n",
    "        if files[-4:] == '.txt':\n",
    "            with open(files, 'r') as datafile:\n",
    "                # Skip the header of arbitrary size and read the column names\n",
    "                line = datafile.readline()\n",
    "                while not line.startswith('\"Station\"'):\n",
    "                    line = datafile.readline()\n",
    "                \n",
    "                ## format the header line for passing to 'read_csv'\n",
    "                line = line.replace('\\n','')\n",
    "                line = line.replace('\"', '')\n",
    "                col_names = line.split(',')\n",
    "                data = pd.read_csv(datafile, names=col_names, sep= ',', parse_dates={'Datetime' : [1,2]},\\\n",
    "                                   index_col='Datetime',skipinitialspace=True,\\\n",
    "                                   converters={'Time': lambda x: str('2359')})\n",
    "                data.index = data.index.tz_localize(pytz.timezone('US/Pacific'))\n",
    "                \n",
    "                ## Read SW data, asign to PST, and get SW that passes QC\n",
    "                sitename = data['Station'][0][0:-2]\n",
    "                grobs[sitename] = data['Solar']\n",
    "                grobs[sitename].index = data.index\n",
    "                grobs[sitename] = pd.DataFrame(grobs[sitename])\n",
    "                grobs[sitename].columns =['SWdwn']\n",
    "    \n",
    "                ## Fill in elevation/lat/lon\n",
    "                if sitename in stdat.index:\n",
    "                    print((\"Processed Site: \"+sitename))\n",
    "                    elev.append(stdat.loc[sitename]['elevation (m)'])\n",
    "                    lat.append(stdat.loc[sitename]['lat'])\n",
    "                    lon.append(stdat.loc[sitename]['lon'])\n",
    "                    stations.append(sitename)\n",
    "                    network.append('CIMIS_IPM')\n",
    "                else:\n",
    "                    print((\"Site: \"+sitename+\" is missing from master list\"))\n",
    "                    \n",
    "    #########################\n",
    "    ##### READ SIO DATA #####\n",
    "    #########################\n",
    "    os.chdir(dir_sio_cdwr)\n",
    "    content = os.listdir(os.getcwd())\n",
    "    num_files = len([name for name in os.listdir('.') if os.path.isfile(name)])\n",
    "    \n",
    "     # Read supporting station information\n",
    "    os.chdir(dir_data_out)\n",
    "    stdat = pd.read_csv('All_StationSummary.v2.csv',sep= ',', \\\n",
    "                    index_col=0,na_values=[-9999,'NaN']) # Read the supporting information\n",
    "    stdat = stdat.groupby('Network').get_group('CDWR')\n",
    "   \n",
    "    for files in content:\n",
    "        # Only read QC formatted files\n",
    "        if files[-10:] == 'Rad.QC.txt':\n",
    "            sitename = files.split('.')[0]\n",
    "            stations.append(sitename)\n",
    "            na_value = ['   NaN']\n",
    "            \n",
    "            # Read SW data, asign to PST, and get SW that passes QC\n",
    "            grobs_yos = pd.read_csv(files,sep= '\\t', parse_dates=True, index_col=0, na_values=na_value)\n",
    "            grobs_yos.index = grobs_yos.index.tz_localize(pytz.utc).tz_convert(tz_pst)\n",
    "            grobs_yos['SWdwn_QC'] = grobs_yos['SWdwn_Wm^-2'].where(grobs_yos['QCFlag'] == 0)\n",
    "        \n",
    "            # New data frame w/ daily means\n",
    "            grobs_yos_daily = grobs_yos['SWdwn_Wm^-2'].resample('D', how='mean').to_frame(name='SWdwn_D')\n",
    "            grobs_yos_daily['SWdwn_D_QC'] = grobs_yos['SWdwn_QC'].resample('D', how='mean')\n",
    "            grobs_yos_daily['SWdwn_D_proc'] = grobs_yos['SWdwn_proc'].resample('D', how='mean')\n",
    "            \n",
    "            # List containing DataFrames with daily, processed only\n",
    "            grobs[sitename] = pd.DataFrame(grobs_yos_daily['SWdwn_D_proc'])\n",
    "            grobs[sitename].columns = ['SWdwn']\n",
    "            \n",
    "            # Fill in elevation/lat/lon/network\n",
    "            elev.append(stdat.loc[sitename]['elevation (m)'])\n",
    "            lat.append(stdat.loc[sitename]['lat'])\n",
    "            lon.append(stdat.loc[sitename]['lon'])\n",
    "            network.append('CDWR')\n",
    "            \n",
    "            print((\"Processed Site: \"+sitename))\n",
    "            \n",
    "    ########################\n",
    "    ##### COMBINE DATA #####\n",
    "    ########################\n",
    "    # Concatenate \n",
    "    grobs_daily = pd.concat(grobs,axis=0,keys=stations)\n",
    "    grobs_daily = pd.DataFrame(grobs_daily)\n",
    "\n",
    "    # Convert to xray\n",
    "    ds = xray.Dataset.from_dataframe(grobs_daily)\n",
    "    ds = ds.rename({'level_0':'station','level_1':'time'})\n",
    "    \n",
    "    # Fill in descriptive variables\n",
    "    ds.coords['lat'] = ('station',lat)\n",
    "    ds.coords['lon'] = ('station',lon)\n",
    "    ds.coords['elev'] = ('station',elev)\n",
    "    ds.coords['network'] = ('station',network)\n",
    "    \n",
    "    ## Output to netcdf\n",
    "    os.chdir(dir_data_out)\n",
    "    ds.to_netcdf('CA.grobs_combined.daily.xray.nc')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}