Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Dev letid geospatial #28

Merged
merged 2 commits into from
Nov 8, 2023
Merged

Dev letid geospatial #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47e80de
geospatial calc_letid_outdoors
martin-springer Oct 20, 2023
d706e79
updated kestrel config
martin-springer Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,7 @@ demo/chunk_files/
demo/logs/
demo/old

TEMP/
TEMP/

# Slurm log files
slurm-*.out
167 changes: 95 additions & 72 deletions pvdeg/geospatial.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from . import standards
from . import humidity
from . import letid

import xarray as xr
import dask.array as da
Expand Down Expand Up @@ -39,20 +40,17 @@ def start_dask(hpc=None):

.. code-block:: python

hpc = {'manager': 'slurm',
'n_jobs': 1, # Max number of nodes used for parallel processing
'cores': 36,
'memory': '96GB',
'queue': 'debug',
'account': 'pvsoiling',
'walltime': '01:00:00',
'interface': 'ib0',
'processes': 18,
'local_directory': '/tmp/scratch',
'shared_temp_directory': '/scratch/mspringe',
'job_extra_directives': ['-o ./logs/slurm-%j.out'],
'silence_logs': 'error',
'death_timeout': '60',}
kestrel = {
'manager': 'slurm',
'n_jobs': 1, # Max number of nodes used for parallel processing
'cores': 104,
'memory': '256GB',
'account': 'pvsoiling',
'walltime': '4:00:00',
'processes': 52,
'local_directory': '/tmp/scratch',
'job_extra_directives': ['-o ./logs/slurm-%j.out'],
'death_timeout': 600,}

Returns
-------
Expand Down Expand Up @@ -131,11 +129,14 @@ def calc_block(weather_ds_block, future_meta_df, func, func_kwargs):
Dataset with results for a block of gids.
"""

res = weather_ds_block.groupby('gid').map(lambda ds_gid: calc_gid(
ds_gid=ds_gid,
meta_gid=future_meta_df.loc[ds_gid['gid'].values].to_dict(),
func=func,
**func_kwargs))
res = weather_ds_block.groupby('gid').map(
lambda ds_gid: calc_gid(
ds_gid=ds_gid,
meta_gid=future_meta_df.loc[ds_gid['gid'].values].to_dict(),
func=func,
**func_kwargs,
)
)
return res


Expand Down Expand Up @@ -166,22 +167,22 @@ def analysis(weather_ds, meta_df, func, template=None, **func_kwargs):
param = template_parameters(func)
template = output_template(weather_ds, **param)

#future_meta_df = client.scatter(meta_df)
kwargs = {'func': func,
'future_meta_df': meta_df,
'func_kwargs': func_kwargs}
# future_meta_df = client.scatter(meta_df)
kwargs = {'func': func, 'future_meta_df': meta_df, 'func_kwargs': func_kwargs}

stacked = weather_ds.map_blocks(calc_block, kwargs=kwargs, template=template).compute()
stacked = weather_ds.map_blocks(
calc_block, kwargs=kwargs, template=template
).compute()

# lats = stacked.latitude.values.flatten()
# lons = stacked.longitude.values.flatten()
stacked = stacked.drop(['gid'])
# stacked = stacked.drop_vars(['latitude', 'longitude'])
stacked.coords['gid'] = pd.MultiIndex.from_arrays([
meta_df['latitude'], meta_df['longitude']],
names=['latitude', 'longitude'])
stacked.coords['gid'] = pd.MultiIndex.from_arrays(
[meta_df['latitude'], meta_df['longitude']], names=['latitude', 'longitude']
)

res = stacked.unstack('gid') #, sparse=True
res = stacked.unstack('gid') # , sparse=True
return res


Expand Down Expand Up @@ -209,13 +210,15 @@ def output_template(ds_gids, shapes, attrs=dict(), add_dims=dict()):
Template for output data.
"""
dims = set([d for dim in shapes.values() for d in dim])
dims_size = dict(ds_gids.dims) | add_dims
dims_size = dict(ds_gids.dims) | add_dims

output_template = xr.Dataset(
data_vars = {var: (dim, da.empty([dims_size[d] for d in dim])
) for var, dim in shapes.items()},
coords = {dim: ds_gids[dim] for dim in dims},
attrs = attrs
data_vars={
var: (dim, da.empty([dims_size[d] for d in dim]))
for var, dim in shapes.items()
},
coords={dim: ds_gids[dim] for dim in dims},
attrs=attrs,
).chunk({dim: ds_gids.chunks[dim] for dim in dims})

return output_template
Expand All @@ -236,25 +239,45 @@ def template_parameters(func):
"""

if func == standards.standoff:

shapes = {'x': ('gid',),
'T98_inf': ('gid',),
'T98_0': ('gid',),
}

attrs = {'x' : {'units': 'cm'},
'T98_0' : {'units': 'Celsius'},
'T98_inf' : {'units': 'Celsius'}}
shapes = {
'x': ('gid',),
'T98_inf': ('gid',),
'T98_0': ('gid',),
}

attrs = {
'x': {'units': 'cm'},
'T98_0': {'units': 'Celsius'},
'T98_inf': {'units': 'Celsius'},
}

add_dims = {}

elif func == humidity.module:
shapes = {
'RH_surface_outside': ('gid', 'time'),
'RH_front_encap': ('gid', 'time'),
'RH_back_encap': ('gid', 'time'),
'RH_backsheet': ('gid', 'time'),
}

shapes = {'RH_surface_outside': ('gid', 'time'),
'RH_front_encap': ('gid', 'time'),
'RH_back_encap': ('gid', 'time'),
'RH_backsheet': ('gid', 'time'),
}
attrs = {}

add_dims = {}

elif func == letid.calc_letid_outdoors:
shapes = {'Temperature': ('gid', 'time'),
'Injection': ('gid', 'time'),
'NA': ('gid', 'time'),
'NB': ('gid', 'time'),
'NC': ('gid', 'time'),
'tau': ('gid', 'time'),
'Jsc': ('gid', 'time'),
'Voc': ('gid', 'time'),
'Isc': ('gid', 'time'),
'FF': ('gid', 'time'),
'Pmp': ('gid', 'time'),
'Pmp_norm': ('gid', 'time')}

attrs = {}

Expand All @@ -263,43 +286,43 @@ def template_parameters(func):
else:
raise ValueError(f"No preset output template for function {func}.")

parameters = {'shapes': shapes,
'attrs': attrs,
'add_dims': add_dims}
parameters = {'shapes': shapes, 'attrs': attrs, 'add_dims': add_dims}

return parameters


def plot_USA(xr_res,
cmap='viridis',
vmin=None,
vmax=None,
title=None,
cb_title=None,
fp=None):

def plot_USA(
xr_res, cmap='viridis', vmin=None, vmax=None, title=None, cb_title=None, fp=None
):
fig = plt.figure()
ax = fig.add_axes([0, 0, 1, 1],
projection=ccrs.LambertConformal(),
frameon=False)
ax = fig.add_axes([0, 0, 1, 1], projection=ccrs.LambertConformal(), frameon=False)
ax.patch.set_visible(False)
ax.set_extent([-120, -74, 22, 50], ccrs.Geodetic())

shapename = 'admin_1_states_provinces_lakes'
states_shp = shpreader.natural_earth(
resolution='110m', category='cultural', name=shapename)
resolution='110m', category='cultural', name=shapename
)
ax.add_geometries(
shpreader.Reader(states_shp).geometries(),
ccrs.PlateCarree(), facecolor='w', edgecolor='gray')

cm = xr_res.plot(transform=ccrs.PlateCarree(),
zorder=10,
add_colorbar=False,
cmap=cmap,
vmin=vmin,
vmax=vmax,
subplot_kws={"projection": ccrs.LambertConformal(
central_longitude=-95, central_latitude=45)})
ccrs.PlateCarree(),
facecolor='w',
edgecolor='gray',
)

cm = xr_res.plot(
transform=ccrs.PlateCarree(),
zorder=10,
add_colorbar=False,
cmap=cmap,
vmin=vmin,
vmax=vmax,
subplot_kws={
"projection": ccrs.LambertConformal(
central_longitude=-95, central_latitude=45
)
},
)

cb = plt.colorbar(cm, shrink=0.5)
cb.set_label(cb_title)
Expand Down
8 changes: 5 additions & 3 deletions pvdeg/letid.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -986,7 +986,7 @@ def calc_letid_outdoors(
# Set up system, run pvlib.modelchain, and get the results we need: cell temp and injection
lat = float(meta["latitude"])
lon = float(meta["longitude"])
tz = meta["Local Time Zone"]
tz = meta["timezone"]
elevation = meta["altitude"]

if tilt is None:
Expand Down Expand Up @@ -1030,7 +1030,7 @@ def calc_letid_outdoors(
{"Temperature": temperature, "Injection": injection}
) # create a DataFrame with cell temperature and injection
timesteps.reset_index(inplace=True) # reset the index so datetime is a column.
timesteps.rename(columns={"index": "Datetime"}, inplace=True)
timesteps.rename(columns={"index": "time"}, inplace=True)
timesteps.reset_index(inplace=True, drop=True)

# create columns for defect state percentages and lifetime, fill with NaNs for now, to fill iteratively below
Expand Down Expand Up @@ -1139,7 +1139,7 @@ def calc_letid_outdoors(
dN_Cdt = (k_BC * n_B * x_bc) - (k_CB * n_C)

t_step = (
timesteps.at[index, "Datetime"] - timesteps.at[index - 1, "Datetime"]
timesteps.at[index, "time"] - timesteps.at[index - 1, "time"]
).total_seconds()

# assign new defect state percentages
Expand All @@ -1156,6 +1156,8 @@ def calc_letid_outdoors(
timesteps["tau"] = tau_now(tau_0, tau_deg, timesteps["NB"])
timesteps = calc_device_params(timesteps, cell_area)

timesteps.set_index('time', inplace=True)

return timesteps


Expand Down
13 changes: 11 additions & 2 deletions pvdeg/weather.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ def get(database, id=None, geospatial=False, **kwargs):
"""


META_MAP = {'elevation' : 'altitude'}
META_MAP = {'elevation' : 'altitude',
'Local Time Zone' : 'timezone'}

if type(id) is tuple:
location = id
Expand Down Expand Up @@ -117,6 +118,9 @@ def read(file_in, file_type, **kwargs):
[psm3, tmy3, epw, h5]
"""

META_MAP = {'elevation' : 'altitude',
'Local Time Zone' : 'timezone'}

supported = ['psm3','tmy3','epw','h5']
file_type = file_type.upper()

Expand All @@ -136,6 +140,11 @@ def read(file_in, file_type, **kwargs):
if not isinstance(meta, dict):
meta = meta.to_dict()

# map meta-names as needed
for key in [*meta.keys()]:
if key in META_MAP.keys():
meta[META_MAP[key]] = meta.pop(key)

return weather_df, meta


Expand Down Expand Up @@ -306,7 +315,7 @@ def get_NSRDB_fnames(satellite, names, NREL_HPC = False, **_):
'Americas' : 'current'}

if NREL_HPC:
hpc_fp = '/datasets/NSRDB/'
hpc_fp = '/kfs2/pdatasets/NSRDB/'
hsds = False
else:
hpc_fp = '/nrel/nsrdb/'
Expand Down
Loading