From 4006222e63162550e2ec539477a9544fe6b689d5 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Thu, 21 Dec 2023 18:06:27 -0500 Subject: [PATCH 1/7] update documentation --- docs/notebooks/getting_started.ipynb | 7106 ++++++++++++++++++++++++-- environment-dev.yml | 1 + environment-docs.yml | 1 + pyproject.toml | 2 + xdatasets/core.py | 35 +- xdatasets/spatial.py | 10 +- xdatasets/temporal.py | 8 +- xdatasets/utils.py | 66 +- xdatasets/workflows.py | 38 +- 9 files changed, 6802 insertions(+), 465 deletions(-) diff --git a/docs/notebooks/getting_started.ipynb b/docs/notebooks/getting_started.ipynb index 8114db8..469a3a1 100644 --- a/docs/notebooks/getting_started.ipynb +++ b/docs/notebooks/getting_started.ipynb @@ -29,7 +29,7 @@ "source": [ "```python\n", "query = {\n", - " \"datasets\": {\"era5_reanalysis_single_levels_dev\": {'variables': [\"t2m\", \"tp\"]}},\n", + " \"datasets\": {\"era5_reanalysis_single_levels\": {'variables': [\"t2m\", \"tp\"]}},\n", " \"space\": {\n", " \"clip\": \"point\", # bbox, point or polygon\n", " \"geometry\": {'Montreal' : (45.508888, -73.561668),\n", @@ -58,7 +58,7 @@ "source": [ "```python\n", "query = {\n", - " \"datasets\": {\"era5_reanalysis_single_levels_dev\": {'variables': [\"t2m\", \"tp\"]}},\n", + " \"datasets\": {\"era5_reanalysis_single_levels\": {'variables': [\"t2m\", \"tp\"]}},\n", " \"space\": {\n", " \"clip\": \"polygon\", # bbox, point or polygon\n", " \"aggregation\": True, # spatial average of the variables within each polygon\n", @@ -128,11 +128,9 @@ " }\n", "\n", " var force = true;\n", - " var py_version = '3.2.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", - " var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n", + " var py_version = '3.3.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", " var reloading = false;\n", " var Bokeh = root.Bokeh;\n", - " var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n", "\n", " if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n", " root._bokeh_timeout = Date.now() + 5000;\n", @@ -244,17 +242,17 @@ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", " document.body.appendChild(element);\n", " } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n", + " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n", " for (var i = 0; i < urls.length; i++) {\n", " skip.push(urls[i])\n", " }\n", " } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n", + " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n", " for (var i = 0; i < urls.length; i++) {\n", " skip.push(urls[i])\n", " }\n", " } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n", + " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n", " for (var i = 0; i < urls.length; i++) {\n", " skip.push(urls[i])\n", " }\n", @@ -330,7 +328,7 @@ " document.body.appendChild(element);\n", " }\n", "\n", - " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.2.2.min.js\", \"https://cdn.holoviz.org/panel/1.2.1/dist/panel.min.js\"];\n", + " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.holoviz.org/panel/1.3.6/dist/panel.min.js\"];\n", " var js_modules = [];\n", " var js_exports = {};\n", " var css_urls = [];\n", @@ -343,7 +341,13 @@ " function run_inline_js() {\n", " if ((root.Bokeh !== undefined) || (force === true)) {\n", " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i].call(root, root.Bokeh);\n", + "\ttry {\n", + " inline_js[i].call(root, root.Bokeh);\n", + "\t} catch(e) {\n", + "\t if (!reloading) {\n", + "\t throw e;\n", + "\t }\n", + "\t}\n", " }\n", " // Cache old bokeh versions\n", " if (Bokeh != undefined && !reloading) {\n", @@ -380,11 +384,10 @@ " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", " setTimeout(load_or_wait, 100);\n", " } else {\n", - " Bokeh = root.Bokeh;\n", - " bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n", " root._bokeh_is_initializing = true\n", " root._bokeh_onload_callbacks = []\n", - " if (!reloading && (!bokeh_loaded || is_dev)) {\n", + " var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n", + " if (!reloading && !bokeh_loaded) {\n", "\troot.Bokeh = undefined;\n", " }\n", " load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n", @@ -398,7 +401,7 @@ " setTimeout(load_or_wait, 100)\n", "}(window));" ], - "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.2.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var is_dev = py_version.indexOf(\"+\") !== -1 || py_version.indexOf(\"-\") !== -1;\n var reloading = false;\n var Bokeh = root.Bokeh;\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.2.1/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.2.1/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.2.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.2.2.min.js\", \"https://cdn.holoviz.org/panel/1.2.1/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n Bokeh = root.Bokeh;\n bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n if (!reloading && (!bokeh_loaded || is_dev)) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.3.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var reloading = false;\n var Bokeh = root.Bokeh;\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.holoviz.org/panel/1.3.6/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n\ttry {\n inline_js[i].call(root, root.Bokeh);\n\t} catch(e) {\n\t if (!reloading) {\n\t throw e;\n\t }\n\t}\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n if (!reloading && !bokeh_loaded) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" }, "metadata": {}, "output_type": "display_data" @@ -674,6 +677,92 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1002" + } + }, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/clisops/core/regrid.py:42: UserWarning: xarray version >= 2023.03.0 is not supported for regridding operations with cf-time indexed arrays. Please use xarray version < 2023.03.0. For more information, see: https://github.com/pydata/xarray/issues/7794.\n", + " warnings.warn(\n" + ] } ], "source": [ @@ -741,10 +830,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Temporal operations: processing tp with era5_reanalysis_single_levels: 100%|██████████████████████████████████████████████| 2/2 [00:04<00:00, 2.46s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1min 1s, sys: 3.71 s, total: 1min 5s\n", + "Wall time: 1min 5s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ + "%%time\n", + "\n", "query = {\n", " \"datasets\": \"era5_reanalysis_single_levels\",\n", " \"space\": {\"clip\": \"point\", \"geometry\": sites}, # bbox, point or polygon\n", @@ -767,31 +881,576 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:      (spatial_agg: 1, timestep: 1, site: 3, time: 23561, source: 1)\n",
+       "Coordinates:\n",
+       "  * spatial_agg  (spatial_agg) object 'point'\n",
+       "  * timestep     (timestep) object 'D'\n",
+       "    latitude     (site) float64 45.5 40.75 25.75\n",
+       "    longitude    (site) float64 -73.5 -74.0 -80.25\n",
+       "  * site         (site) <U8 'Montreal' 'New York' 'Miami'\n",
+       "  * time         (time) datetime64[ns] 1959-01-01 1959-01-02 ... 2023-07-04\n",
+       "  * source       (source) <U29 'era5_reanalysis_single_levels'\n",
+       "Data variables:\n",
+       "    t2m_nanmean  (spatial_agg, timestep, time, site) float32 260.7 ... 302.0\n",
+       "    tp_nansum    (spatial_agg, timestep, time, site) float32 0.0005895 ... 0....\n",
+       "Attributes: (12/30)\n",
+       "    GRIB_NV:                                  0\n",
+       "    GRIB_Nx:                                  1440\n",
+       "    GRIB_Ny:                                  721\n",
+       "    GRIB_cfName:                              unknown\n",
+       "    GRIB_cfVarName:                           t2m\n",
+       "    GRIB_dataType:                            an\n",
+       "    ...                                       ...\n",
+       "    GRIB_totalNumber:                         0\n",
+       "    GRIB_typeOfLevel:                         surface\n",
+       "    GRIB_units:                               K\n",
+       "    long_name:                                2 metre temperature\n",
+       "    standard_name:                            unknown\n",
+       "    units:                                    K
" + ], + "text/plain": [ + "\n", + "Dimensions: (spatial_agg: 1, timestep: 1, site: 3, time: 23561, source: 1)\n", + "Coordinates:\n", + " * spatial_agg (spatial_agg) object 'point'\n", + " * timestep (timestep) object 'D'\n", + " latitude (site) float64 45.5 40.75 25.75\n", + " longitude (site) float64 -73.5 -74.0 -80.25\n", + " * site (site) \n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] HoloViews(DynamicMap, height=450, sizing_mode='fixed', widget_location='bottom', width=750)\n", + " [1] WidgetBox(align=('center', 'end'))\n", + " [0] Select(margin=(20, 20, 20, 20), name='spatial_agg', options=['point'], value='point', width=250)" + ] + }, + "execution_count": 6, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1004" + } + }, + "output_type": "execute_result" + } + ], "source": [ "title = f\"Comparison of total precipitation across three cities in North America from \\\n", "{xds.data.time.dt.year.min().values} to {xds.data.time.dt.year.max().values}\"\n", "\n", "xds.data.sel(\n", " timestep=\"D\",\n", - " spatial_agg=\"point\",\n", - " time_agg=\"nansum\",\n", " source=\"era5_reanalysis_single_levels\",\n", ").hvplot(\n", " title=title,\n", " x=\"time\",\n", - " y=\"tp\",\n", + " y=\"tp_nansum\",\n", " grid=True,\n", " width=750,\n", " height=450,\n", @@ -803,22 +1462,116 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] HoloViews(DynamicMap, height=450, sizing_mode='fixed', widget_location='bottom', width=750)\n", + " [1] WidgetBox(align=('center', 'end'))\n", + " [0] Select(margin=(20, 20, 20, 20), name='spatial_agg', options=['point'], value='point', width=250)" + ] + }, + "execution_count": 7, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1122" + } + }, + "output_type": "execute_result" + } + ], "source": [ "title = f\"Comparison of 2m temperature across three cities in North America from \\\n", "{xds.data.time.dt.year.min().values} to {xds.data.time.dt.year.max().values}\"\n", "\n", "xds.data.sel(\n", " timestep=\"D\",\n", - " spatial_agg=\"point\",\n", - " time_agg=\"nanmean\",\n", " source=\"era5_reanalysis_single_levels\",\n", ").hvplot(\n", " title=title,\n", " x=\"time\",\n", - " y=\"t2m\",\n", + " y=\"t2m_nanmean\",\n", " grid=True,\n", " width=750,\n", " height=450,\n", @@ -834,53 +1587,198 @@ "source": [ "### Clip on polygons with no averaging in space\n", "\n", - "Let's first access certain polygon features, which can be in the form of shapefiles, geojson, or any other format compatible with `geopandas`. In this example, we are using `JSON` (geojson) files." + "First, let's explore specific polygon features. With `xdatasets`, you can access geographical datasets, such as watershed boundaries linked to streamflow stations. These datasets follow a nomenclature where they are named after the hydrological dataset, with `\"_polygons\"` appended. For example, if the hydrological dataset is named `deh`, its corresponding watershed boundaries dataset will be labeled `deh_polygons`. The query below retrieves all polygons for the `deh_polygons` dataset.\n", + "\n", + "```python\n", + "gdf = xd.Query(\n", + " **{\n", + " \"datasets\": \"deh_polygons\"\n", + "}).data\n", + "\n", + "gdf\n", + "```\n", + "\n", + "As the data is loaded into memory, the process of loading all polygons may take some time. To expedite this, we recommend employing filters, as illustrated below. It's important to note that the filters are consistent for both hydrological and corresponding geographical datasets. Consequently, only watershed boundaries associated with existing hydrological data will be returned." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StationSuperficiegeometry
0042102623.479187POLYGON ((-78.57120 46.70742, -78.57112 46.707...
1042103579.479614POLYGON ((-78.49014 46.64514, -78.49010 46.645...
\n", + "
" + ], + "text/plain": [ + " Station Superficie geometry\n", + "0 042102 623.479187 POLYGON ((-78.57120 46.70742, -78.57112 46.707...\n", + "1 042103 579.479614 POLYGON ((-78.49014 46.64514, -78.49010 46.645..." + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "bucket = Path(\"https://s3.us-east-2.wasabisys.com/watersheds-polygons/MELCC/json\")\n", + "import xdatasets as xd\n", + "\n", + "gdf = xd.Query(\n", + " **{\n", + " \"datasets\": {\n", + " \"deh_polygons\": {\n", + " \"id\": [\"0421*\"],\n", + " \"regulated\": [\"Natural\"],\n", + " }\n", + " }\n", + " }\n", + ").data.reset_index()\n", "\n", - "paths = [\n", - " bucket.joinpath(\"023003/023003.json\"),\n", - " bucket.joinpath(\"031101/031101.json\"),\n", - " bucket.joinpath(\"040111/040111.json\"),\n", - "]" + "gdf" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Subsequently, all of the files can be opened and consolidated into a single `geopandas.GeoDataFrame` object." + "Let's examine the geographic locations of the polygon features." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], - "source": [ - "gdf = pd.concat([gpd.read_file(path) for path in paths]).reset_index(drop=True)\n", - "gdf" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's examine the geographic locations of the polygon features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + ":Overlay\n", + " .WMTS.I :WMTS [Longitude,Latitude]\n", + " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)" + ] + }, + "execution_count": 9, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1247" + } + }, + "output_type": "execute_result" + } + ], "source": [ "gdf.hvplot(\n", " geo=True,\n", @@ -903,27 +1801,55 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Spatial operations: processing polygon 042102 with era5_reanalysis_single_levels: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 1it [00:00, 5.25it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 2it [00:00, 5.39it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 26 s, sys: 923 ms, total: 26.9 s\n", + "Wall time: 27.7 s\n" + ] + } + ], "source": [ - "# FIXME: New dimensions ('lat', 'bnds') must be a superset of existing dimensions ('lat', 'bnd')\n", + "%%time\n", + "\n", + "query = {\n", + " \"datasets\": {\"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]}},\n", + " \"space\": {\n", + " \"clip\": \"polygon\", # bbox, point or polygon\n", + " \"averaging\": False, # spatial average of the variables within each polygon\n", + " \"geometry\": gdf,\n", + " \"unique_id\": \"Station\", # unique column name in geodataframe\n", + " },\n", + " \"time\": {\n", + " \"start\": \"1959-01-01\",\n", + " \"end\": \"1963-08-31\",\n", + " },\n", + "}\n", "\n", - "# query = {\n", - "# \"datasets\": {\"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]}},\n", - "# \"space\": {\n", - "# \"clip\": \"polygon\", # bbox, point or polygon\n", - "# \"averaging\": False, # spatial average of the variables within each polygon\n", - "# \"geometry\": gdf,\n", - "# \"unique_id\": \"Station\", # unique column name in geodataframe\n", - "# },\n", - "# \"time\": {\n", - "# \"start\": \"1959-01-01\",\n", - "# \"end\": \"1963-08-31\",\n", - "# },\n", - "# }\n", - "#\n", - "# xds = xd.Query(**query)" + "xds = xd.Query(**query)" ] }, { @@ -935,414 +1861,5755 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# xds.data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Weights are much easier to comprehend visually, so let's examine the weights returned for the station *023003*. Notice that when selecting a single feature (Station *023003* in this case), the shape of our spatial dimensions is reduced to a 2x2 pixel area (longitude x latitude) that encompasses the entire feature." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# station = \"023003\"\n", - "#\n", - "# ds_station = xds.data.sel(Station=station)\n", - "# ds_clipped = xds.bbox_clip(ds_station).squeeze()\n", - "# ds_clipped" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME: This cell raises `Exception: An axis may only be assigned one projection type`\n", - "\n", - "# (\n", - "# (\n", - "# ds_clipped.t2m.isel(time=0).hvplot(\n", - "# title=\"The 2m temperature for pixels that intersect with the polygon on January 1, 1959\",\n", - "# tiles=\"ESRI\",\n", - "# geo=True,\n", - "# alpha=0.6,\n", - "# colormap=\"isolum\",\n", - "# width=750,\n", - "# height=450,\n", - "# )\n", - "# * gdf[gdf.Station == station].hvplot(\n", - "# geo=True,\n", - "# width=750,\n", - "# height=450,\n", - "# legend=\"top\",\n", - "# hover_cols=[\"Station\", \"Superficie\"],\n", - "# )\n", - "# )\n", - "# + ds_clipped.weights.hvplot(\n", - "# title=\"The weights that should be assigned to each pixel when performing spatial averaging\",\n", - "# tiles=\"ESRI\",\n", - "# alpha=0.6,\n", - "# colormap=\"isolum\",\n", - "# geo=True,\n", - "# width=750,\n", - "# height=450,\n", - "# )\n", - "# * gdf[gdf.Station == station].hvplot(\n", - "# geo=True,\n", - "# width=750,\n", - "# height=450,\n", - "# legend=\"top\",\n", - "# hover_cols=[\"Station\", \"Superficie\"],\n", - "# )\n", - "# ).cols(1)" - ] - }, - { - "cell_type": "markdown", + "execution_count": 11, "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:    (Station: 2, time: 40896, latitude: 3, longitude: 2, source: 1)\n",
+       "Coordinates:\n",
+       "  * latitude   (latitude) float64 46.25 46.5 46.75\n",
+       "  * longitude  (longitude) float64 -78.5 -78.25\n",
+       "  * time       (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n",
+       "  * Station    (Station) object '042102' '042103'\n",
+       "  * source     (source) <U29 'era5_reanalysis_single_levels'\n",
+       "Data variables:\n",
+       "    t2m        (Station, time, latitude, longitude) float32 260.3 259.9 ... nan\n",
+       "    tp         (Station, time, latitude, longitude) float32 nan nan ... 0.0 nan\n",
+       "    weights    (Station, latitude, longitude) float64 0.01953 0.1244 ... nan\n",
+       "Attributes:\n",
+       "    Conventions:               CF-1.6\n",
+       "    history:                   2022-11-10 02:03:41 GMT by grib_to_netcdf-2.25...\n",
+       "    pangeo-forge:inputs_hash:  713ae1666a879e6169213b0f03cd75f0b7f2a1769195e6...\n",
+       "    pangeo-forge:recipe_hash:  ad6e205f84d68b1e012856dc70630e06de594c50cfbe7c...\n",
+       "    pangeo-forge:version:      0.9.4
" + ], + "text/plain": [ + "\n", + "Dimensions: (Station: 2, time: 40896, latitude: 3, longitude: 2, source: 1)\n", + "Coordinates:\n", + " * latitude (latitude) float64 46.25 46.5 46.75\n", + " * longitude (longitude) float64 -78.5 -78.25\n", + " * time (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n", + " * Station (Station) object '042102' '042103'\n", + " * source (source) **Note**\n", - "> If users prefer to pass multiple dictionaries instead of a single large one, the following format is also considered acceptable." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME: new dimensions ('lat', 'bnds') must be a superset of existing dimensions ('lat', 'bnd')\n", - "\n", - "# datasets = {\n", - "# \"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]},\n", - "# \"era5_land_reanalysis_dev\": {\"variables\": [\"t2m\", \"tp\"]},\n", - "# }\n", - "# space = {\n", - "# \"clip\": \"polygon\", # bbox, point or polygon\n", - "# \"averaging\": True,\n", - "# \"geometry\": gdf, # 3 polygons\n", - "# \"unique_id\": \"Station\",\n", - "# }\n", - "# time = {\n", - "# \"timestep\": \"D\",\n", - "# \"aggregation\": {\"tp\": [np.nansum], \"t2m\": [np.nanmax, np.nanmin]},\n", - "# \"start\": \"1950-01-01\",\n", - "# \"timezone\": \"America/Montreal\",\n", - "# }\n", - "#\n", - "# xds = xd.Query(datasets=datasets, space=space, time=time)" + "Weights are much easier to comprehend visually, so let's examine the weights returned for the station *042102*. Notice that when selecting a single feature (Station *042102* in this case), the shape of our spatial dimensions is reduced to a 3x2 pixel area (longitude x latitude) that encompasses the entire feature." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], - "source": [ - "# xds.data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:    (time: 40896, latitude: 3, longitude: 2)\n",
+       "Coordinates:\n",
+       "  * latitude   (latitude) float64 46.25 46.5 46.75\n",
+       "  * longitude  (longitude) float64 -78.5 -78.25\n",
+       "  * time       (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n",
+       "    Station    <U6 '042102'\n",
+       "    source     <U29 'era5_reanalysis_single_levels'\n",
+       "Data variables:\n",
+       "    t2m        (time, latitude, longitude) float32 260.3 259.9 ... 288.2 nan\n",
+       "    tp         (time, latitude, longitude) float32 nan nan nan ... 0.0 0.0 nan\n",
+       "    weights    (latitude, longitude) float64 0.01953 0.1244 ... 0.0481 nan\n",
+       "Attributes:\n",
+       "    Conventions:               CF-1.6\n",
+       "    history:                   2022-11-10 02:03:41 GMT by grib_to_netcdf-2.25...\n",
+       "    pangeo-forge:inputs_hash:  713ae1666a879e6169213b0f03cd75f0b7f2a1769195e6...\n",
+       "    pangeo-forge:recipe_hash:  ad6e205f84d68b1e012856dc70630e06de594c50cfbe7c...\n",
+       "    pangeo-forge:version:      0.9.4
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 40896, latitude: 3, longitude: 2)\n", + "Coordinates:\n", + " * latitude (latitude) float64 46.25 46.5 46.75\n", + " * longitude (longitude) float64 -78.5 -78.25\n", + " * time (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n", + " Station list.\n", + "station = \"042102\"\n", "\n", - "# xds.data.squeeze().t2m.hvplot(\n", - "# x=\"time\",\n", - "# by=\"time_agg\",\n", - "# groupby=[\"source\", \"Station\"],\n", - "# width=750,\n", - "# height=400,\n", - "# grid=True,\n", - "# widget_location=\"bottom\",\n", - "# )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The resulting dataset can be explored in the data attribute :" + "ds_station = xds.data.sel(Station=station)\n", + "ds_clipped = xds.bbox_clip(ds_station).squeeze()\n", + "ds_clipped" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + ":Layout\n", + " .Overlay.I :Overlay\n", + " .WMTS.I :WMTS [Longitude,Latitude]\n", + " .Image.I :Image [longitude,latitude] (t2m)\n", + " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)\n", + " .Overlay.II :Overlay\n", + " .WMTS.I :WMTS [Longitude,Latitude]\n", + " .Image.I :Image [longitude,latitude] (weights)\n", + " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)" + ] + }, + "execution_count": 13, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1346" + } + }, + "output_type": "execute_result" + } + ], "source": [ - "# FIXME: squeeze() does not work on xds.data -> list.\n", - "\n", - "# xds.data.squeeze().tp.sel(time_agg=\"nansum\").hvplot(\n", - "# x=\"time\",\n", - "# groupby=[\"source\", \"Station\"],\n", - "# width=750,\n", - "# height=400,\n", - "# color=\"blue\",\n", - "# grid=True,\n", - "# widget_location=\"bottom\",\n", - "# )" + "(\n", + " (\n", + " ds_clipped.t2m.isel(time=0).hvplot(\n", + " title=\"The 2m temperature for pixels that intersect with the polygon on January 1, 1959\",\n", + " tiles=\"ESRI\",\n", + " geo=True,\n", + " alpha=0.6,\n", + " colormap=\"isolum\",\n", + " width=750,\n", + " height=450,\n", + " )\n", + " * gdf[gdf.Station == station].hvplot(\n", + " geo=True,\n", + " width=750,\n", + " height=450,\n", + " legend=\"top\",\n", + " hover_cols=[\"Station\", \"Superficie\"],\n", + " )\n", + " )\n", + " + ds_clipped.weights.hvplot(\n", + " title=\"The weights that should be assigned to each pixel when performing spatial averaging\",\n", + " tiles=\"ESRI\",\n", + " alpha=0.6,\n", + " colormap=\"isolum\",\n", + " geo=True,\n", + " width=750,\n", + " height=450,\n", + " )\n", + " * gdf[gdf.Station == station].hvplot(\n", + " geo=True,\n", + " width=750,\n", + " height=450,\n", + " legend=\"top\",\n", + " hover_cols=[\"Station\", \"Superficie\"],\n", + " )\n", + ").cols(1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Bounding box (bbox) around polygons\n", - "\n", - "The following query seeks the variable `tp` from the `era5_land_reanalysis_dev` dataset, covering the period between January 1, 1959, and December 31, 1970, for the bounding box that delimits the three polygons mentioned earlier.\n", + "The two plots depicted above show the 2m temperature for each pixel that intersects with the polygon from Station `042102` and the corresponding weights to be applied to each pixel. In the lower plot, it is apparent that the majority of the polygon is situated in the central pixels, which results in those pixelx having a weight of approximately 80%. It is evident that the two lower and the upper pixels have much less intersection with the polygon, which results in their respective weights being smaller (hover on the plot to verify the weights).\n", "\n", - "Additional steps are carried out in the process, including converting to the local time zone." + "In various libraries, either all pixels that intersect with the geometries are kept, or only pixels with centers within the polygon are retained. However, as shown in the previous example, utilizing such methods can introduce significant biases in the final calculations." ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "query = {\n", - " \"datasets\": {\"era5_land_reanalysis_dev\": {\"variables\": [\"tp\"]}},\n", - " \"space\": {\n", - " \"clip\": \"bbox\", # bbox, point or polygon\n", - " \"geometry\": gdf,\n", - " },\n", - " \"time\": {\n", - " \"start\": \"1959-01-01\",\n", - " \"end\": \"1970-12-31\",\n", - " \"timezone\": \"America/Montreal\",\n", - " },\n", - "}\n", + "### Clip on polygons with averaging in space\n", "\n", + "The following query seeks the variables `t2m` and `tp` from the `era5_reanalysis_single_levels` and `era5_land_reanalysis` datasets, covering the period between January 1, 1950, to present, for the three polygons mentioned earlier. Note that when the `aggregation` parameter is set to `True`, spatial averaging takes place. In addition, the weighted mask (raster) described earlier will be applied to generate a time series for each polygon.\n", "\n", - "xds = xd.Query(**query)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME: This query does not return TP - instead returns d2m, sf, sp, ssrd, strd, u10, v10\n", + "Additional steps are carried out in the process, including converting the original hourly time step to a daily time step. During this conversion, various temporal aggregations will be applied to each variable and a conversion to the local time zone will take place.\n", "\n", - "xds.data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's find out which day (24-hour period) was the rainiest in the entire region for the data retrieved in previous cell." + "> **Note**\n", + "> If users prefer to pass multiple dictionaries instead of a single large one, the following format is also considered acceptable." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xdatasets/workflows.py:51: UserWarning: \"start_date\" not found within input date time range. Defaulting to minimum time step in xarray object.\n", + " ds = subset_time(ds, start_date=start_time, end_date=end_time)\n", + "Spatial operations: processing polygon 042102 with era5_reanalysis_single_levels: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 1it [00:00, 4.63it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 2it [00:00, 4.69it/s]\n", + "Temporal operations: processing tp with era5_reanalysis_single_levels: 100%|██████████████████████████████████████████████| 2/2 [00:07<00:00, 3.69s/it]\n", + "Spatial operations: processing polygon 042102 with era5_land_reanalysis: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_land_reanalysis: : 1it [00:00, 4.24it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", + " warnings.warn(\n", + "Spatial operations: processing polygon 042103 with era5_land_reanalysis: : 2it [00:00, 4.30it/s]\n", + "Temporal operations: processing tp with era5_land_reanalysis: 100%|███████████████████████████████████████████████████████| 2/2 [00:07<00:00, 3.79s/it]\n" + ] + } + ], "source": [ - "# indexer = (\n", - "# xds.data.sel(source=\"era5_land_reanalysis_dev\")\n", - "# .tp.sum([\"latitude\", \"longitude\"])\n", - "# .rolling(time=24)\n", - "# .sum()\n", - "# .argmax(\"time\")\n", - "# .values\n", - "# )\n", + "datasets = {\n", + " \"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]},\n", + " \"era5_land_reanalysis\": {\"variables\": [\"t2m\", \"tp\"]},\n", + "}\n", + "space = {\n", + " \"clip\": \"polygon\", # bbox, point or polygon\n", + " \"averaging\": True,\n", + " \"geometry\": gdf, # 3 polygons\n", + " \"unique_id\": \"Station\",\n", + "}\n", + "time = {\n", + " \"timestep\": \"D\",\n", + " \"aggregation\": {\"tp\": [np.nansum], \"t2m\": [np.nanmax, np.nanmin]},\n", + " \"start\": \"1950-01-01\",\n", + " \"timezone\": \"America/Montreal\",\n", + "}\n", "\n", - "# xds.data.isel(time=indexer).time.dt.date.values.tolist()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's visualise the evolution of the hourly precipitation during that day. Note that each image (raster) delimits exactly the bounding box required to cover all polygons in the query. Please note that for full interactivity, running the code in a Jupyter Notebook is necessary.\n", - "\n" + "xds = xd.Query(datasets=datasets, space=space, time=time)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# da = xds.data.tp.isel(time=slice(indexer - 24, indexer))\n", - "# # da = da.where(da>0.0001, drop=True)\n", - "\n", - "# (da * 1000).sel(source=\"era5_land_reanalysis_dev\").squeeze().hvplot.quadmesh(\n", - "# width=750,\n", - "# height=450,\n", - "# geo=True,\n", - "# tiles=\"ESRI\",\n", - "# groupby=[\"time\"],\n", - "# legend=\"top\",\n", - "# cmap=\"gist_ncar\",\n", - "# widget_location=\"bottom\",\n", - "# widget_type=\"scrubber\",\n", - "# dynamic=False,\n", - "# clim=(0.01, 10),\n", - "# )" - ] - }, - { - "cell_type": "markdown", + "execution_count": 15, "metadata": {}, - "source": [ - "## Query hydrological datasets\n", - "Hydrological queries are still being tested and output format is likely to change. Stay tuned!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# FIXME: MELCC not yet supported.\n", + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:      (spatial_agg: 1, timestep: 1, Station: 2, time: 27012,\n",
+       "                  source: 2)\n",
+       "Coordinates:\n",
+       "  * spatial_agg  (spatial_agg) object 'polygon'\n",
+       "  * timestep     (timestep) object 'D'\n",
+       "  * Station      (Station) object '042102' '042103'\n",
+       "  * time         (time) datetime64[ns] 1950-01-01 1950-01-02 ... 2023-12-15\n",
+       "  * source       (source) <U29 'era5_land_reanalysis' 'era5_reanalysis_single...\n",
+       "Data variables:\n",
+       "    t2m_nanmax   (spatial_agg, timestep, Station, time, source) float64 272.8...\n",
+       "    t2m_nanmin   (spatial_agg, timestep, Station, time, source) float64 268.7...\n",
+       "    tp_nansum    (spatial_agg, timestep, Station, time, source) float64 0.000...\n",
+       "Attributes: (12/30)\n",
+       "    GRIB_NV:                                  0\n",
+       "    GRIB_Nx:                                  1440\n",
+       "    GRIB_Ny:                                  721\n",
+       "    GRIB_cfName:                              unknown\n",
+       "    GRIB_cfVarName:                           t2m\n",
+       "    GRIB_dataType:                            an\n",
+       "    ...                                       ...\n",
+       "    GRIB_totalNumber:                         0\n",
+       "    GRIB_typeOfLevel:                         surface\n",
+       "    GRIB_units:                               K\n",
+       "    long_name:                                2 metre temperature\n",
+       "    standard_name:                            unknown\n",
+       "    units:                                    K
" + ], + "text/plain": [ + "\n", + "Dimensions: (spatial_agg: 1, timestep: 1, Station: 2, time: 27012,\n", + " source: 2)\n", + "Coordinates:\n", + " * spatial_agg (spatial_agg) object 'polygon'\n", + " * timestep (timestep) object 'D'\n", + " * Station (Station) object '042102' '042103'\n", + " * time (time) datetime64[ns] 1950-01-01 1950-01-02 ... 2023-12-15\n", + " * source (source) \n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] HoloViews(DynamicMap, height=400, sizing_mode='fixed', widget_location='bottom', width=750)\n", + " [1] WidgetBox(align=('center', 'end'))\n", + " [0] Select(margin=(20, 20, 5, 20), name='Station', options=['042102', '042103'], value='042102', width=250)\n", + " [1] Select(margin=(5, 20, 20, 20), name='source', options=['era5_land_reanalysis', ...], value='era5_land_reanalysis', width=250)" + ] + }, + "execution_count": 22, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1729" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "(\n", + " xds.data[[\"t2m_nanmax\", \"t2m_nanmin\"]]\n", + " .squeeze()\n", + " .hvplot(\n", + " x=\"time\",\n", + " groupby=[\"Station\", \"source\"],\n", + " width=750,\n", + " height=400,\n", + " grid=True,\n", + " widget_location=\"bottom\",\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting dataset can be explored for the `total_precipitation` (tp) data attribute :" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] HoloViews(DynamicMap, height=400, sizing_mode='fixed', widget_location='bottom', width=750)\n", + " [1] WidgetBox(align=('center', 'end'))\n", + " [0] Select(margin=(20, 20, 5, 20), name='Station', options=['042102', '042103'], value='042102', width=250)\n", + " [1] Select(margin=(5, 20, 20, 20), name='source', options=['era5_land_reanalysis', ...], value='era5_land_reanalysis', width=250)" + ] + }, + "execution_count": 24, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p1915" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "(\n", + " xds.data[[\"tp_nansum\"]]\n", + " .squeeze()\n", + " .hvplot(\n", + " x=\"time\",\n", + " groupby=[\"Station\", \"source\"],\n", + " width=750,\n", + " height=400,\n", + " grid=True,\n", + " widget_location=\"bottom\",\n", + " color=\"blue\",\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bounding box (bbox) around polygons\n", + "\n", + "The following query seeks the variable `tp` from the `era5_land_reanalysis_dev` dataset, covering the period between January 1, 1959, and December 31, 1970, for the bounding box that delimits the three polygons mentioned earlier.\n", + "\n", + "Additional steps are carried out in the process, including converting to the local time zone." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "query = {\n", + " \"datasets\": {\"era5_land_reanalysis\": {\"variables\": [\"tp\"]}},\n", + " \"space\": {\n", + " \"clip\": \"bbox\", # bbox, point or polygon\n", + " \"geometry\": gdf,\n", + " },\n", + " \"time\": {\n", + " \"start\": \"1959-01-01\",\n", + " \"end\": \"1970-12-31\",\n", + " \"timezone\": \"America/Montreal\",\n", + " },\n", + "}\n", + "\n", + "\n", + "xds = xd.Query(**query)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:    (time: 105192, latitude: 9, longitude: 11, source: 1)\n",
+       "Coordinates:\n",
+       "  * latitude   (latitude) float64 46.9 46.8 46.7 46.6 46.5 46.4 46.3 46.2 46.1\n",
+       "  * longitude  (longitude) float64 -78.9 -78.8 -78.7 -78.6 ... -78.1 -78.0 -77.9\n",
+       "  * time       (time) datetime64[ns] 1959-01-01 ... 1970-12-31T23:00:00\n",
+       "  * source     (source) <U20 'era5_land_reanalysis'\n",
+       "Data variables:\n",
+       "    tp         (time, latitude, longitude) float32 1.08e-07 1.08e-07 ... 0.0 0.0\n",
+       "Attributes:\n",
+       "    pangeo-forge:inputs_hash:  6edb2d7ac9d207d7933d56a7c36d09d12b07e35dfdbf89...\n",
+       "    pangeo-forge:recipe_hash:  46fd6fcf6c2c3cd36d2b10bb6c1196bd26f98c5c504d70...\n",
+       "    pangeo-forge:version:      0.9.4\n",
+       "    timezone:                  America/Montreal
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 105192, latitude: 9, longitude: 11, source: 1)\n", + "Coordinates:\n", + " * latitude (latitude) float64 46.9 46.8 46.7 46.6 46.5 46.4 46.3 46.2 46.1\n", + " * longitude (longitude) float64 -78.9 -78.8 -78.7 -78.6 ... -78.1 -78.0 -77.9\n", + " * time (time) datetime64[ns] 1959-01-01 ... 1970-12-31T23:00:00\n", + " * source (source) \n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] HoloViews(HoloMap, height=450, sizing_mode='fixed', widget_location='bottom', widget_type='scrubber', width=750)\n", + " [1] WidgetBox(align=('center', 'end'))\n", + " [0] Player(end=23, width=550)" + ] + }, + "execution_count": 29, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "p2014" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "da = xds.data.tp.isel(time=slice(indexer - 24, indexer))\n", + "da = da.where(da > 0.0001, drop=True)\n", + "\n", + "(da * 1000).squeeze().hvplot.quadmesh(\n", + " width=750,\n", + " height=450,\n", + " geo=True,\n", + " tiles=\"ESRI\",\n", + " groupby=[\"time\"],\n", + " legend=\"top\",\n", + " cmap=\"gist_ncar\",\n", + " widget_location=\"bottom\",\n", + " widget_type=\"scrubber\",\n", + " dynamic=False,\n", + " clim=(0.01, 10),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query hydrological datasets\n", + "Hydrological queries are still being tested and output format is likely to change. Stay tuned!" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:        (id: 744, variable: 2, spatial_agg: 2, timestep: 1,\n",
+       "                    time_agg: 1, source: 1, time: 60631)\n",
+       "Coordinates: (12/15)\n",
+       "    drainage_area  (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
+       "    end_date       (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array<chunksize=(2, 744, 2, 1, 1, 1), meta=np.ndarray>\n",
+       "  * id             (id) object '010101' '010801' '010802' ... '104804' '120201'\n",
+       "    latitude       (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
+       "    longitude      (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
+       "    name           (id) object dask.array<chunksize=(744,), meta=np.ndarray>\n",
+       "    ...             ...\n",
+       "  * spatial_agg    (spatial_agg) object 'point' 'watershed'\n",
+       "    start_date     (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array<chunksize=(2, 744, 2, 1, 1, 1), meta=np.ndarray>\n",
+       "  * time           (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2025-12-31\n",
+       "  * time_agg       (time_agg) object 'mean'\n",
+       "  * timestep       (timestep) object 'D'\n",
+       "  * variable       (variable) object 'level' 'streamflow'\n",
+       "Data variables:\n",
+       "    level          (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array<chunksize=(1, 60631, 1, 1, 1, 1, 1), meta=np.ndarray>\n",
+       "    streamflow     (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array<chunksize=(1, 60631, 1, 1, 1, 1, 1), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (id: 744, variable: 2, spatial_agg: 2, timestep: 1,\n", + " time_agg: 1, source: 1, time: 60631)\n", + "Coordinates: (12/15)\n", + " drainage_area (id) float32 dask.array\n", + " end_date (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array\n", + " * id (id) object '010101' '010801' '010802' ... '104804' '120201'\n", + " latitude (id) float32 dask.array\n", + " longitude (id) float32 dask.array\n", + " name (id) object dask.array\n", + " ... ...\n", + " * spatial_agg (spatial_agg) object 'point' 'watershed'\n", + " start_date (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array\n", + " * time (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2025-12-31\n", + " * time_agg (time_agg) object 'mean'\n", + " * timestep (timestep) object 'D'\n", + " * variable (variable) object 'level' 'streamflow'\n", + "Data variables:\n", + " level (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array\n", + " streamflow (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query = {\"datasets\": \"deh\"}\n", + "xds = xd.Query(**query)\n", + "xds.data" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:        (id: 5, time: 20454)\n",
+       "Coordinates: (12/15)\n",
+       "    drainage_area  (id) float32 1.09e+03 647.0 59.8 626.0 1.2e+03\n",
+       "    end_date       (id) datetime64[ns] 2006-10-13 2023-12-17 ... 1996-08-13\n",
+       "  * id             (id) object '020302' '020404' '020502' '020602' '020802'\n",
+       "    latitude       (id) float32 48.77 48.81 48.98 48.98 49.2\n",
+       "    longitude      (id) float32 -64.52 -64.92 -64.43 -64.7 -65.29\n",
+       "    name           (id) object 'Saint' 'York' ... 'Dartmouth' 'Madeleine'\n",
+       "    ...             ...\n",
+       "    spatial_agg    <U9 'watershed'\n",
+       "    start_date     (id) datetime64[ns] 1989-08-12 1980-10-01 ... 1970-01-01\n",
+       "  * time           (time) datetime64[ns] 1970-01-01 1970-01-02 ... 2025-12-31\n",
+       "    time_agg       <U4 'mean'\n",
+       "    timestep       <U1 'D'\n",
+       "    variable       <U10 'streamflow'\n",
+       "Data variables:\n",
+       "    streamflow     (id, time) float32 nan nan nan nan nan ... nan nan nan nan
" + ], + "text/plain": [ + "\n", + "Dimensions: (id: 5, time: 20454)\n", + "Coordinates: (12/15)\n", + " drainage_area (id) float32 1.09e+03 647.0 59.8 626.0 1.2e+03\n", + " end_date (id) datetime64[ns] 2006-10-13 2023-12-17 ... 1996-08-13\n", + " * id (id) object '020302' '020404' '020502' '020602' '020802'\n", + " latitude (id) float32 48.77 48.81 48.98 48.98 49.2\n", + " longitude (id) float32 -64.52 -64.92 -64.43 -64.7 -65.29\n", + " name (id) object 'Saint' 'York' ... 'Dartmouth' 'Madeleine'\n", + " ... ...\n", + " spatial_agg \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:        (data_type: 2, id: 7881, spatial_agg: 2, timestep: 1,\n",
+       "                    time_agg: 1, latitude: 2800, longitude: 4680, time: 59413)\n",
+       "Coordinates: (12/15)\n",
+       "  * data_type      (data_type) <U5 'flow' 'level'\n",
+       "    drainage_area  (id) float64 dask.array<chunksize=(10,), meta=np.ndarray>\n",
+       "    end_date       (id, data_type, spatial_agg, timestep, time_agg) object dask.array<chunksize=(7881, 2, 2, 1, 1), meta=np.ndarray>\n",
+       "  * id             (id) <U7 '01AA002' '01AD001' ... '11AF004' '11AF005'\n",
+       "  * latitude       (latitude) float64 85.0 84.97 84.95 ... 15.07 15.05 15.02\n",
+       "  * longitude      (longitude) float64 -167.0 -167.0 -166.9 ... -50.05 -50.02\n",
+       "    ...             ...\n",
+       "    source         (id) object dask.array<chunksize=(7881,), meta=np.ndarray>\n",
+       "  * spatial_agg    (spatial_agg) object 'point' 'watershed'\n",
+       "    start_date     (id, data_type, spatial_agg, timestep, time_agg) object dask.array<chunksize=(7881, 2, 2, 1, 1), meta=np.ndarray>\n",
+       "  * time           (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2022-08-31\n",
+       "  * time_agg       (time_agg) <U4 'mean'\n",
+       "  * timestep       (timestep) <U3 'day'\n",
+       "Data variables:\n",
+       "    mask           (id, latitude, longitude) float64 dask.array<chunksize=(1, 500, 500), meta=np.ndarray>\n",
+       "    value          (id, time, data_type, spatial_agg, timestep, time_agg) float64 dask.array<chunksize=(10, 59413, 1, 1, 1, 1), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (data_type: 2, id: 7881, spatial_agg: 2, timestep: 1,\n", + " time_agg: 1, latitude: 2800, longitude: 4680, time: 59413)\n", + "Coordinates: (12/15)\n", + " * data_type (data_type) \n", + " end_date (id, data_type, spatial_agg, timestep, time_agg) object dask.array\n", + " * id (id) \n", + " * spatial_agg (spatial_agg) object 'point' 'watershed'\n", + " start_date (id, data_type, spatial_agg, timestep, time_agg) object dask.array\n", + " * time (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2022-08-31\n", + " * time_agg (time_agg) \n", + " value (id, time, data_type, spatial_agg, timestep, time_agg) float64 dask.array" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "query = {\"datasets\": \"hydat\"}\n", "xds = xd.Query(**query)\n", "xds.data" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME: No such keys found for query\n", - "\n", - "# query = {\n", - "# \"datasets\": {\n", - "# \"hydrometric\": {\n", - "# \"variables\": [\"streamflow\", \"t2m_nanmax\", \"t2m_nanmin\", \"tp_nansum\"],\n", - "# \"id\": [\"01010070\", \"01016500\", \"01017290\", \"02*\"],\n", - "# }\n", - "# },\n", - "# }\n", - "# xds = xd.Query(**query)\n", - "# ds_hydro = xds.data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# ds_hydro" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import panel as pn\n", - "\n", - "# id1 = pn.widgets.Select(value=\"01010070\", options=list(xds.data.id.values), name=\"id\")\n", - "# source = pn.widgets.Select(\n", - "# value=\"hydrometric\", options=list(xds.data.source.values), name=\"source\"\n", - "# )\n", - "\n", - "\n", - "# @pn.depends(id1, source)\n", - "# def plot_hydrograph_and_weather(id1, source):\n", - "# da = ds_hydro.sel(id=id1, source=source)\n", - "# dx = da[\"streamflow\"].dropna(\"time\", how=\"any\")\n", - "\n", - "# trace1 = da[\"streamflow\"].hvplot(\n", - "# grid=True,\n", - "# widget_location=\"bottom\",\n", - "# color=\"black\",\n", - "# xlim=(dx.time[0].values, dx.time[-1].values),\n", - "# title=f\"Daily streamflow at location {id1}\",\n", - "# width=750,\n", - "# height=300,\n", - "# )\n", - "# trace2 = da[[\"t2m_nanmax\", \"t2m_nanmin\"]].hvplot(\n", - "# grid=True,\n", - "# widget_location=\"bottom\",\n", - "# color=[\"red\", \"blue\"],\n", - "# xlim=(dx.time[0].values, dx.time[-1].values),\n", - "# title=f\"Daily minimum and maximum temperature at location {id1}\",\n", - "# width=750,\n", - "# height=300,\n", - "# )\n", - "\n", - "# trace3 = da[[\"tp_nansum\"]].hvplot(\n", - "# grid=True,\n", - "# color=[\"turquoise\"],\n", - "# xlim=(dx.time[0].values, dx.time[-1].values),\n", - "# title=f\"Daily precipitation at location {id1}\",\n", - "# width=750,\n", - "# height=300,\n", - "# )\n", - "\n", - "# return pn.Column(trace1, trace2, trace3)\n", - "\n", - "\n", - "# pn.Row(\n", - "# pn.Column(\"## Hydrometric Data Explorer\", id1, source, plot_hydrograph_and_weather)\n", - "# )" - ] } ], "metadata": { + "celltoolbar": "Aucun(e)", "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "xdatasets", "language": "python", - "name": "python3" + "name": "xdatasets" }, "language_info": { "codemirror_mode": { @@ -1354,7 +7621,10 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.7" + }, + "nbsphinx": { + "execute": "never" } }, "nbformat": 4, diff --git a/environment-dev.yml b/environment-dev.yml index 83ecfd5..1c33d0a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -6,6 +6,7 @@ dependencies: - bottleneck - clisops - dask + - dask-geopandas - geopandas - intake - intake-geopandas diff --git a/environment-docs.yml b/environment-docs.yml index 2403b0f..32aac1b 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -5,6 +5,7 @@ channels: dependencies: - python >=3.9,<3.10 - cartopy + - dask-geopandas - distributed >=2.0 - furo - geoviews diff --git a/pyproject.toml b/pyproject.toml index 903356b..04c4985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,9 +45,11 @@ dependencies = [ "cftime>=1.4.1", "clisops>=0.9.2", "dask[array]>=2.6", + "dask-geopandas", "geopandas", "intake", "intake-xarray>=0.6.1", + "intake-geopandas", "ipython", "jsonpickle", "numba", diff --git a/xdatasets/core.py b/xdatasets/core.py index 716cc11..3266b5f 100644 --- a/xdatasets/core.py +++ b/xdatasets/core.py @@ -10,7 +10,12 @@ from .scripting import LOGGING_CONFIG from .utils import cache_catalog from .validations import _validate_space_params -from .workflows import climate_request, hydrometric_request, user_provided_dataset +from .workflows import ( + climate_request, + gis_request, + hydrometric_request, + user_provided_dataset, +) logging.config.dictConfig(LOGGING_CONFIG) logger = logging.getLogger(__name__) @@ -259,12 +264,20 @@ def load_query( try: # Try naively merging datasets into single dataset - ds = xr.merge(dsets) - ds = ds + if type(dsets[0]) == xr.Dataset: + # if more than one dataset, then we add source as a dimension + # so we can merge two or more datasets together + if len(dsets) > 1: + for idx, dset in enumerate(dsets): + for var in dset.keys(): + dset[var] = dset[var].expand_dims("source", axis=-1) + dsets[idx] = dset + ds = xr.merge(dsets) + ds = ds + elif len(dsets) == 1: + ds = dsets[0] except: - logging.warn( - "Couldn't merge datasets so we pass a dictionary of datasets. " - ) + logging.warn("Couldn't merge datasets so we pass a list of datasets. ") # Look into passing a DataTree instead ds = dsets pass @@ -300,6 +313,12 @@ def _process_one_dataset(self, dataset_name, variables, space, time, **kwargs): ds = hydrometric_request( dataset_name, variables, space, time, self.catalog, **kwargs ) + if dataset_category in ["geography"]: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + ds = gis_request( + dataset_name, variables, space, time, self.catalog, **kwargs + ) elif dataset_category in ["user-provided"]: with warnings.catch_warnings(): @@ -308,5 +327,5 @@ def _process_one_dataset(self, dataset_name, variables, space, time, **kwargs): return ds - def bbox_clip(self, ds): - return ds.where(~ds.isnull(), drop=True) + def bbox_clip(self, ds, variable="weights"): + return ds.where(~ds[variable].isnull(), drop=True) diff --git a/xdatasets/spatial.py b/xdatasets/spatial.py index dc90f96..f4ae7f2 100644 --- a/xdatasets/spatial.py +++ b/xdatasets/spatial.py @@ -58,8 +58,8 @@ def aggregate(ds_in, ds_weights): def clip_by_polygon(ds, space, dataset_name): - # We are not using clisops for weighted averages because it is too unstable for now and requires conda environment. - # We use a modified version of the xagg package from which we have removed the xesmf/esmpy dependency + # We are not using clisops for weighted averages because it is too unstable for now. + # We use the xagg package instead. indexer = shape_bbox_indexer(ds, space["geometry"]) ds_copy = ds.isel(indexer).copy() @@ -84,7 +84,10 @@ def clip_by_polygon(ds, space, dataset_name): with HiddenPrints(): ds_weights = create_weights_mask(da.isel(time=0), geom) if space["averaging"] is True: - da = aggregate(da, ds_weights) + da_tmp = aggregate(da, ds_weights) + for var in da_tmp.variables: + da_tmp[var].attrs = da[var].attrs + da = da_tmp else: da = xr.merge([da, ds_weights]) da = da.where(da.weights.notnull(), drop=True) @@ -97,6 +100,7 @@ def clip_by_polygon(ds, space, dataset_name): try: data = data.swap_dims({"geom": space["unique_id"]}) data = data.drop_vars("geom") + data[space["unique_id"]].attrs["cf_role"] = "timeseries_id" if space["unique_id"] not in data.coords: data = data.assign_coords( diff --git a/xdatasets/temporal.py b/xdatasets/temporal.py index 1c2e08e..b16faa4 100644 --- a/xdatasets/temporal.py +++ b/xdatasets/temporal.py @@ -43,17 +43,17 @@ def temporal_aggregation(ds, time, dataset_name, spatial_agg): .reduce(oper, dim="time") .expand_dims( { - "time_agg": [oper.__name__], + # "time_agg": [oper.__name__], "spatial_agg": [spatial_agg], "timestep": [time["timestep"]], } ) ) # da = da.transpose('id','time', 'timestep','time_agg','spatial_agg') - oper_list.append(da) + oper_list.append(da.rename(f"{var}_{oper.__name__}")) # ds_new = ds_new.merge(xr.concat(oper_list, dim='time_agg')) - ds_list.append(xr.concat(oper_list, dim="time_agg")) + ds_list.append(xr.merge(oper_list)) else: try: @@ -64,6 +64,8 @@ def temporal_aggregation(ds, time, dataset_name, spatial_agg): if ds_list: ds_new = xr.merge(ds_list) + # for var in ds_new: + # ds_new[var].attrs = ds[var].attrs # if requested timestep is lower # bfill the timestep and add a warning diff --git a/xdatasets/utils.py b/xdatasets/utils.py index b7b0a89..b60aff0 100644 --- a/xdatasets/utils.py +++ b/xdatasets/utils.py @@ -2,6 +2,7 @@ import sys import tempfile import urllib.request +import warnings from functools import reduce from pathlib import Path @@ -35,37 +36,40 @@ def open_dataset( -------- xarray.open_dataset """ - try: - import intake # noqa: F401 - except ImportError as e: - raise ImportError( - "tutorial.open_dataset depends on intake and intake-xarray to download and manage datasets." - " To proceed please install intake and intake-xarray." - ) from e - - cat = catalog - dataset_info = [ - (category, dataset_name) - for category in cat._entries.keys() - for dataset_name in cat[category]._entries.keys() - if dataset_name == name - ] - - data = reduce(lambda array, index: array[index], dataset_info, cat) - - # add proxy infos - proxies = urllib.request.getproxies() - storage_options = data.storage_options - storage_options["config_kwargs"]["proxies"] = proxies - - if data.describe()["driver"][0] == "geopandasfile": - data = data(storage_options=storage_options).read() - elif data.describe()["driver"][0] == "zarr": - data = data(storage_options=storage_options).to_dask() - else: - raise NotImplementedError( - f"Dataset {name} is not available. Please request further datasets to our github issues pages" - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + try: + import intake # noqa: F401 + except ImportError as e: + raise ImportError( + "tutorial.open_dataset depends on intake and intake-xarray to download and manage datasets." + " To proceed please install intake and intake-xarray." + ) from e + + cat = catalog + dataset_info = [ + (category, dataset_name) + for category in cat._entries.keys() + for dataset_name in cat[category]._entries.keys() + if dataset_name == name + ] + + data = reduce(lambda array, index: array[index], dataset_info, cat) + + # add proxy infos + proxies = urllib.request.getproxies() + storage_options = data.storage_options + storage_options["config_kwargs"]["proxies"] = proxies + + if data.describe()["driver"][0] == "geopandasfile": + data = data(storage_options=storage_options).read() + elif data.describe()["driver"][0] == "zarr": + data = data(storage_options=storage_options).to_dask() + else: + raise NotImplementedError( + f"Dataset {name} is not available. Please request further datasets to our github issues pages" + ) return data diff --git a/xdatasets/workflows.py b/xdatasets/workflows.py index 3bfb0c9..6295f21 100644 --- a/xdatasets/workflows.py +++ b/xdatasets/workflows.py @@ -1,5 +1,6 @@ import fnmatch import itertools +import warnings import pandas as pd import xarray as xr @@ -69,11 +70,44 @@ def climate_request(dataset_name, variables, space, time, catalog): # Add source name to dataset # np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning) ds = ds.assign_coords(source=("source", [dataset_name])) - for var in ds.keys(): - ds[var] = ds[var].expand_dims("source", axis=-1) + + # This is painfully slow if the dataset is large enough and will be commented for now + # for var in ds.keys(): + # ds[var] = ds[var].expand_dims("source", axis=-1) return ds +def gis_request(dataset_name, variables, space, time, catalog, **kwargs): + # This parameter should be set in the catalog + unique_id = catalog["geography"][dataset_name].metadata["unique_id"] + + if any(kwargs): + ds = hydrometric_request( + dataset_name.replace("_polygons", ""), + variables, + space, + time, + catalog, + **kwargs, + ) + filters = {"filters": [(unique_id, "in", tuple(ds.id.values.tolist()))]} + else: + filters = None + warnings.warn( + "No filters fetches the complete dataset (few minutes). Use filters to expedite data retrieval." + ) + + gdf = ( + catalog["geography"][dataset_name](geopandas_kwargs=filters) + .read() + .infer_objects() + .set_index(unique_id) + ) + gdf.index = gdf.index.astype("str") + + return gdf + + def hydrometric_request(dataset_name, variables, space, time, catalog, **kwargs): ds = open_dataset(dataset_name, catalog) From 580a1c357e5c65f7c33ca48150a8300b969bb4ba Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Thu, 21 Dec 2023 18:22:07 -0500 Subject: [PATCH 2/7] remove nb ouput --- docs/notebooks/getting_started.ipynb | 6982 +------------------------- 1 file changed, 45 insertions(+), 6937 deletions(-) diff --git a/docs/notebooks/getting_started.ipynb b/docs/notebooks/getting_started.ipynb index 469a3a1..6ec3d79 100644 --- a/docs/notebooks/getting_started.ipynb +++ b/docs/notebooks/getting_started.ipynb @@ -96,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-10-03T18:40:37.823425184Z", @@ -111,660 +111,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-10-03T18:40:54.073256475Z", "start_time": "2023-10-03T18:40:49.561354059Z" } }, - "outputs": [ - { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " var force = true;\n", - " var py_version = '3.3.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", - " var reloading = false;\n", - " var Bokeh = root.Bokeh;\n", - "\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks;\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - " if (js_modules == null) js_modules = [];\n", - " if (js_exports == null) js_exports = {};\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - "\n", - " if (root._bokeh_is_loading > 0) {\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " if (!reloading) {\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " }\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - " window._bokeh_on_load = on_load\n", - "\n", - " function on_error() {\n", - " console.error(\"failed to load \" + url);\n", - " }\n", - "\n", - " var skip = [];\n", - " if (window.requirejs) {\n", - " window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n", - " require([\"jspanel\"], function(jsPanel) {\n", - "\twindow.jsPanel = jsPanel\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-modal\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-tooltip\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-hint\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-layout\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-contextmenu\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"jspanel-dock\"], function() {\n", - "\ton_load()\n", - " })\n", - " require([\"gridstack\"], function(GridStack) {\n", - "\twindow.GridStack = GridStack\n", - "\ton_load()\n", - " })\n", - " require([\"notyf\"], function() {\n", - "\ton_load()\n", - " })\n", - " root._bokeh_is_loading = css_urls.length + 9;\n", - " } else {\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n", - " }\n", - "\n", - " var existing_stylesheets = []\n", - " var links = document.getElementsByTagName('link')\n", - " for (var i = 0; i < links.length; i++) {\n", - " var link = links[i]\n", - " if (link.href != null) {\n", - "\texisting_stylesheets.push(link.href)\n", - " }\n", - " }\n", - " for (var i = 0; i < css_urls.length; i++) {\n", - " var url = css_urls[i];\n", - " if (existing_stylesheets.indexOf(url) !== -1) {\n", - "\ton_load()\n", - "\tcontinue;\n", - " }\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n", - " for (var i = 0; i < urls.length; i++) {\n", - " skip.push(urls[i])\n", - " }\n", - " } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n", - " for (var i = 0; i < urls.length; i++) {\n", - " skip.push(urls[i])\n", - " }\n", - " } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n", - " var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n", - " for (var i = 0; i < urls.length; i++) {\n", - " skip.push(urls[i])\n", - " }\n", - " } var existing_scripts = []\n", - " var scripts = document.getElementsByTagName('script')\n", - " for (var i = 0; i < scripts.length; i++) {\n", - " var script = scripts[i]\n", - " if (script.src != null) {\n", - "\texisting_scripts.push(script.src)\n", - " }\n", - " }\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n", - "\tif (!window.requirejs) {\n", - "\t on_load();\n", - "\t}\n", - "\tcontinue;\n", - " }\n", - " var element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " for (var i = 0; i < js_modules.length; i++) {\n", - " var url = js_modules[i];\n", - " if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n", - "\tif (!window.requirejs) {\n", - "\t on_load();\n", - "\t}\n", - "\tcontinue;\n", - " }\n", - " var element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.src = url;\n", - " element.type = \"module\";\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " for (const name in js_exports) {\n", - " var url = js_exports[name];\n", - " if (skip.indexOf(url) >= 0 || root[name] != null) {\n", - "\tif (!window.requirejs) {\n", - "\t on_load();\n", - "\t}\n", - "\tcontinue;\n", - " }\n", - " var element = document.createElement('script');\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.type = \"module\";\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " element.textContent = `\n", - " import ${name} from \"${url}\"\n", - " window.${name} = ${name}\n", - " window._bokeh_on_load()\n", - " `\n", - " document.head.appendChild(element);\n", - " }\n", - " if (!js_urls.length && !js_modules.length) {\n", - " on_load()\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.holoviz.org/panel/1.3.6/dist/panel.min.js\"];\n", - " var js_modules = [];\n", - " var js_exports = {};\n", - " var css_urls = [];\n", - " var inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {} // ensure no trailing comma for IE\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if ((root.Bokeh !== undefined) || (force === true)) {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - "\ttry {\n", - " inline_js[i].call(root, root.Bokeh);\n", - "\t} catch(e) {\n", - "\t if (!reloading) {\n", - "\t throw e;\n", - "\t }\n", - "\t}\n", - " }\n", - " // Cache old bokeh versions\n", - " if (Bokeh != undefined && !reloading) {\n", - "\tvar NewBokeh = root.Bokeh;\n", - "\tif (Bokeh.versions === undefined) {\n", - "\t Bokeh.versions = new Map();\n", - "\t}\n", - "\tif (NewBokeh.version !== Bokeh.version) {\n", - "\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n", - "\t}\n", - "\troot.Bokeh = Bokeh;\n", - " }} else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " }\n", - " root._bokeh_is_initializing = false\n", - " }\n", - "\n", - " function load_or_wait() {\n", - " // Implement a backoff loop that tries to ensure we do not load multiple\n", - " // versions of Bokeh and its dependencies at the same time.\n", - " // In recent versions we use the root._bokeh_is_initializing flag\n", - " // to determine whether there is an ongoing attempt to initialize\n", - " // bokeh, however for backward compatibility we also try to ensure\n", - " // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n", - " // before older versions are fully initialized.\n", - " if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n", - " root._bokeh_is_initializing = false;\n", - " root._bokeh_onload_callbacks = undefined;\n", - " console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n", - " load_or_wait();\n", - " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", - " setTimeout(load_or_wait, 100);\n", - " } else {\n", - " root._bokeh_is_initializing = true\n", - " root._bokeh_onload_callbacks = []\n", - " var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n", - " if (!reloading && !bokeh_loaded) {\n", - "\troot.Bokeh = undefined;\n", - " }\n", - " load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n", - "\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - "\trun_inline_js();\n", - " });\n", - " }\n", - " }\n", - " // Give older versions of the autoload script a head-start to ensure\n", - " // they initialize before we start loading newer version.\n", - " setTimeout(load_or_wait, 100)\n", - "}(window));" - ], - "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.3.2'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var reloading = false;\n var Bokeh = root.Bokeh;\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'jspanel': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/jspanel', 'jspanel-modal': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal', 'jspanel-tooltip': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip', 'jspanel-hint': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint', 'jspanel-layout': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout', 'jspanel-contextmenu': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu', 'jspanel-dock': 'https://cdn.jsdelivr.net/npm/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock', 'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@7.2.3/dist/gridstack-all', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'jspanel': {'exports': 'jsPanel'}, 'gridstack': {'exports': 'GridStack'}}});\n require([\"jspanel\"], function(jsPanel) {\n\twindow.jsPanel = jsPanel\n\ton_load()\n })\n require([\"jspanel-modal\"], function() {\n\ton_load()\n })\n require([\"jspanel-tooltip\"], function() {\n\ton_load()\n })\n require([\"jspanel-hint\"], function() {\n\ton_load()\n })\n require([\"jspanel-layout\"], function() {\n\ton_load()\n })\n require([\"jspanel-contextmenu\"], function() {\n\ton_load()\n })\n require([\"jspanel-dock\"], function() {\n\ton_load()\n })\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 9;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } if (((window['jsPanel'] !== undefined) && (!(window['jsPanel'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/jspanel.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/modal/jspanel.modal.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/tooltip/jspanel.tooltip.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/hint/jspanel.hint.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/layout/jspanel.layout.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/contextmenu/jspanel.contextmenu.js', 'https://cdn.holoviz.org/panel/1.3.6/dist/bundled/floatpanel/jspanel4@4.12.0/dist/extensions/dock/jspanel.dock.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/gridstack/gridstack@7.2.3/dist/gridstack-all.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/1.3.6/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.3.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.3.2.min.js\", \"https://cdn.holoviz.org/panel/1.3.6/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n\ttry {\n inline_js[i].call(root, root.Bokeh);\n\t} catch(e) {\n\t if (!reloading) {\n\t throw e;\n\t }\n\t}\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n if (!reloading && !bokeh_loaded) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", - " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", - "}\n", - "\n", - "\n", - " function JupyterCommManager() {\n", - " }\n", - "\n", - " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", - " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", - " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", - " comm_manager.register_target(comm_id, function(comm) {\n", - " comm.on_msg(msg_handler);\n", - " });\n", - " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", - " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", - " comm.onMsg = msg_handler;\n", - " });\n", - " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", - " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", - " var messages = comm.messages[Symbol.asyncIterator]();\n", - " function processIteratorResult(result) {\n", - " var message = result.value;\n", - " console.log(message)\n", - " var content = {data: message.data, comm_id};\n", - " var buffers = []\n", - " for (var buffer of message.buffers || []) {\n", - " buffers.push(new DataView(buffer))\n", - " }\n", - " var metadata = message.metadata || {};\n", - " var msg = {content, buffers, metadata}\n", - " msg_handler(msg);\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " return messages.next().then(processIteratorResult);\n", - " })\n", - " }\n", - " }\n", - "\n", - " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", - " if (comm_id in window.PyViz.comms) {\n", - " return window.PyViz.comms[comm_id];\n", - " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", - " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", - " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", - " if (msg_handler) {\n", - " comm.on_msg(msg_handler);\n", - " }\n", - " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", - " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", - " comm.open();\n", - " if (msg_handler) {\n", - " comm.onMsg = msg_handler;\n", - " }\n", - " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", - " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", - " comm_promise.then((comm) => {\n", - " window.PyViz.comms[comm_id] = comm;\n", - " if (msg_handler) {\n", - " var messages = comm.messages[Symbol.asyncIterator]();\n", - " function processIteratorResult(result) {\n", - " var message = result.value;\n", - " var content = {data: message.data};\n", - " var metadata = message.metadata || {comm_id};\n", - " var msg = {content, metadata}\n", - " msg_handler(msg);\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " }) \n", - " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", - " return comm_promise.then((comm) => {\n", - " comm.send(data, metadata, buffers, disposeOnDone);\n", - " });\n", - " };\n", - " var comm = {\n", - " send: sendClosure\n", - " };\n", - " }\n", - " window.PyViz.comms[comm_id] = comm;\n", - " return comm;\n", - " }\n", - " window.PyViz.comm_manager = new JupyterCommManager();\n", - " \n", - "\n", - "\n", - "var JS_MIME_TYPE = 'application/javascript';\n", - "var HTML_MIME_TYPE = 'text/html';\n", - "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", - "var CLASS_NAME = 'output';\n", - "\n", - "/**\n", - " * Render data to the DOM node\n", - " */\n", - "function render(props, node) {\n", - " var div = document.createElement(\"div\");\n", - " var script = document.createElement(\"script\");\n", - " node.appendChild(div);\n", - " node.appendChild(script);\n", - "}\n", - "\n", - "/**\n", - " * Handle when a new output is added\n", - " */\n", - "function handle_add_output(event, handle) {\n", - " var output_area = handle.output_area;\n", - " var output = handle.output;\n", - " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - " if (id !== undefined) {\n", - " var nchildren = toinsert.length;\n", - " var html_node = toinsert[nchildren-1].children[0];\n", - " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", - " var scripts = [];\n", - " var nodelist = html_node.querySelectorAll(\"script\");\n", - " for (var i in nodelist) {\n", - " if (nodelist.hasOwnProperty(i)) {\n", - " scripts.push(nodelist[i])\n", - " }\n", - " }\n", - "\n", - " scripts.forEach( function (oldScript) {\n", - " var newScript = document.createElement(\"script\");\n", - " var attrs = [];\n", - " var nodemap = oldScript.attributes;\n", - " for (var j in nodemap) {\n", - " if (nodemap.hasOwnProperty(j)) {\n", - " attrs.push(nodemap[j])\n", - " }\n", - " }\n", - " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", - " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", - " oldScript.parentNode.replaceChild(newScript, oldScript);\n", - " });\n", - " if (JS_MIME_TYPE in output.data) {\n", - " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", - " }\n", - " output_area._hv_plot_id = id;\n", - " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", - " window.PyViz.plot_index[id] = Bokeh.index[id];\n", - " } else {\n", - " window.PyViz.plot_index[id] = null;\n", - " }\n", - " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " var bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " var script_attrs = bk_div.children[0].attributes;\n", - " for (var i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - "}\n", - "\n", - "/**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - "function handle_clear_output(event, handle) {\n", - " var id = handle.cell.output_area._hv_plot_id;\n", - " var server_id = handle.cell.output_area._bokeh_server_id;\n", - " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", - " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", - " if (server_id !== null) {\n", - " comm.send({event_type: 'server_delete', 'id': server_id});\n", - " return;\n", - " } else if (comm !== null) {\n", - " comm.send({event_type: 'delete', 'id': id});\n", - " }\n", - " delete PyViz.plot_index[id];\n", - " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", - " var doc = window.Bokeh.index[id].model.document\n", - " doc.clear();\n", - " const i = window.Bokeh.documents.indexOf(doc);\n", - " if (i > -1) {\n", - " window.Bokeh.documents.splice(i, 1);\n", - " }\n", - " }\n", - "}\n", - "\n", - "/**\n", - " * Handle kernel restart event\n", - " */\n", - "function handle_kernel_cleanup(event, handle) {\n", - " delete PyViz.comms[\"hv-extension-comm\"];\n", - " window.PyViz.plot_index = {}\n", - "}\n", - "\n", - "/**\n", - " * Handle update_display_data messages\n", - " */\n", - "function handle_update_output(event, handle) {\n", - " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", - " handle_add_output(event, handle)\n", - "}\n", - "\n", - "function register_renderer(events, OutputArea) {\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " var toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[0]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " events.on('output_added.OutputArea', handle_add_output);\n", - " events.on('output_updated.OutputArea', handle_update_output);\n", - " events.on('clear_output.CodeCell', handle_clear_output);\n", - " events.on('delete.Cell', handle_clear_output);\n", - " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", - "\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " safe: true,\n", - " index: 0\n", - " });\n", - "}\n", - "\n", - "if (window.Jupyter !== undefined) {\n", - " try {\n", - " var events = require('base/js/events');\n", - " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " } catch(err) {\n", - " }\n", - "}\n" - ], - "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.holoviews_exec.v0+json": "", - "text/html": [ - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1002" - } - }, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/clisops/core/regrid.py:42: UserWarning: xarray version >= 2023.03.0 is not supported for regridding operations with cf-time indexed arrays. Please use xarray version < 2023.03.0. For more information, see: https://github.com/pydata/xarray/issues/7794.\n", - " warnings.warn(\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "\n", @@ -795,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -830,32 +184,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Temporal operations: processing tp with era5_reanalysis_single_levels: 100%|██████████████████████████████████████████████| 2/2 [00:04<00:00, 2.46s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 1s, sys: 3.71 s, total: 1min 5s\n", - "Wall time: 1min 5s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "%%time\n", "\n", @@ -881,565 +212,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:      (spatial_agg: 1, timestep: 1, site: 3, time: 23561, source: 1)\n",
-       "Coordinates:\n",
-       "  * spatial_agg  (spatial_agg) object 'point'\n",
-       "  * timestep     (timestep) object 'D'\n",
-       "    latitude     (site) float64 45.5 40.75 25.75\n",
-       "    longitude    (site) float64 -73.5 -74.0 -80.25\n",
-       "  * site         (site) <U8 'Montreal' 'New York' 'Miami'\n",
-       "  * time         (time) datetime64[ns] 1959-01-01 1959-01-02 ... 2023-07-04\n",
-       "  * source       (source) <U29 'era5_reanalysis_single_levels'\n",
-       "Data variables:\n",
-       "    t2m_nanmean  (spatial_agg, timestep, time, site) float32 260.7 ... 302.0\n",
-       "    tp_nansum    (spatial_agg, timestep, time, site) float32 0.0005895 ... 0....\n",
-       "Attributes: (12/30)\n",
-       "    GRIB_NV:                                  0\n",
-       "    GRIB_Nx:                                  1440\n",
-       "    GRIB_Ny:                                  721\n",
-       "    GRIB_cfName:                              unknown\n",
-       "    GRIB_cfVarName:                           t2m\n",
-       "    GRIB_dataType:                            an\n",
-       "    ...                                       ...\n",
-       "    GRIB_totalNumber:                         0\n",
-       "    GRIB_typeOfLevel:                         surface\n",
-       "    GRIB_units:                               K\n",
-       "    long_name:                                2 metre temperature\n",
-       "    standard_name:                            unknown\n",
-       "    units:                                    K
" - ], - "text/plain": [ - "\n", - "Dimensions: (spatial_agg: 1, timestep: 1, site: 3, time: 23561, source: 1)\n", - "Coordinates:\n", - " * spatial_agg (spatial_agg) object 'point'\n", - " * timestep (timestep) object 'D'\n", - " latitude (site) float64 45.5 40.75 25.75\n", - " longitude (site) float64 -73.5 -74.0 -80.25\n", - " * site (site) \n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "Column\n", - " [0] HoloViews(DynamicMap, height=450, sizing_mode='fixed', widget_location='bottom', width=750)\n", - " [1] WidgetBox(align=('center', 'end'))\n", - " [0] Select(margin=(20, 20, 20, 20), name='spatial_agg', options=['point'], value='point', width=250)" - ] - }, - "execution_count": 6, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1004" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "title = f\"Comparison of total precipitation across three cities in North America from \\\n", "{xds.data.time.dt.year.min().values} to {xds.data.time.dt.year.max().values}\"\n", @@ -1462,105 +246,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.holoviews_exec.v0+json": "", - "text/html": [ - "
\n", - "
\n", - "
\n", - "" - ], - "text/plain": [ - "Column\n", - " [0] HoloViews(DynamicMap, height=450, sizing_mode='fixed', widget_location='bottom', width=750)\n", - " [1] WidgetBox(align=('center', 'end'))\n", - " [0] Select(margin=(20, 20, 20, 20), name='spatial_agg', options=['point'], value='point', width=250)" - ] - }, - "execution_count": 7, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1122" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "title = f\"Comparison of 2m temperature across three cities in North America from \\\n", "{xds.data.time.dt.year.min().values} to {xds.data.time.dt.year.max().values}\"\n", @@ -1603,63 +291,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StationSuperficiegeometry
0042102623.479187POLYGON ((-78.57120 46.70742, -78.57112 46.707...
1042103579.479614POLYGON ((-78.49014 46.64514, -78.49010 46.645...
\n", - "
" - ], - "text/plain": [ - " Station Superficie geometry\n", - "0 042102 623.479187 POLYGON ((-78.57120 46.70742, -78.57112 46.707...\n", - "1 042103 579.479614 POLYGON ((-78.49014 46.64514, -78.49010 46.645..." - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import xdatasets as xd\n", "\n", @@ -1686,99 +320,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.holoviews_exec.v0+json": "", - "text/html": [ - "
\n", - "
\n", - "
\n", - "" - ], - "text/plain": [ - ":Overlay\n", - " .WMTS.I :WMTS [Longitude,Latitude]\n", - " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)" - ] - }, - "execution_count": 9, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1247" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gdf.hvplot(\n", " geo=True,\n", @@ -1801,37 +345,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Spatial operations: processing polygon 042102 with era5_reanalysis_single_levels: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 1it [00:00, 5.25it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 2it [00:00, 5.39it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 26 s, sys: 923 ms, total: 26.9 s\n", - "Wall time: 27.7 s\n" - ] - } - ], + "outputs": [], "source": [ "%%time\n", "\n", @@ -1861,520 +377,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:    (Station: 2, time: 40896, latitude: 3, longitude: 2, source: 1)\n",
-       "Coordinates:\n",
-       "  * latitude   (latitude) float64 46.25 46.5 46.75\n",
-       "  * longitude  (longitude) float64 -78.5 -78.25\n",
-       "  * time       (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n",
-       "  * Station    (Station) object '042102' '042103'\n",
-       "  * source     (source) <U29 'era5_reanalysis_single_levels'\n",
-       "Data variables:\n",
-       "    t2m        (Station, time, latitude, longitude) float32 260.3 259.9 ... nan\n",
-       "    tp         (Station, time, latitude, longitude) float32 nan nan ... 0.0 nan\n",
-       "    weights    (Station, latitude, longitude) float64 0.01953 0.1244 ... nan\n",
-       "Attributes:\n",
-       "    Conventions:               CF-1.6\n",
-       "    history:                   2022-11-10 02:03:41 GMT by grib_to_netcdf-2.25...\n",
-       "    pangeo-forge:inputs_hash:  713ae1666a879e6169213b0f03cd75f0b7f2a1769195e6...\n",
-       "    pangeo-forge:recipe_hash:  ad6e205f84d68b1e012856dc70630e06de594c50cfbe7c...\n",
-       "    pangeo-forge:version:      0.9.4
" - ], - "text/plain": [ - "\n", - "Dimensions: (Station: 2, time: 40896, latitude: 3, longitude: 2, source: 1)\n", - "Coordinates:\n", - " * latitude (latitude) float64 46.25 46.5 46.75\n", - " * longitude (longitude) float64 -78.5 -78.25\n", - " * time (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n", - " * Station (Station) object '042102' '042103'\n", - " * source (source) \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:    (time: 40896, latitude: 3, longitude: 2)\n",
-       "Coordinates:\n",
-       "  * latitude   (latitude) float64 46.25 46.5 46.75\n",
-       "  * longitude  (longitude) float64 -78.5 -78.25\n",
-       "  * time       (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n",
-       "    Station    <U6 '042102'\n",
-       "    source     <U29 'era5_reanalysis_single_levels'\n",
-       "Data variables:\n",
-       "    t2m        (time, latitude, longitude) float32 260.3 259.9 ... 288.2 nan\n",
-       "    tp         (time, latitude, longitude) float32 nan nan nan ... 0.0 0.0 nan\n",
-       "    weights    (latitude, longitude) float64 0.01953 0.1244 ... 0.0481 nan\n",
-       "Attributes:\n",
-       "    Conventions:               CF-1.6\n",
-       "    history:                   2022-11-10 02:03:41 GMT by grib_to_netcdf-2.25...\n",
-       "    pangeo-forge:inputs_hash:  713ae1666a879e6169213b0f03cd75f0b7f2a1769195e6...\n",
-       "    pangeo-forge:recipe_hash:  ad6e205f84d68b1e012856dc70630e06de594c50cfbe7c...\n",
-       "    pangeo-forge:version:      0.9.4
" - ], - "text/plain": [ - "\n", - "Dimensions: (time: 40896, latitude: 3, longitude: 2)\n", - "Coordinates:\n", - " * latitude (latitude) float64 46.25 46.5 46.75\n", - " * longitude (longitude) float64 -78.5 -78.25\n", - " * time (time) datetime64[ns] 1959-01-01 ... 1963-08-31T23:00:00\n", - " Station \n", - "
\n", - "\n", - "" - ], - "text/plain": [ - ":Layout\n", - " .Overlay.I :Overlay\n", - " .WMTS.I :WMTS [Longitude,Latitude]\n", - " .Image.I :Image [longitude,latitude] (t2m)\n", - " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)\n", - " .Overlay.II :Overlay\n", - " .WMTS.I :WMTS [Longitude,Latitude]\n", - " .Image.I :Image [longitude,latitude] (weights)\n", - " .Polygons.I :Polygons [Longitude,Latitude] (Station,Superficie)" - ] - }, - "execution_count": 13, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1346" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " (\n", @@ -3039,46 +473,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xdatasets/workflows.py:51: UserWarning: \"start_date\" not found within input date time range. Defaulting to minimum time step in xarray object.\n", - " ds = subset_time(ds, start_date=start_time, end_date=end_time)\n", - "Spatial operations: processing polygon 042102 with era5_reanalysis_single_levels: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 1it [00:00, 4.63it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_reanalysis_single_levels: : 2it [00:00, 4.69it/s]\n", - "Temporal operations: processing tp with era5_reanalysis_single_levels: 100%|██████████████████████████████████████████████| 2/2 [00:07<00:00, 3.69s/it]\n", - "Spatial operations: processing polygon 042102 with era5_land_reanalysis: : 0it [00:00, ?it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_land_reanalysis: : 1it [00:00, 4.24it/s]/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "/home/slanglois/mambaforge/envs/xdatasets/lib/python3.11/site-packages/xarray/core/utils.py:494: FutureWarning: The return type of `Dataset.dims` will be changed to return a set of dimension names in future, in order to be more consistent with `DataArray.dims`. To access a mapping from dimension names to lengths, please use `Dataset.sizes`.\n", - " warnings.warn(\n", - "Spatial operations: processing polygon 042103 with era5_land_reanalysis: : 2it [00:00, 4.30it/s]\n", - "Temporal operations: processing tp with era5_land_reanalysis: 100%|███████████████████████████████████████████████████████| 2/2 [00:07<00:00, 3.79s/it]\n" - ] - } - ], + "outputs": [], "source": [ "datasets = {\n", " \"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]},\n", @@ -3102,596 +499,18 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:      (spatial_agg: 1, timestep: 1, Station: 2, time: 27012,\n",
-       "                  source: 2)\n",
-       "Coordinates:\n",
-       "  * spatial_agg  (spatial_agg) object 'polygon'\n",
-       "  * timestep     (timestep) object 'D'\n",
-       "  * Station      (Station) object '042102' '042103'\n",
-       "  * time         (time) datetime64[ns] 1950-01-01 1950-01-02 ... 2023-12-15\n",
-       "  * source       (source) <U29 'era5_land_reanalysis' 'era5_reanalysis_single...\n",
-       "Data variables:\n",
-       "    t2m_nanmax   (spatial_agg, timestep, Station, time, source) float64 272.8...\n",
-       "    t2m_nanmin   (spatial_agg, timestep, Station, time, source) float64 268.7...\n",
-       "    tp_nansum    (spatial_agg, timestep, Station, time, source) float64 0.000...\n",
-       "Attributes: (12/30)\n",
-       "    GRIB_NV:                                  0\n",
-       "    GRIB_Nx:                                  1440\n",
-       "    GRIB_Ny:                                  721\n",
-       "    GRIB_cfName:                              unknown\n",
-       "    GRIB_cfVarName:                           t2m\n",
-       "    GRIB_dataType:                            an\n",
-       "    ...                                       ...\n",
-       "    GRIB_totalNumber:                         0\n",
-       "    GRIB_typeOfLevel:                         surface\n",
-       "    GRIB_units:                               K\n",
-       "    long_name:                                2 metre temperature\n",
-       "    standard_name:                            unknown\n",
-       "    units:                                    K
" - ], - "text/plain": [ - "\n", - "Dimensions: (spatial_agg: 1, timestep: 1, Station: 2, time: 27012,\n", - " source: 2)\n", - "Coordinates:\n", - " * spatial_agg (spatial_agg) object 'polygon'\n", - " * timestep (timestep) object 'D'\n", - " * Station (Station) object '042102' '042103'\n", - " * time (time) datetime64[ns] 1950-01-01 1950-01-02 ... 2023-12-15\n", - " * source (source) \n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "Column\n", - " [0] HoloViews(DynamicMap, height=400, sizing_mode='fixed', widget_location='bottom', width=750)\n", - " [1] WidgetBox(align=('center', 'end'))\n", - " [0] Select(margin=(20, 20, 5, 20), name='Station', options=['042102', '042103'], value='042102', width=250)\n", - " [1] Select(margin=(5, 20, 20, 20), name='source', options=['era5_land_reanalysis', ...], value='era5_land_reanalysis', width=250)" - ] - }, - "execution_count": 22, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1729" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " xds.data[[\"t2m_nanmax\", \"t2m_nanmin\"]]\n", @@ -3716,106 +535,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": {}, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.holoviews_exec.v0+json": "", - "text/html": [ - "
\n", - "
\n", - "
\n", - "" - ], - "text/plain": [ - "Column\n", - " [0] HoloViews(DynamicMap, height=400, sizing_mode='fixed', widget_location='bottom', width=750)\n", - " [1] WidgetBox(align=('center', 'end'))\n", - " [0] Select(margin=(20, 20, 5, 20), name='Station', options=['042102', '042103'], value='042102', width=250)\n", - " [1] Select(margin=(5, 20, 20, 20), name='source', options=['era5_land_reanalysis', ...], value='era5_land_reanalysis', width=250)" - ] - }, - "execution_count": 24, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p1915" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " xds.data[[\"tp_nansum\"]]\n", @@ -3845,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -3868,474 +590,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:    (time: 105192, latitude: 9, longitude: 11, source: 1)\n",
-       "Coordinates:\n",
-       "  * latitude   (latitude) float64 46.9 46.8 46.7 46.6 46.5 46.4 46.3 46.2 46.1\n",
-       "  * longitude  (longitude) float64 -78.9 -78.8 -78.7 -78.6 ... -78.1 -78.0 -77.9\n",
-       "  * time       (time) datetime64[ns] 1959-01-01 ... 1970-12-31T23:00:00\n",
-       "  * source     (source) <U20 'era5_land_reanalysis'\n",
-       "Data variables:\n",
-       "    tp         (time, latitude, longitude) float32 1.08e-07 1.08e-07 ... 0.0 0.0\n",
-       "Attributes:\n",
-       "    pangeo-forge:inputs_hash:  6edb2d7ac9d207d7933d56a7c36d09d12b07e35dfdbf89...\n",
-       "    pangeo-forge:recipe_hash:  46fd6fcf6c2c3cd36d2b10bb6c1196bd26f98c5c504d70...\n",
-       "    pangeo-forge:version:      0.9.4\n",
-       "    timezone:                  America/Montreal
" - ], - "text/plain": [ - "\n", - "Dimensions: (time: 105192, latitude: 9, longitude: 11, source: 1)\n", - "Coordinates:\n", - " * latitude (latitude) float64 46.9 46.8 46.7 46.6 46.5 46.4 46.3 46.2 46.1\n", - " * longitude (longitude) float64 -78.9 -78.8 -78.7 -78.6 ... -78.1 -78.0 -77.9\n", - " * time (time) datetime64[ns] 1959-01-01 ... 1970-12-31T23:00:00\n", - " * source (source) \n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "Column\n", - " [0] HoloViews(HoloMap, height=450, sizing_mode='fixed', widget_location='bottom', widget_type='scrubber', width=750)\n", - " [1] WidgetBox(align=('center', 'end'))\n", - " [0] Player(end=23, width=550)" - ] - }, - "execution_count": 29, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "p2014" - } - }, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "da = xds.data.tp.isel(time=slice(indexer - 24, indexer))\n", "da = da.where(da > 0.0001, drop=True)\n", @@ -4514,1343 +664,11 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": { "tags": [] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:        (id: 744, variable: 2, spatial_agg: 2, timestep: 1,\n",
-       "                    time_agg: 1, source: 1, time: 60631)\n",
-       "Coordinates: (12/15)\n",
-       "    drainage_area  (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
-       "    end_date       (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array<chunksize=(2, 744, 2, 1, 1, 1), meta=np.ndarray>\n",
-       "  * id             (id) object '010101' '010801' '010802' ... '104804' '120201'\n",
-       "    latitude       (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
-       "    longitude      (id) float32 dask.array<chunksize=(744,), meta=np.ndarray>\n",
-       "    name           (id) object dask.array<chunksize=(744,), meta=np.ndarray>\n",
-       "    ...             ...\n",
-       "  * spatial_agg    (spatial_agg) object 'point' 'watershed'\n",
-       "    start_date     (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array<chunksize=(2, 744, 2, 1, 1, 1), meta=np.ndarray>\n",
-       "  * time           (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2025-12-31\n",
-       "  * time_agg       (time_agg) object 'mean'\n",
-       "  * timestep       (timestep) object 'D'\n",
-       "  * variable       (variable) object 'level' 'streamflow'\n",
-       "Data variables:\n",
-       "    level          (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array<chunksize=(1, 60631, 1, 1, 1, 1, 1), meta=np.ndarray>\n",
-       "    streamflow     (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array<chunksize=(1, 60631, 1, 1, 1, 1, 1), meta=np.ndarray>
" - ], - "text/plain": [ - "\n", - "Dimensions: (id: 744, variable: 2, spatial_agg: 2, timestep: 1,\n", - " time_agg: 1, source: 1, time: 60631)\n", - "Coordinates: (12/15)\n", - " drainage_area (id) float32 dask.array\n", - " end_date (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array\n", - " * id (id) object '010101' '010801' '010802' ... '104804' '120201'\n", - " latitude (id) float32 dask.array\n", - " longitude (id) float32 dask.array\n", - " name (id) object dask.array\n", - " ... ...\n", - " * spatial_agg (spatial_agg) object 'point' 'watershed'\n", - " start_date (variable, id, spatial_agg, timestep, time_agg, source) datetime64[ns] dask.array\n", - " * time (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2025-12-31\n", - " * time_agg (time_agg) object 'mean'\n", - " * timestep (timestep) object 'D'\n", - " * variable (variable) object 'level' 'streamflow'\n", - "Data variables:\n", - " level (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array\n", - " streamflow (id, time, variable, spatial_agg, timestep, time_agg, source) float32 dask.array" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "query = {\"datasets\": \"deh\"}\n", "xds = xd.Query(**query)\n", @@ -5859,443 +677,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:        (id: 5, time: 20454)\n",
-       "Coordinates: (12/15)\n",
-       "    drainage_area  (id) float32 1.09e+03 647.0 59.8 626.0 1.2e+03\n",
-       "    end_date       (id) datetime64[ns] 2006-10-13 2023-12-17 ... 1996-08-13\n",
-       "  * id             (id) object '020302' '020404' '020502' '020602' '020802'\n",
-       "    latitude       (id) float32 48.77 48.81 48.98 48.98 49.2\n",
-       "    longitude      (id) float32 -64.52 -64.92 -64.43 -64.7 -65.29\n",
-       "    name           (id) object 'Saint' 'York' ... 'Dartmouth' 'Madeleine'\n",
-       "    ...             ...\n",
-       "    spatial_agg    <U9 'watershed'\n",
-       "    start_date     (id) datetime64[ns] 1989-08-12 1980-10-01 ... 1970-01-01\n",
-       "  * time           (time) datetime64[ns] 1970-01-01 1970-01-02 ... 2025-12-31\n",
-       "    time_agg       <U4 'mean'\n",
-       "    timestep       <U1 'D'\n",
-       "    variable       <U10 'streamflow'\n",
-       "Data variables:\n",
-       "    streamflow     (id, time) float32 nan nan nan nan nan ... nan nan nan nan
" - ], - "text/plain": [ - "\n", - "Dimensions: (id: 5, time: 20454)\n", - "Coordinates: (12/15)\n", - " drainage_area (id) float32 1.09e+03 647.0 59.8 626.0 1.2e+03\n", - " end_date (id) datetime64[ns] 2006-10-13 2023-12-17 ... 1996-08-13\n", - " * id (id) object '020302' '020404' '020502' '020602' '020802'\n", - " latitude (id) float32 48.77 48.81 48.98 48.98 49.2\n", - " longitude (id) float32 -64.52 -64.92 -64.43 -64.7 -65.29\n", - " name (id) object 'Saint' 'York' ... 'Dartmouth' 'Madeleine'\n", - " ... ...\n", - " spatial_agg \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:        (data_type: 2, id: 7881, spatial_agg: 2, timestep: 1,\n",
-       "                    time_agg: 1, latitude: 2800, longitude: 4680, time: 59413)\n",
-       "Coordinates: (12/15)\n",
-       "  * data_type      (data_type) <U5 'flow' 'level'\n",
-       "    drainage_area  (id) float64 dask.array<chunksize=(10,), meta=np.ndarray>\n",
-       "    end_date       (id, data_type, spatial_agg, timestep, time_agg) object dask.array<chunksize=(7881, 2, 2, 1, 1), meta=np.ndarray>\n",
-       "  * id             (id) <U7 '01AA002' '01AD001' ... '11AF004' '11AF005'\n",
-       "  * latitude       (latitude) float64 85.0 84.97 84.95 ... 15.07 15.05 15.02\n",
-       "  * longitude      (longitude) float64 -167.0 -167.0 -166.9 ... -50.05 -50.02\n",
-       "    ...             ...\n",
-       "    source         (id) object dask.array<chunksize=(7881,), meta=np.ndarray>\n",
-       "  * spatial_agg    (spatial_agg) object 'point' 'watershed'\n",
-       "    start_date     (id, data_type, spatial_agg, timestep, time_agg) object dask.array<chunksize=(7881, 2, 2, 1, 1), meta=np.ndarray>\n",
-       "  * time           (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2022-08-31\n",
-       "  * time_agg       (time_agg) <U4 'mean'\n",
-       "  * timestep       (timestep) <U3 'day'\n",
-       "Data variables:\n",
-       "    mask           (id, latitude, longitude) float64 dask.array<chunksize=(1, 500, 500), meta=np.ndarray>\n",
-       "    value          (id, time, data_type, spatial_agg, timestep, time_agg) float64 dask.array<chunksize=(10, 59413, 1, 1, 1, 1), meta=np.ndarray>
" - ], - "text/plain": [ - "\n", - "Dimensions: (data_type: 2, id: 7881, spatial_agg: 2, timestep: 1,\n", - " time_agg: 1, latitude: 2800, longitude: 4680, time: 59413)\n", - "Coordinates: (12/15)\n", - " * data_type (data_type) \n", - " end_date (id, data_type, spatial_agg, timestep, time_agg) object dask.array\n", - " * id (id) \n", - " * spatial_agg (spatial_agg) object 'point' 'watershed'\n", - " start_date (id, data_type, spatial_agg, timestep, time_agg) object dask.array\n", - " * time (time) datetime64[ns] 1860-01-01 1860-01-02 ... 2022-08-31\n", - " * time_agg (time_agg) \n", - " value (id, time, data_type, spatial_agg, timestep, time_agg) float64 dask.array" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "query = {\"datasets\": \"hydat\"}\n", "xds = xd.Query(**query)\n", @@ -7622,9 +733,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.7" - }, - "nbsphinx": { - "execute": "never" } }, "nbformat": 4, From 7b74dbc0a9df58ba92f90d112462c7673c8fe742 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Tue, 9 Jan 2024 17:36:12 -0500 Subject: [PATCH 3/7] add cf_role and remove autoreload in notebooks --- docs/notebooks/getting_started.ipynb | 17 +---------------- xdatasets/spatial.py | 2 +- xdatasets/workflows.py | 4 ++-- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/docs/notebooks/getting_started.ipynb b/docs/notebooks/getting_started.ipynb index 6ec3d79..a1e983e 100644 --- a/docs/notebooks/getting_started.ipynb +++ b/docs/notebooks/getting_started.ipynb @@ -61,7 +61,7 @@ " \"datasets\": {\"era5_reanalysis_single_levels\": {'variables': [\"t2m\", \"tp\"]}},\n", " \"space\": {\n", " \"clip\": \"polygon\", # bbox, point or polygon\n", - " \"aggregation\": True, # spatial average of the variables within each polygon\n", + " \"averaging\": True, # spatial average of the variables within each polygon\n", " \"geometry\": gdf,\n", " \"unique_id\": \"Station\" # unique column name in geodataframe\n", " },\n", @@ -94,21 +94,6 @@ "In order to use `xdatasets`, you must import at least `xdatasets`, `pandas`, `geopandas`, and `numpy`. Additionally, we import `pathlib` to interact with files." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-10-03T18:40:37.823425184Z", - "start_time": "2023-10-03T18:40:37.809132786Z" - } - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/xdatasets/spatial.py b/xdatasets/spatial.py index f4ae7f2..9359dd2 100644 --- a/xdatasets/spatial.py +++ b/xdatasets/spatial.py @@ -100,7 +100,6 @@ def clip_by_polygon(ds, space, dataset_name): try: data = data.swap_dims({"geom": space["unique_id"]}) data = data.drop_vars("geom") - data[space["unique_id"]].attrs["cf_role"] = "timeseries_id" if space["unique_id"] not in data.coords: data = data.assign_coords( @@ -111,6 +110,7 @@ def clip_by_polygon(ds, space, dataset_name): ) } ) + data[space["unique_id"]].attrs["cf_role"] = "timeseries_id" except KeyError: pass return data diff --git a/xdatasets/workflows.py b/xdatasets/workflows.py index 6295f21..dc8bec3 100644 --- a/xdatasets/workflows.py +++ b/xdatasets/workflows.py @@ -239,7 +239,7 @@ def user_provided_dataset(dataset_name, variables, space, time, ds): # Add source name to dataset # np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning) ds = ds.assign_coords(source=("source", [dataset_name])) - for var in ds.keys(): - ds[var] = ds[var].expand_dims("source", axis=-1) + # for var in ds.keys(): + # ds[var] = ds[var].expand_dims("source", axis=-1) return ds From 8d242adc2c063018464637910dfdcbda250c3fb3 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Wed, 10 Jan 2024 12:36:40 -0500 Subject: [PATCH 4/7] update changes --- CHANGES.rst | 6 ++++++ docs/notebooks/getting_started.ipynb | 2 +- xdatasets/core.py | 5 +++-- xdatasets/spatial.py | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4135287..6e87135 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,12 @@ Changelog ========= +v0.3.2 (2024-01-10) +------------------- + +* Update documentation. (:pull:`42`) +* Added a functionality to extract geometries to a `geopandas.GeoDataFrame` format. (:pull:`42`) + v0.3.1 (2023-12-01) ------------------- diff --git a/docs/notebooks/getting_started.ipynb b/docs/notebooks/getting_started.ipynb index a1e983e..aaa2a01 100644 --- a/docs/notebooks/getting_started.ipynb +++ b/docs/notebooks/getting_started.ipynb @@ -437,7 +437,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The two plots depicted above show the 2m temperature for each pixel that intersects with the polygon from Station `042102` and the corresponding weights to be applied to each pixel. In the lower plot, it is apparent that the majority of the polygon is situated in the central pixels, which results in those pixelx having a weight of approximately 80%. It is evident that the two lower and the upper pixels have much less intersection with the polygon, which results in their respective weights being smaller (hover on the plot to verify the weights).\n", + "The two plots depicted above show the 2m temperature for each pixel that intersects with the polygon from Station `042102` and the corresponding weights to be applied to each pixel. In the lower plot, it is apparent that the majority of the polygon is situated in the central pixels, which results in those pixels having a weight of approximately 80%. It is evident that the two lower and the upper pixels have much less intersection with the polygon, which results in their respective weights being smaller (hover on the plot to verify the weights).\n", "\n", "In various libraries, either all pixels that intersect with the geometries are kept, or only pixels with centers within the polygon are retained. However, as shown in the previous example, utilizing such methods can introduce significant biases in the final calculations." ] diff --git a/xdatasets/core.py b/xdatasets/core.py index 3266b5f..122fde5 100644 --- a/xdatasets/core.py +++ b/xdatasets/core.py @@ -78,7 +78,7 @@ class Query: ... "space": {"clip": "point", "geometry": sites}, ... "time": { ... "timestep": "D", - ... "aggregation": {"tp": np.nansum, "t2m": np.nanmean}, + ... "averaging": {"tp": np.nansum, "t2m": np.nanmean}, ... "start": "1950-01-01", ... "end": "1955-12-31", ... "timezone": "America/Montreal", @@ -264,6 +264,8 @@ def load_query( try: # Try naively merging datasets into single dataset + ds = None + if type(dsets[0]) == xr.Dataset: # if more than one dataset, then we add source as a dimension # so we can merge two or more datasets together @@ -273,7 +275,6 @@ def load_query( dset[var] = dset[var].expand_dims("source", axis=-1) dsets[idx] = dset ds = xr.merge(dsets) - ds = ds elif len(dsets) == 1: ds = dsets[0] except: diff --git a/xdatasets/spatial.py b/xdatasets/spatial.py index 9359dd2..cc19b7d 100644 --- a/xdatasets/spatial.py +++ b/xdatasets/spatial.py @@ -4,7 +4,7 @@ import pandas as pd import xarray as xr from clisops.core.subset import shape_bbox_indexer, subset_gridpoint -from tqdm import tqdm +from tqdm.auto import tqdm from .utils import HiddenPrints @@ -65,7 +65,7 @@ def clip_by_polygon(ds, space, dataset_name): ds_copy = ds.isel(indexer).copy() arrays = [] - pbar = tqdm(space["geometry"].iterrows()) + pbar = tqdm(space["geometry"].iterrows(), position=0, leave=True) for idx, row in pbar: item = ( row[space["unique_id"]] From a7e77fbbadba569ce4d0788f971b1a0a64823413 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Wed, 10 Jan 2024 12:37:02 -0500 Subject: [PATCH 5/7] add cf_role attribute --- xdatasets/spatial.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xdatasets/spatial.py b/xdatasets/spatial.py index cc19b7d..25d71f6 100644 --- a/xdatasets/spatial.py +++ b/xdatasets/spatial.py @@ -128,4 +128,7 @@ def clip_by_point(ds, space, dataset_name): data = data.rename({"lat": "latitude", "lon": "longitude"}) data = data.assign_coords({"site": ("site", list(space["geometry"].keys()))}) + + data["site"].attrs["cf_role"] = "timeseries_id" + return data From 8e92dd5a1626e4fa59367f5a72da0954cb1d4081 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Wed, 10 Jan 2024 13:51:48 -0500 Subject: [PATCH 6/7] remove magic command %%time in notebooks --- docs/notebooks/getting_started.ipynb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/notebooks/getting_started.ipynb b/docs/notebooks/getting_started.ipynb index aaa2a01..9365591 100644 --- a/docs/notebooks/getting_started.ipynb +++ b/docs/notebooks/getting_started.ipynb @@ -173,8 +173,6 @@ "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "\n", "query = {\n", " \"datasets\": \"era5_reanalysis_single_levels\",\n", " \"space\": {\"clip\": \"point\", \"geometry\": sites}, # bbox, point or polygon\n", @@ -334,8 +332,6 @@ "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "\n", "query = {\n", " \"datasets\": {\"era5_reanalysis_single_levels\": {\"variables\": [\"t2m\", \"tp\"]}},\n", " \"space\": {\n", From d3d0a0c5ceae1981bf1d58493c6387245c6eb819 Mon Sep 17 00:00:00 2001 From: sebastienlanglois Date: Wed, 10 Jan 2024 14:25:45 -0500 Subject: [PATCH 7/7] force nbconvert version https://github.com/jupyter/nbconvert/issues/2092 --- environment-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-docs.yml b/environment-docs.yml index 32aac1b..cadaec2 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -17,6 +17,7 @@ dependencies: - ipython - jupyter_client - matplotlib + - nbconvert<7.14 - nbsphinx - nc-time-axis - netCDF4