Skip to content

Commit

Permalink
fix JSON seriealizable for CWL output file
Browse files Browse the repository at this point in the history
  • Loading branch information
@fmigneault
fmigneault committed Sep 27, 2023
1 parent 4d8e72a commit 78b081f
Showing 1 changed file with 27 additions and 32 deletions.
59 changes: 27 additions & 32 deletions tests/functional/test_ncml2stac.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,14 +106,13 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 16,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2023-09-27 22:13:03.244828 [INFO] :: PYESSV :: Loading vocabularies from /home/francis/.esdoc/pyessv-archive ... please wait\n",
"2023-09-27 22:13:03.656407 [INFO] :: PYESSV :: Loading vocabularies from /home/francis/.esdoc/pyessv-archive ... please wait\n",
"2023-09-27 22:47:33.681584 [INFO] :: PYESSV :: Loading vocabularies from /home/francis/.esdoc/pyessv-archive ... please wait\n",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
"<ncml:netcdf xmlns:ncml=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\" location=\"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/xclim/cmip6/sic_SImon_CCCma-CanESM5_ssp245_r13i1p2f1_2020.nc\">\n",
" <ncml:attribute name=\"CCCma_model_hash\" value=\"fc4bb7db954c862d023b546e19aec6c588bc0552\" />\n",
Expand Down Expand Up @@ -301,13 +300,18 @@
],
"source": [
"import json\n",
"import pystac\n",
"from pystac.extensions import datacube\n",
"import requests\n",
"import tempfile\n",
"from datetime import datetime, date\n",
"from dateutil.parser import parse as dt_parse\n",
"from enum import Enum\n",
"\n",
"import numpy as np\n",
"import requests\n",
"import xncml\n",
"import pystac\n",
"from pystac.extensions import datacube\n",
"from pydantic.networks import Url\n",
"from shapely.geometry.polygon import Polygon\n",
"from dateutil.parser import parse as dt_parse\n",
"\n",
"import pyessv\n",
"pyessv.init()\n",
Expand All @@ -330,8 +334,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-09-27T22:13:04.191448522Z",
"start_time": "2023-09-27T22:13:02.465029823Z"
"end_time": "2023-09-27T22:47:34.154049178Z",
"start_time": "2023-09-27T22:47:33.700947842Z"
}
},
"id": "4fc2f66493dc56c5"
Expand Down Expand Up @@ -481,37 +485,28 @@
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [
{
"ename": "TypeError",
"evalue": "Object of type datetime is not JSON serializable",
"output_type": "error",
"traceback": [
"\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
"\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)",
"Cell \u001B[0;32mIn[7], line 2\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m tempfile\u001B[38;5;241m.\u001B[39mNamedTemporaryFile(mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mw\u001B[39m\u001B[38;5;124m\"\u001B[39m, encoding\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mutf-8\u001B[39m\u001B[38;5;124m\"\u001B[39m, delete\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m out_file:\n\u001B[0;32m----> 2\u001B[0m \u001B[43mjson\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdump\u001B[49m\u001B[43m(\u001B[49m\u001B[43mstac_item\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mout_file\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 4\u001B[0m output: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mCWLFilePathOutput\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m=\u001B[39m out_file\u001B[38;5;241m.\u001B[39mname\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/__init__.py:179\u001B[0m, in \u001B[0;36mdump\u001B[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001B[0m\n\u001B[1;32m 173\u001B[0m iterable \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mcls\u001B[39m(skipkeys\u001B[38;5;241m=\u001B[39mskipkeys, ensure_ascii\u001B[38;5;241m=\u001B[39mensure_ascii,\n\u001B[1;32m 174\u001B[0m check_circular\u001B[38;5;241m=\u001B[39mcheck_circular, allow_nan\u001B[38;5;241m=\u001B[39mallow_nan, indent\u001B[38;5;241m=\u001B[39mindent,\n\u001B[1;32m 175\u001B[0m separators\u001B[38;5;241m=\u001B[39mseparators,\n\u001B[1;32m 176\u001B[0m default\u001B[38;5;241m=\u001B[39mdefault, sort_keys\u001B[38;5;241m=\u001B[39msort_keys, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkw)\u001B[38;5;241m.\u001B[39miterencode(obj)\n\u001B[1;32m 177\u001B[0m \u001B[38;5;66;03m# could accelerate with writelines in some versions of Python, at\u001B[39;00m\n\u001B[1;32m 178\u001B[0m \u001B[38;5;66;03m# a debuggability cost\u001B[39;00m\n\u001B[0;32m--> 179\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m chunk \u001B[38;5;129;01min\u001B[39;00m iterable:\n\u001B[1;32m 180\u001B[0m fp\u001B[38;5;241m.\u001B[39mwrite(chunk)\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/encoder.py:431\u001B[0m, in \u001B[0;36m_make_iterencode.<locals>._iterencode\u001B[0;34m(o, _current_indent_level)\u001B[0m\n\u001B[1;32m 429\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m _iterencode_list(o, _current_indent_level)\n\u001B[1;32m 430\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(o, \u001B[38;5;28mdict\u001B[39m):\n\u001B[0;32m--> 431\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m _iterencode_dict(o, _current_indent_level)\n\u001B[1;32m 432\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 433\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m markers \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/encoder.py:405\u001B[0m, in \u001B[0;36m_make_iterencode.<locals>._iterencode_dict\u001B[0;34m(dct, _current_indent_level)\u001B[0m\n\u001B[1;32m 403\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 404\u001B[0m chunks \u001B[38;5;241m=\u001B[39m _iterencode(value, _current_indent_level)\n\u001B[0;32m--> 405\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m chunks\n\u001B[1;32m 406\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m newline_indent \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 407\u001B[0m _current_indent_level \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/encoder.py:405\u001B[0m, in \u001B[0;36m_make_iterencode.<locals>._iterencode_dict\u001B[0;34m(dct, _current_indent_level)\u001B[0m\n\u001B[1;32m 403\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 404\u001B[0m chunks \u001B[38;5;241m=\u001B[39m _iterencode(value, _current_indent_level)\n\u001B[0;32m--> 405\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m chunks\n\u001B[1;32m 406\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m newline_indent \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 407\u001B[0m _current_indent_level \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/encoder.py:438\u001B[0m, in \u001B[0;36m_make_iterencode.<locals>._iterencode\u001B[0;34m(o, _current_indent_level)\u001B[0m\n\u001B[1;32m 436\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mCircular reference detected\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 437\u001B[0m markers[markerid] \u001B[38;5;241m=\u001B[39m o\n\u001B[0;32m--> 438\u001B[0m o \u001B[38;5;241m=\u001B[39m \u001B[43m_default\u001B[49m\u001B[43m(\u001B[49m\u001B[43mo\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 439\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m _iterencode(o, _current_indent_level)\n\u001B[1;32m 440\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m markers \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
"File \u001B[0;32m~/dev/conda/envs/weaver/lib/python3.10/json/encoder.py:179\u001B[0m, in \u001B[0;36mJSONEncoder.default\u001B[0;34m(self, o)\u001B[0m\n\u001B[1;32m 160\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mdefault\u001B[39m(\u001B[38;5;28mself\u001B[39m, o):\n\u001B[1;32m 161\u001B[0m \u001B[38;5;124;03m\"\"\"Implement this method in a subclass such that it returns\u001B[39;00m\n\u001B[1;32m 162\u001B[0m \u001B[38;5;124;03m a serializable object for ``o``, or calls the base implementation\u001B[39;00m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;124;03m (to raise a ``TypeError``).\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 177\u001B[0m \n\u001B[1;32m 178\u001B[0m \u001B[38;5;124;03m \"\"\"\u001B[39;00m\n\u001B[0;32m--> 179\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mObject of type \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mo\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__name__\u001B[39m\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m \u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m 180\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mis not JSON serializable\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
"\u001B[0;31mTypeError\u001B[0m: Object of type datetime is not JSON serializable"
]
}
],
"execution_count": 17,
"outputs": [],
"source": [
"def json_encode(obj):\n",
" if isinstance(obj, (np.ndarray, np.number)):\n",
" return obj.tolist()\n",
" if isinstance(obj, (Url, Enum)):\n",
" return str(obj)\n",
" if isinstance(obj, (datetime, date)):\n",
" return obj.isoformat()\n",
" raise TypeError(f\"Type {type(obj)} not serializable\")\n",
"\n",
"with tempfile.NamedTemporaryFile(mode=\"w\", encoding=\"utf-8\", delete=False) as out_file:\n",
" json.dump(stac_item, out_file)\n",
" json.dump(stac_item, out_file, default=json_encode)\n",
"\n",
"output: \"CWLFilePathOutput\" = out_file.name"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-09-27T22:13:04.497948452Z",
"start_time": "2023-09-27T22:13:04.305098948Z"
"end_time": "2023-09-27T22:47:38.832969763Z",
"start_time": "2023-09-27T22:47:38.710714517Z"
}
},
"id": "e4fa98fcad8b5556"
Expand Down

0 comments on commit 78b081f

Please sign in to comment.