From b2c8bc2b5c48042fa9aff759b7942b926facf8cf Mon Sep 17 00:00:00 2001 From: benjamin Date: Mon, 19 Jul 2021 23:05:08 +0200 Subject: [PATCH] Use pdbufr to read DWD radar data in bufr format --- .github/workflows/install.sh | 2 +- .github/workflows/tests.yml | 13 ++- CHANGELOG.rst | 1 + poetry.lock | 103 +++++++++++++----- pyproject.toml | 28 ++--- tests/__init__.py | 4 + tests/provider/dwd/radar/test_api_historic.py | 39 +++++++ wetterdienst/provider/dwd/radar/access.py | 32 +++++- wetterdienst/provider/dwd/radar/api.py | 5 + 9 files changed, 177 insertions(+), 50 deletions(-) diff --git a/.github/workflows/install.sh b/.github/workflows/install.sh index 6d6bd537b..b6d8b2028 100755 --- a/.github/workflows/install.sh +++ b/.github/workflows/install.sh @@ -10,7 +10,7 @@ fi echo "Installing package and requirements for ${flavor}" if [[ "${flavor}" = "testing" ]]; then - poetry install --no-interaction --extras=sql --extras=export --extras=restapi --extras=explorer + poetry install --no-interaction --extras=sql --extras=export --extras=restapi --extras=explorer --extras=bufr elif [[ "${flavor}" = "docs" ]]; then poetry install --no-interaction --extras=docs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b1a4c1d46..06ee0e582 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,14 +53,15 @@ jobs: path: ${{ steps.poetry-cache-dir.outputs.dir }} key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}-${{ env.CACHE_NUMBER }} + - name: Install eccodes (Mac only) + run: | + if [ "$RUNNER_OS" == "macOS" ]; then + brew install eccodes + fi + shell: bash + - name: Install dependencies - #run: poetry install --no-interaction --no-root --extras=sql --extras=export --extras=restapi --extras=explorer run: .github/workflows/install.sh testing - #if: steps.poetry-cache-flag.outputs.cache-hit != 'true' - - #- name: Install library - # run: poetry install --no-interaction - # #if: steps.poetry-cache-flag.outputs.cache-hit != 'true' - name: Test run: poetry run poe test diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 31d4a0a61..02b5aaf81 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Added - Enable selecting a parameter precisely from a dataset by passing a tuple like [("precipitation_height", "kl")] or [("precipitation_height", "precipitation_more")], or for cli/restapi use "precipitation_height/kl" +- Allow parsing DWD radar data in bufr format to a pandas DataFrame Changed ======= diff --git a/poetry.lock b/poetry.lock index 7793308ed..e10998623 100644 --- a/poetry.lock +++ b/poetry.lock @@ -16,7 +16,7 @@ python-versions = "*" [[package]] name = "anyio" -version = "3.2.1" +version = "3.3.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" category = "dev" optional = false @@ -85,7 +85,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "20.3.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -146,7 +146,7 @@ lxml = ["lxml"] [[package]] name = "bitstring" -version = "3.1.8" +version = "3.1.9" description = "Simple construction, analysis and modification of binary data." category = "dev" optional = false @@ -223,7 +223,7 @@ python-versions = "*" name = "cffi" version = "1.14.6" description = "Foreign Function Interface for Python calling C code." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -477,17 +477,17 @@ packaging = "*" [[package]] name = "dictdiffer" -version = "0.8.1" +version = "0.9.0" description = "Dictdiffer is a library that helps you to diff and patch dictionaries." category = "dev" optional = false python-versions = "*" [package.extras] -all = ["Sphinx (>=1.4.4)", "sphinx-rtd-theme (>=0.1.9)", "check-manifest (>=0.25)", "coverage (>=4.0)", "isort (>=4.2.2)", "mock (>=1.3.0)", "pydocstyle (>=1.0.0)", "pytest-cov (>=1.8.0)", "pytest-pep8 (>=1.0.6)", "pytest (>=2.8.0)", "tox (>=3.7.0)", "numpy (>=1.11.0)"] -docs = ["Sphinx (>=1.4.4)", "sphinx-rtd-theme (>=0.1.9)"] -numpy = ["numpy (>=1.11.0)"] -tests = ["check-manifest (>=0.25)", "coverage (>=4.0)", "isort (>=4.2.2)", "mock (>=1.3.0)", "pydocstyle (>=1.0.0)", "pytest-cov (>=1.8.0)", "pytest-pep8 (>=1.0.6)", "pytest (>=2.8.0)", "tox (>=3.7.0)"] +all = ["Sphinx (>=3)", "sphinx-rtd-theme (>=0.2)", "check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "sphinx (>=3)", "tox (>=3.7.0)", "numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "pytest (==5.4.3)", "pytest-pycodestyle (>=2)", "pytest-pydocstyle (>=2)", "pytest (>=6)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2.2.0)", "numpy (>=1.20.0)"] +docs = ["Sphinx (>=3)", "sphinx-rtd-theme (>=0.2)"] +numpy = ["numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "numpy (>=1.20.0)"] +tests = ["check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "sphinx (>=3)", "tox (>=3.7.0)", "pytest (==5.4.3)", "pytest-pycodestyle (>=2)", "pytest-pydocstyle (>=2)", "pytest (>=6)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2.2.0)"] [[package]] name = "docutils" @@ -520,6 +520,20 @@ python-versions = "*" [package.dependencies] numpy = ">=1.14" +[[package]] +name = "eccodes" +version = "1.3.3" +description = "Python interface to the ecCodes GRIB and BUFR decoder/encoder" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +attrs = "*" +cffi = "*" +findlibs = "*" +numpy = "*" + [[package]] name = "entrypoints" version = "0.3" @@ -565,6 +579,14 @@ python-versions = "*" [package.dependencies] six = "*" +[[package]] +name = "findlibs" +version = "0.0.2" +description = "A packages to search for shared libraries on various platforms" +category = "main" +optional = true +python-versions = "*" + [[package]] name = "flake8" version = "3.9.2" @@ -774,7 +796,7 @@ smmap = ">=3.0.1,<5" [[package]] name = "gitpython" -version = "3.1.18" +version = "3.1.19" description = "Python Git Library" category = "dev" optional = false @@ -782,7 +804,7 @@ python-versions = ">=3.6" [package.dependencies] gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.0", markers = "python_version < \"3.8\""} +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} [[package]] name = "h11" @@ -1055,7 +1077,7 @@ traitlets = "*" [[package]] name = "jupyter-server" -version = "1.9.0" +version = "1.10.1" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." category = "dev" optional = false @@ -1080,7 +1102,7 @@ traitlets = ">=4.2.1" websocket-client = "*" [package.extras] -test = ["coverage", "pytest", "pytest-cov", "pytest-mock", "requests", "pytest-tornasync", "pytest-console-scripts", "ipykernel"] +test = ["coverage", "pytest (>=6.0)", "pytest-cov", "pytest-mock", "requests", "pytest-tornasync", "pytest-console-scripts", "ipykernel"] [[package]] name = "jupyter-server-mathjax" @@ -1457,6 +1479,22 @@ category = "main" optional = false python-versions = ">=2.6" +[[package]] +name = "pdbufr" +version = "0.9.0" +description = "Pandas reader for the BUFR format using ecCodes." +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +attrs = "*" +eccodes = "*" +pandas = "*" + +[package.extras] +tests = ["flake8", "pytest", "pytest-cov"] + [[package]] name = "percy" version = "2.0.2" @@ -1653,7 +1691,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pycparser" version = "2.20" description = "C parser in Python" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -2547,7 +2585,7 @@ docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [extras] -bufr = [] +bufr = ["pdbufr"] cratedb = ["crate"] docs = ["sphinx", "sphinx-material", "tomlkit", "sphinx-autodoc-typehints", "sphinxcontrib-svg2pdfconverter", "matplotlib", "ipython"] duckdb = ["duckdb"] @@ -2558,14 +2596,14 @@ ipython = ["ipython", "ipython-genutils", "matplotlib"] mpl = ["matplotlib"] mysql = ["mysqlclient"] postgresql = ["psycopg2-binary"] -radar = ["wradlib"] +radar = ["wradlib", "pdbufr"] restapi = ["fastapi", "uvicorn"] sql = ["duckdb"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "8aea04e5970bb964fea99a89e93cc11c09ab7c74b6380f9556e1372d0c1b08ee" +content-hash = "ee560ce6b52b126fb7ae1b1cef4ce71163516171bfb4ad4901f2f610fbd1da51" [metadata.files] aenum = [ @@ -2578,8 +2616,8 @@ alabaster = [ {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] anyio = [ - {file = "anyio-3.2.1-py3-none-any.whl", hash = "sha256:442678a3c7e1cdcdbc37dcfe4527aa851b1b0c9162653b516e9f509821691d50"}, - {file = "anyio-3.2.1.tar.gz", hash = "sha256:07968db9fa7c1ca5435a133dc62f988d84ef78e1d9b22814a59d1c62618afbc5"}, + {file = "anyio-3.3.0-py3-none-any.whl", hash = "sha256:929a6852074397afe1d989002aa96d457e3e1e5441357c60d03e7eea0e65e1b0"}, + {file = "anyio-3.3.0.tar.gz", hash = "sha256:ae57a67583e5ff8b4af47666ff5651c3732d45fd26c929253748e796af860374"}, ] appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, @@ -2638,9 +2676,9 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] bitstring = [ - {file = "bitstring-3.1.8-py2-none-any.whl", hash = "sha256:3ae43c75c1839e1734b37eceba69353eb60dad1d4d7b3a07641b43fd44f02d65"}, - {file = "bitstring-3.1.8-py3-none-any.whl", hash = "sha256:7b736320a2cfc6f009cd364dbd4346f67391637dabfa2bfa6e8299a80c298619"}, - {file = "bitstring-3.1.8.tar.gz", hash = "sha256:c2e327184804d74847ff6c4eb8a750c250d98cf32015311e4a791d8b5264625e"}, + {file = "bitstring-3.1.9-py2-none-any.whl", hash = "sha256:e3e340e58900a948787a05e8c08772f1ccbe133f6f41fe3f0fa19a18a22bbf4f"}, + {file = "bitstring-3.1.9-py3-none-any.whl", hash = "sha256:0de167daa6a00c9386255a7cac931b45e6e24e0ad7ea64f1f92a64ac23ad4578"}, + {file = "bitstring-3.1.9.tar.gz", hash = "sha256:a5848a3f63111785224dca8bb4c0a75b62ecdef56a042c8d6be74b16f7e860e7"}, ] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, @@ -2892,8 +2930,8 @@ deprecation = [ {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, ] dictdiffer = [ - {file = "dictdiffer-0.8.1-py2.py3-none-any.whl", hash = "sha256:d79d9a39e459fe33497c858470ca0d2e93cb96621751de06d631856adfd9c390"}, - {file = "dictdiffer-0.8.1.tar.gz", hash = "sha256:1adec0d67cdf6166bda96ae2934ddb5e54433998ceab63c984574d187cc563d2"}, + {file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"}, + {file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"}, ] docutils = [ {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, @@ -2928,6 +2966,9 @@ duckdb = [ {file = "duckdb-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:087eb3fd2db4ae3223d19b81f53f52dcdd294259dba3bc41fd9556bfaa4ff611"}, {file = "duckdb-0.2.7.tar.gz", hash = "sha256:6aa0ce0a283fd3ba1fd58f8b2d50d43ffb2eddd92af0157ff1ddb7e9103f6e93"}, ] +eccodes = [ + {file = "eccodes-1.3.3.tar.gz", hash = "sha256:60284cfc753a57239515a89bef8d65d78cb36fdd042214027d4ac71303f029b5"}, +] entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, @@ -2944,6 +2985,9 @@ fasteners = [ {file = "fasteners-0.16.3-py2.py3-none-any.whl", hash = "sha256:8408e52656455977053871990bd25824d85803b9417aa348f10ba29ef0c751f7"}, {file = "fasteners-0.16.3.tar.gz", hash = "sha256:b1ab4e5adfbc28681ce44b3024421c4f567e705cc3963c732bf1cba3348307de"}, ] +findlibs = [ + {file = "findlibs-0.0.2.tar.gz", hash = "sha256:6c7e038496f9a97783ab2cd5736bb68522d5bebd8b0eb17c976b6a4ae4032c8d"}, +] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, @@ -3002,8 +3046,8 @@ gitdb = [ {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, ] gitpython = [ - {file = "GitPython-3.1.18-py3-none-any.whl", hash = "sha256:fce760879cd2aebd2991b3542876dc5c4a909b30c9d69dfc488e504a8db37ee8"}, - {file = "GitPython-3.1.18.tar.gz", hash = "sha256:b838a895977b45ab6f0cc926a9045c8d1c44e2b653c1fcc39fe91f42c6e8f05b"}, + {file = "GitPython-3.1.19-py3-none-any.whl", hash = "sha256:17690588e36bd0cee21921ce621fad1e8be45a37afa486ff846fb8efcf53c34c"}, + {file = "GitPython-3.1.19.tar.gz", hash = "sha256:18f4039b96b5567bc4745eb851737ce456a2d499cecd71e84f5c0950e92d0e53"}, ] h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, @@ -3090,8 +3134,8 @@ jupyter-core = [ {file = "jupyter_core-4.7.1.tar.gz", hash = "sha256:79025cb3225efcd36847d0840f3fc672c0abd7afd0de83ba8a1d3837619122b4"}, ] jupyter-server = [ - {file = "jupyter_server-1.9.0-py3-none-any.whl", hash = "sha256:1a6bfcf4cd58a84dfe9d3060a76bf98428c08b8a177202fc0cadcec5f7d74090"}, - {file = "jupyter_server-1.9.0.tar.gz", hash = "sha256:7d19006380f6217458a9db309b54e3dab87ced6c06329c61823907bef2a6f51b"}, + {file = "jupyter_server-1.10.1-py3-none-any.whl", hash = "sha256:b3eef770ffa34595ed26a6e4460866eaf0f4ff710eccc7648f701bb8c1d0443c"}, + {file = "jupyter_server-1.10.1.tar.gz", hash = "sha256:fe6b589bd8d8fe08f608e90ce7da1e6bbfd020d99897453b45149a7853e9188f"}, ] jupyter-server-mathjax = [ {file = "jupyter_server_mathjax-0.2.3-py3-none-any.whl", hash = "sha256:740de2ed0d370f1856faddfaf8c09a6d7435d09d3672f24826451467b268969d"}, @@ -3456,6 +3500,9 @@ pbr = [ {file = "pbr-5.6.0-py2.py3-none-any.whl", hash = "sha256:c68c661ac5cc81058ac94247278eeda6d2e6aecb3e227b0387c30d277e7ef8d4"}, {file = "pbr-5.6.0.tar.gz", hash = "sha256:42df03e7797b796625b1029c0400279c7c34fd7df24a7d7818a1abb5b38710dd"}, ] +pdbufr = [ + {file = "pdbufr-0.9.0.tar.gz", hash = "sha256:c9f9e19ebae7d27e86166d72a088c699044917a66c800cfccfa72005a1cda945"}, +] percy = [ {file = "percy-2.0.2-py2.py3-none-any.whl", hash = "sha256:c1647b768810e9453220a7721a5d52cec560dee913d13c1e29b713703f4f223e"}, {file = "percy-2.0.2.tar.gz", hash = "sha256:6238612dc401fa5c221c0ad7738f7ea43e48fe2695f6423e785ee2bc940f021d"}, diff --git a/pyproject.toml b/pyproject.toml index b96d81ab7..3fdcd5a30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,7 +104,7 @@ click-params = "^0.1.1" cloup = "^0.8.0" # Optional dependencies aka. "extras" -matplotlib = { version = "^3.3.2", optional = true } +matplotlib = { version = "^3.3.2", optional = true } openpyxl = { version = "^3.0.7", optional = true } pyarrow = { version = "^3.0.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" } @@ -119,23 +119,25 @@ psycopg2-binary = { version = "^2.8.6", optional = true } # HTTP REST API service fastapi = { version = "^0.61.1", optional = true } uvicorn = { version = "^0.13.3", optional = true } + +# Radar wradlib = { version = "^1.9.0", optional = true } +pdbufr = { version = "^0.9.0", optional = true } # Explorer UI service plotly = { version = "^4.14.3", optional = true } dash = { version = "^1.19.0", optional = true } dash-bootstrap-components = { version = "^0.12.0", optional = true } -sphinx = { version = "^3.2.1", optional = true } -sphinx-material = { version = "^0.0.30", optional = true } -sphinx-autodoc-typehints = { version = "^1.11.0", optional = true } -sphinxcontrib-svg2pdfconverter = { version = "^1.1.0", optional = true } -tomlkit = { version = "^0.7.0", optional = true } -ipython = { version = "^7.10.1", optional = true } -ipython-genutils = { version = "^0.2.0", optional = true } -zarr = { version = "^2.7.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" } # not supported through numcodecs -xarray = { version = "^0.17.0", optional = true } - +sphinx = { version = "^3.2.1", optional = true } +sphinx-material = { version = "^0.0.30", optional = true } +sphinx-autodoc-typehints = { version = "^1.11.0", optional = true } +sphinxcontrib-svg2pdfconverter = { version = "^1.1.0", optional = true } +tomlkit = { version = "^0.7.0", optional = true } +ipython = { version = "^7.10.1", optional = true } +ipython-genutils = { version = "^0.2.0", optional = true } +zarr = { version = "^2.7.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" } # not supported through numcodecs +xarray = { version = "^0.17.0", optional = true } [tool.poetry.dev-dependencies] black = "^20.8b1" @@ -183,8 +185,8 @@ influxdb = ["influxdb", "influxdb-client"] cratedb = ["crate"] mysql = ["mysqlclient"] postgresql = ["psycopg2-binary"] -radar = ["wradlib", "pybufrkit", "h5py"] -bufr = ["pybufrkit"] +radar = ["wradlib", "pybufrkit", "h5py", "pdbufr"] +bufr = ["pybufrkit", "pdbufr"] [tool.poetry.scripts] wetterdienst = 'wetterdienst.ui.cli:cli' diff --git a/tests/__init__.py b/tests/__init__.py index 504ec2562..23971b684 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,3 +10,7 @@ mac_arm64_unsupported = pytest.mark.skipif( mac_arm64, reason="can't be tested under mac arm64 due to h5py incompatibility" ) +mac_only = pytest.mark.skipif( + sys.platform.startswith("win") or sys.platform.startswith("lin"), + reason="can't be tested under windows due to eccodes incompatibility", +) diff --git a/tests/provider/dwd/radar/test_api_historic.py b/tests/provider/dwd/radar/test_api_historic.py index f4cc50bbd..82d3e19b6 100644 --- a/tests/provider/dwd/radar/test_api_historic.py +++ b/tests/provider/dwd/radar/test_api_historic.py @@ -11,6 +11,7 @@ import pytest import requests +from tests import mac_only from tests.provider.dwd.radar import ( station_reference_pattern_de, station_reference_pattern_sorted, @@ -404,6 +405,44 @@ def test_radar_request_site_historic_pe_bufr(): decoder.process(payload, info_only=True) +@mac_only +@pytest.mark.remote +def test_radar_request_site_historic_pe_bufr_dataframe(): + """ + Verify acquisition of radar/site/PE_ECHO_TOP data works + when using a specific date. + + This time, we will use the BUFR data format. + """ + + # Acquire data from yesterday at this time. + timestamp = datetime.utcnow() - timedelta(days=1) + + request = DwdRadarValues( + parameter=DwdRadarParameter.PE_ECHO_TOP, + start_date=timestamp, + site=DwdRadarSite.BOO, + fmt=DwdRadarDataFormat.BUFR, + read_bufr=True, + ) + + results = list(request.query()) + + if len(results) == 0: + raise pytest.skip("Data currently not available") + + df = results[0].data + + assert not df.empty + + assert df.columns == [ + "station_id", + "latitude", + "longitude", + "horizontalReflectivity", + ] + + @pytest.mark.remote @pytest.mark.parametrize( "format", diff --git a/wetterdienst/provider/dwd/radar/access.py b/wetterdienst/provider/dwd/radar/access.py index 89aa57a01..55069b228 100644 --- a/wetterdienst/provider/dwd/radar/access.py +++ b/wetterdienst/provider/dwd/radar/access.py @@ -8,11 +8,12 @@ from dataclasses import dataclass from datetime import datetime from io import BytesIO -from typing import Generator, Optional, Tuple +from typing import Generator, Optional, Tuple, Union import pandas as pd from wetterdienst.exceptions import FailedDownload +from wetterdienst.metadata.columns import Columns from wetterdienst.metadata.extension import Extension from wetterdienst.metadata.period import Period from wetterdienst.metadata.resolution import Resolution @@ -46,7 +47,7 @@ class RadarResult: Currently, this will relate to exactly one radar data file. """ - data: BytesIO + data: Union[BytesIO, pd.DataFrame] timestamp: datetime = None url: str = None filename: str = None @@ -79,6 +80,7 @@ def collect_radar_data( start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, verify: Optional[bool] = True, + read_bufr: bool = False, ) -> RadarResult: """ Collect radar data for given parameters. @@ -95,10 +97,14 @@ def collect_radar_data( :param start_date: Start date :param end_date: End date :param verify: Whether to verify the response + :param read_bufr: read bufr to pd.DataFrame :return: ``RadarResult`` item """ + if read_bufr: + import pdbufr + # Find latest file. if start_date == DwdRadarDate.LATEST: @@ -212,6 +218,28 @@ def collect_radar_data( if result.timestamp is None: result.timestamp = date_time + if fmt == DwdRadarDataFormat.BUFR: + df = pdbufr.read_bufr( + result.data, + columns=( + "stationNumber", + "latitude", + "longitude", + "horizontalReflectivity", + ), + ) + + df = df.rename( + columns={ + "stationNumber": Columns.STATION_ID.value, + "latitude": Columns.LATITUDE.value, + "longitude": Columns.LONGITUDE.value, + } + ) + + # rewrite data to result + result.data = df + if verify: if fmt == DwdRadarDataFormat.HDF5: verify_hdf5(result.data) diff --git a/wetterdienst/provider/dwd/radar/api.py b/wetterdienst/provider/dwd/radar/api.py index 2d05ae277..e6e1fd3c7 100644 --- a/wetterdienst/provider/dwd/radar/api.py +++ b/wetterdienst/provider/dwd/radar/api.py @@ -55,6 +55,7 @@ def __init__( end_date: Optional[Union[str, datetime, timedelta]] = None, resolution: Optional[Union[str, Resolution, DwdRadarResolution]] = None, period: Optional[Union[str, Period, DwdRadarPeriod]] = None, + **kwargs, ) -> None: """ :param parameter: The radar moment to request @@ -67,6 +68,7 @@ def __init__( :param resolution: Time resolution for RadarParameter.RADOLAN_CDC, either daily or hourly or 5 minutes. :param period: Period type for RadarParameter.RADOLAN_CDC + :param kwargs: kwargs used for collecting the data """ # Convert parameters to enum types. @@ -82,6 +84,8 @@ def __init__( period, DwdRadarPeriod, Period ) + self.read_bufr = kwargs.get("read_bufr") + # Sanity checks. if self.parameter == DwdRadarParameter.RADOLAN_CDC: @@ -261,6 +265,7 @@ def query(self) -> RadarResult: end_date=self.end_date, resolution=self.resolution, period=self.period, + read_bufr=self.read_bufr, ): progressbar.update() yield item