From 901dcd681065caf9a7b05e8a6cb62324978d8535 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Wed, 7 Aug 2024 09:21:26 -0600 Subject: [PATCH 1/3] ci: update pre-commit, switch to ruff --- .pre-commit-config.yaml | 24 +++++++++--------------- CODE_OF_CONDUCT.md | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile-dev | 2 +- environment.yml | 1 + requirements-dev.txt | 4 +--- scripts/format | 4 +--- scripts/lint | 2 +- 8 files changed, 16 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b21d114..3da305d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,34 +2,28 @@ # Please run `pre-commit run --all-files` when adding or changing entries. repos: - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 + rev: v2.3.0 hooks: - id: codespell args: [--ignore-words=.codespellignore] types_or: [jupyter, markdown, python, shell] - - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.6 hooks: - - id: flake8 + - id: ruff + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.2.0 + rev: v1.11.1 hooks: - id: mypy additional_dependencies: - - click != 8.1.0 + - click - stactools + - pytest - types-requests - types-python-dateutil - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.41.0 hooks: - id: markdownlint diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 171ec16..eaf5f71 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal +level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards diff --git a/docker/Dockerfile b/docker/Dockerfile index 5dae677..d03a4a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/stac-utils/stactools:$STACTOOLS_VERSION ARG DOCKER_NAMESPACE_PACKAGE_DIR ARG DOCKER_WORKDIR -ENV PYTHON_VERSION=3.10 +ENV PYTHON_VERSION=3.11 WORKDIR $DOCKER_WORKDIR # For caching purposes, install dependencies but remove the actual package diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev index 79e8906..5b9b613 100644 --- a/docker/Dockerfile-dev +++ b/docker/Dockerfile-dev @@ -3,7 +3,7 @@ FROM ghcr.io/stac-utils/stactools:${STACTOOLS_VERSION}-dev ARG DOCKER_NAMESPACE_PACKAGE_DIR ARG DOCKER_WORKDIR -ENV PYTHON_VERSION=3.10 +ENV PYTHON_VERSION=3.11 WORKDIR $DOCKER_WORKDIR RUN apt-get -y -q update \ diff --git a/environment.yml b/environment.yml index ff2a782..c65cc4f 100644 --- a/environment.yml +++ b/environment.yml @@ -6,4 +6,5 @@ dependencies: - conda-forge::gdal>=3.3 - conda-forge::geos>=3.3 - conda-forge::rasterio>=1.3 + - conda-forge::sqlite - conda-forge::libstdcxx-ng # gdal dependency. Make sure it's from the same channel as gdal diff --git a/requirements-dev.txt b/requirements-dev.txt index d115243..3cfac75 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,8 @@ -black codespell -flake8 -isort mypy pre-commit pytest pytest-cov +ruff types-python-dateutil types-requests diff --git a/scripts/format b/scripts/format index baf9730..7aa3782 100755 --- a/scripts/format +++ b/scripts/format @@ -17,9 +17,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${1:-}" = "--help" ]; then usage else - # Sort imports - pre-commit run isort --all-files # Code formatting - pre-commit run black --all-files + pre-commit run ruff-format --all-files fi fi diff --git a/scripts/lint b/scripts/lint index b2211f9..41ee170 100755 --- a/scripts/lint +++ b/scripts/lint @@ -18,7 +18,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then usage else # Lint - pre-commit run flake8 --all-files + pre-commit run ruff --all-files # Type checking pre-commit run mypy --all-files fi From bbed9019f699e3e340a9d053f94c5efbb8d6d0c0 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Wed, 7 Aug 2024 10:14:06 -0600 Subject: [PATCH 2/3] tests: skip many external data tests --- README.md | 7 +++++ tests/conftest.py | 30 +++++++++++++++++++ tests/ocean_heat_content/test_commands.py | 3 ++ tests/ocean_heat_content/test_stac.py | 12 +++++++- .../test_commands.py | 3 ++ .../test_stac.py | 7 +++-- .../test_commands.py | 2 ++ .../sea_surface_temperature_whoi/test_stac.py | 3 +- tests/test_dataset.py | 3 +- 9 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/conftest.py diff --git a/README.md b/README.md index e854267..5c221d7 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,13 @@ To run the tests: pytest ``` +Many tests are skipped by default because access external data, which makes them slow and a bit flakey. +To run these tests: + +```shell +pytest --external-data +``` + ### Updating static resources There are several .json files in the source tree that are used to populate diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..896bbeb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,30 @@ +from typing import Any + +import pytest +from pytest import Config, Parser + + +def pytest_addoption(parser: Parser) -> None: + parser.addoption( + "--external-data", + action="store_true", + default=False, + help="run tests that require external data", + ) + + +def pytest_configure(config: Config) -> None: + config.addinivalue_line( + "markers", + "external_data: marks tests requiring external data, " + "and disables them by default (enable with --external-data)", + ) + + +def pytest_collection_modifyitems(config: Config, items: Any) -> None: + if config.getoption("--external-data"): + return + skip_network_access = pytest.mark.skip(reason="need --external-data option to run") + for item in items: + if "external_data" in item.keywords: + item.add_marker(skip_network_access) diff --git a/tests/ocean_heat_content/test_commands.py b/tests/ocean_heat_content/test_commands.py index 2bda5ca..2dc66f1 100644 --- a/tests/ocean_heat_content/test_commands.py +++ b/tests/ocean_heat_content/test_commands.py @@ -1,5 +1,6 @@ from pathlib import Path +import pytest from pystac import Collection, ItemCollection from tests import run_command, test_data @@ -16,6 +17,7 @@ def test_create_collection(tmp_path: Path) -> None: collection.validate() +@pytest.mark.external_data def test_create_items(tmp_path: Path) -> None: destination = tmp_path / "item-collection.json" infile = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") @@ -38,6 +40,7 @@ def test_download() -> None: assert result.exit_code == 0 +@pytest.mark.external_data def test_cogify(tmp_path: Path) -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") result = run_command(f"noaa-cdr ocean-heat-content cogify {path} -o {tmp_path}") diff --git a/tests/ocean_heat_content/test_stac.py b/tests/ocean_heat_content/test_stac.py index 8b4dee8..89c429a 100644 --- a/tests/ocean_heat_content/test_stac.py +++ b/tests/ocean_heat_content/test_stac.py @@ -9,7 +9,6 @@ from pystac.extensions.projection import ProjectionExtension from pystac.extensions.raster import RasterExtension from pystac.extensions.scientific import ScientificExtension - from stactools.noaa_cdr.ocean_heat_content import cog, stac from .. import test_data @@ -51,6 +50,7 @@ def test_create_collection() -> None: collection.validate_all() +@pytest.mark.external_data def test_create_items_one_netcdf(tmp_path: Path) -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") items = stac.create_items([path], str(tmp_path)) @@ -88,6 +88,7 @@ def test_create_items_one_netcdf(tmp_path: Path) -> None: item.validate() +@pytest.mark.external_data def test_create_items_one_netcdf_cog_hrefs(tmp_path: Path) -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") items = stac.create_items([path], str(tmp_path)) @@ -104,6 +105,7 @@ def test_create_items_one_netcdf_cog_hrefs(tmp_path: Path) -> None: assert len(new_items) == len(items) +@pytest.mark.external_data def test_create_items_two_netcdfs_same_items(tmp_path: Path) -> None: paths = [ test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc"), @@ -118,6 +120,7 @@ def test_create_items_two_netcdfs_same_items(tmp_path: Path) -> None: item.validate() +@pytest.mark.external_data def test_create_items_two_netcdfs_different_items() -> None: paths = [ test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc"), @@ -133,6 +136,7 @@ def test_create_items_two_netcdfs_different_items() -> None: item.validate() +@pytest.mark.external_data def test_create_items_one_netcdf_latest_only(tmp_path: Path) -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") items = stac.create_items([path], str(tmp_path), latest_only=True) @@ -155,6 +159,7 @@ def test_create_items_one_netcdf_latest_only(tmp_path: Path) -> None: ("mean_total_steric_sea_level_anomaly_0-2000_yearly.nc", 17), ], ) +@pytest.mark.external_data def test_cogify(tmp_path: Path, infile: str, num_cogs: int) -> None: external_data_path = test_data.get_external_data(infile) cogs = cog.cogify(external_data_path, str(tmp_path)) @@ -165,6 +170,7 @@ def test_cogify(tmp_path: Path, infile: str, num_cogs: int) -> None: assert Path(c.asset().href).exists() +@pytest.mark.external_data def test_cogify_href(tmp_path: Path) -> None: href = ( "https://www.ncei.noaa.gov/data/oceans/ncei/archive/data" @@ -178,6 +184,7 @@ def test_cogify_href(tmp_path: Path) -> None: assert Path(c.asset().href).exists() +@pytest.mark.external_data def test_cogify_href_no_output_directory() -> None: href = ( "https://www.ncei.noaa.gov/data/oceans/ncei/archive/data" @@ -187,12 +194,14 @@ def test_cogify_href_no_output_directory() -> None: cog.cogify(href) +@pytest.mark.external_data def test_unitless(tmp_path: Path) -> None: path = test_data.get_external_data("mean_salinity_anomaly_0-2000_yearly.nc") cogs = cog.cogify(path, str(tmp_path)) assert "unit" not in cogs[0].asset().extra_fields["raster:bands"][0] +@pytest.mark.external_data def test_cogify_cog_href(tmp_path: Path) -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") cogs = cog.cogify(path, str(tmp_path)) @@ -222,6 +231,7 @@ def test_cogify_cog_href(tmp_path: Path) -> None: ("heat_content_anomaly_0-2000_seasonal.nc", 2005, "seasonal", 2000), ], ) +@pytest.mark.external_data def test_create_netcdf_item( infile: str, year: int, interval: str, max_depth: int ) -> None: diff --git a/tests/sea_surface_temperature_optimum_interpolation/test_commands.py b/tests/sea_surface_temperature_optimum_interpolation/test_commands.py index 9596825..65bdeb7 100644 --- a/tests/sea_surface_temperature_optimum_interpolation/test_commands.py +++ b/tests/sea_surface_temperature_optimum_interpolation/test_commands.py @@ -1,5 +1,6 @@ from pathlib import Path +import pytest from pystac import Collection, Item from tests import run_command, test_data @@ -19,6 +20,7 @@ def test_create_collection(tmp_path: Path) -> None: collection.validate() +@pytest.mark.external_data def test_create_item(tmp_path: Path) -> None: destination = tmp_path / "item.json" infile = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc") @@ -33,6 +35,7 @@ def test_create_item(tmp_path: Path) -> None: item.validate() +@pytest.mark.external_data def test_create_item_with_cogs(tmp_path: Path) -> None: path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc") result = run_command( diff --git a/tests/sea_surface_temperature_optimum_interpolation/test_stac.py b/tests/sea_surface_temperature_optimum_interpolation/test_stac.py index 44e6ed1..8aa4fd1 100644 --- a/tests/sea_surface_temperature_optimum_interpolation/test_stac.py +++ b/tests/sea_surface_temperature_optimum_interpolation/test_stac.py @@ -1,12 +1,13 @@ import datetime from pathlib import Path +import pytest +import stactools.noaa_cdr.stac from dateutil.tz import tzutc from pystac.extensions.projection import ProjectionExtension from pystac.extensions.raster import RasterExtension - -import stactools.noaa_cdr.stac from stactools.noaa_cdr.sea_surface_temperature_optimum_interpolation import stac + from tests import test_data @@ -20,6 +21,7 @@ def test_create_collection() -> None: collection.validate() +@pytest.mark.external_data def test_create_item() -> None: path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc") item = stac.create_item(path) @@ -58,6 +60,7 @@ def test_create_item() -> None: item.validate() +@pytest.mark.external_data def test_add_items(tmp_path: Path) -> None: path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc") item = stac.create_item(path) diff --git a/tests/sea_surface_temperature_whoi/test_commands.py b/tests/sea_surface_temperature_whoi/test_commands.py index a27e639..c307f2f 100644 --- a/tests/sea_surface_temperature_whoi/test_commands.py +++ b/tests/sea_surface_temperature_whoi/test_commands.py @@ -1,10 +1,12 @@ from pathlib import Path +import pytest from pystac import Collection, ItemCollection from .. import run_command, test_data +@pytest.mark.external_data def test_create_cog_items(tmp_path: Path) -> None: path = test_data.get_external_data( "SEAFLUX-OSB-CDR_V02R00_SST_D20210831_C20211223.nc" diff --git a/tests/sea_surface_temperature_whoi/test_stac.py b/tests/sea_surface_temperature_whoi/test_stac.py index f422de2..09746bf 100644 --- a/tests/sea_surface_temperature_whoi/test_stac.py +++ b/tests/sea_surface_temperature_whoi/test_stac.py @@ -1,13 +1,14 @@ from pathlib import Path +import pytest from pystac.extensions.raster import RasterExtension from pystac.extensions.scientific import ScientificExtension - from stactools.noaa_cdr.sea_surface_temperature_whoi import stac from .. import test_data +@pytest.mark.external_data def test_create_cog_items(tmp_path: Path) -> None: path = test_data.get_external_data( "SEAFLUX-OSB-CDR_V02R00_SST_D20210831_C20211223.nc" diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 8718c13..e95cfc9 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,11 +1,11 @@ import pytest import xarray - from stactools.noaa_cdr import dataset from . import test_data +@pytest.mark.external_data def test_data_variable_name() -> None: path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc") with xarray.open_dataset(path, decode_times=False) as ds: @@ -23,6 +23,7 @@ def test_data_variable_name() -> None: dataset.data_variable_name(ds) +@pytest.mark.external_data def test_data_variable_names() -> None: path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc") with xarray.open_dataset(path) as ds: From f1f9bfcf8870ee32604239265d7b4f4f8acc2a5c Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Wed, 7 Aug 2024 10:19:14 -0600 Subject: [PATCH 3/3] deps: remove python 3.8, add python 3.11 and 3.12 --- .github/workflows/continuous-integration.yml | 2 +- setup.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 27805f9..cecb602 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] defaults: run: shell: bash -l {0} diff --git a/setup.cfg b/setup.cfg index d5df51a..c7ef0ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,9 +18,10 @@ keywords = classifiers = Development Status :: 4 - Beta License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 [options] package_dir =