From ec9a20ea821189abe7317c5099cf10f96288e069 Mon Sep 17 00:00:00 2001 From: Isabel Zimmerman <54685329+isabelizimm@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:26:02 -0700 Subject: [PATCH 1/4] maint: update `dev.txt` and gcs protocol names (#242) * update to gcs naming * update to latest version of quartodoc --- pins/constructors.py | 4 +- pins/tests/test_constructors.py | 2 + requirements/dev.txt | 251 +++++++++++++++++--------------- 3 files changed, 140 insertions(+), 117 deletions(-) diff --git a/pins/constructors.py b/pins/constructors.py index 33ba3214..357e39af 100644 --- a/pins/constructors.py +++ b/pins/constructors.py @@ -58,11 +58,11 @@ def board_deparse(board: BaseBoard): return f"board_connect(server_url={repr(url)}{allow_pickle})" elif prot in ["file", ("file", "local")]: return f"board_folder({repr(board.board)}{allow_pickle})" - elif prot in [["s3", "s3a"], ("s3", "s3a")]: + elif set(prot) == {"s3", "s3a"}: return f"board_s3({repr(board.board)}{allow_pickle})" elif prot == "abfs": return f"board_azure({repr(board.board)}{allow_pickle})" - elif prot == ("gcs", "gs"): + elif set(prot) == {"gcs", "gs"} or prot == "gs": return f"board_gcs({repr(board.board)}{allow_pickle})" elif prot == "http": return f"board_url({repr(board.board)}, {board.pin_paths}{allow_pickle})" diff --git a/pins/tests/test_constructors.py b/pins/tests/test_constructors.py index b64e658e..59794ecd 100644 --- a/pins/tests/test_constructors.py +++ b/pins/tests/test_constructors.py @@ -47,6 +47,8 @@ def construct_from_board(board): ) elif fs_name == "abfs": board = c.board_azure(board.board) + elif fs_name == "gs": + board = c.board_gcs(board.board) else: board = getattr(c, f"board_{fs_name}")(board.board) diff --git a/requirements/dev.txt b/requirements/dev.txt index 5b5afa9c..5c1249c4 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,13 +2,16 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --extra=doc --extra=test --output-file=- setup.cfg +# pip-compile --extra=doc --extra=test --output-file=- --strip-extras setup.cfg # +--index-url https://pypi.python.org/simple/ +--trusted-host pypi.org + adlfs==2022.2.0 # via pins (setup.cfg) -aiobotocore==2.5.2 +aiobotocore==2.13.1 # via s3fs -aiohttp==3.8.5 +aiohttp==3.9.5 # via # adlfs # aiobotocore @@ -18,73 +21,71 @@ aioitertools==0.11.0 # via aiobotocore aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic appdirs==1.4.4 # via pins (setup.cfg) -appnope==0.1.3 +appnope==0.1.4 # via # ipykernel # ipython -asttokens==2.2.1 +asttokens==2.4.1 # via stack-data -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 +attrs==23.2.0 # via # aiohttp # jsonschema # pytest # referencing # sphobjinv -azure-core==1.28.0 +azure-core==1.30.2 # via # adlfs # azure-identity # azure-storage-blob azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.13.0 +azure-identity==1.17.1 # via adlfs -azure-storage-blob==12.17.0 +azure-storage-blob==12.20.0 # via adlfs backcall==0.2.0 # via ipython -beartype==0.15.0 +beartype==0.18.5 # via plum-dispatch -botocore==1.29.161 +botocore==1.34.131 # via aiobotocore -build==0.10.0 +build==1.2.1 # via pip-tools -cachetools==5.3.1 +cachetools==5.4.0 # via google-auth -certifi==2023.7.22 +certifi==2024.7.4 # via # requests # sphobjinv -cffi==1.15.1 +cffi==1.16.0 # via # azure-datalake-store # cryptography -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 +charset-normalizer==3.3.2 + # via requests +click==8.1.7 # via # pip-tools # quartodoc colorama==0.4.6 # via griffe -comm==0.1.3 +comm==0.2.2 # via ipykernel -cramjam==2.6.2 +cramjam==2.8.3 # via fastparquet -cryptography==41.0.2 +cryptography==42.0.8 # via # azure-identity # azure-storage-blob # msal # pyjwt -debugpy==1.6.7 +debugpy==1.8.2 # via ipykernel decopatch==1.4.10 # via pytest-cases @@ -92,67 +93,69 @@ decorator==5.1.1 # via # gcsfs # ipython -executing==1.2.0 +executing==2.0.1 # via stack-data -fastjsonschema==2.18.0 +fastjsonschema==2.20.0 # via nbformat -fastparquet==2023.7.0 +fastparquet==2024.5.0 # via pins (setup.cfg) -frozenlist==1.4.0 +frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2024.6.1 # via # adlfs # fastparquet # gcsfs # pins (setup.cfg) # s3fs -gcsfs==2023.6.0 +gcsfs==2024.6.1 # via pins (setup.cfg) -google-api-core==2.11.1 +google-api-core==2.19.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.22.0 +google-auth==2.32.0 # via # gcsfs # google-api-core # google-auth-oauthlib # google-cloud-core # google-cloud-storage -google-auth-oauthlib==1.0.0 +google-auth-oauthlib==1.2.1 # via gcsfs -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via google-cloud-storage -google-cloud-storage==2.10.0 +google-cloud-storage==2.17.0 # via gcsfs google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.1 # via google-cloud-storage -googleapis-common-protos==1.59.1 +googleapis-common-protos==1.63.2 # via google-api-core -griffe==0.32.3 +griffe==0.48.0 # via quartodoc -humanize==4.7.0 +humanize==4.10.0 # via pins (setup.cfg) -idna==3.4 +idna==3.7 # via # requests # yarl -importlib-metadata==6.8.0 +importlib-metadata==8.0.0 # via # pins (setup.cfg) # quartodoc -importlib-resources==6.0.0 +importlib-resources==6.4.0 # via # pins (setup.cfg) # quartodoc iniconfig==2.0.0 # via pytest -ipykernel==6.25.0 +ipykernel==6.29.5 # via pins (setup.cfg) ipython==8.12.0 # via @@ -160,99 +163,107 @@ ipython==8.12.0 # pins (setup.cfg) isodate==0.6.1 # via azure-storage-blob -jedi==0.18.2 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jinja2==3.1.4 # via pins (setup.cfg) jmespath==1.0.1 # via botocore -joblib==1.3.1 +joblib==1.4.2 # via pins (setup.cfg) -jsonschema==4.18.4 +jsonschema==4.23.0 # via # nbformat # sphobjinv -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.3.0 +jupyter-client==8.6.2 # via # ipykernel # nbclient -jupyter-core==5.3.1 +jupyter-core==5.7.2 # via # ipykernel # jupyter-client # nbclient # nbformat -makefun==1.15.1 +makefun==1.15.4 # via # decopatch # pytest-cases -markupsafe==2.1.3 +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 # via jinja2 -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via # ipykernel # ipython -msal==1.23.0 +mdurl==0.1.2 + # via markdown-it-py +msal==1.29.0 # via # azure-datalake-store # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.2.0 # via azure-identity -multidict==6.0.4 +multidict==6.0.5 # via # aiohttp # yarl -nbclient==0.8.0 +nbclient==0.10.0 # via pins (setup.cfg) -nbformat==5.9.1 +nbformat==5.10.4 # via # nbclient # pins (setup.cfg) -nest-asyncio==1.5.6 +nest-asyncio==1.6.0 # via ipykernel -numpy==1.25.1 +numpy==2.0.0 # via # fastparquet # pandas # pyarrow oauthlib==3.2.2 # via requests-oauthlib -packaging==23.1 +packaging==24.1 # via # build # fastparquet # ipykernel # pytest -pandas==2.0.3 + # pytest-cases +pandas==2.2.2 # via # fastparquet # pins (setup.cfg) -parso==0.8.3 +parso==0.8.4 # via jedi -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==7.1.0 +pip-tools==7.4.1 # via pins (setup.cfg) -platformdirs==3.9.1 +platformdirs==4.2.2 # via jupyter-core -pluggy==1.2.0 +pluggy==1.5.0 # via pytest -plum-dispatch==2.1.1 +plum-dispatch==2.5.1.post1 # via quartodoc -portalocker==2.7.0 +portalocker==2.10.1 # via msal-extensions -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.47 # via ipython -protobuf==4.23.4 +proto-plus==1.24.0 + # via google-api-core +protobuf==5.27.2 # via # google-api-core # googleapis-common-protos -psutil==5.9.5 + # proto-plus +psutil==6.0.0 # via ipykernel ptyprocess==0.7.0 # via pexpect @@ -260,59 +271,67 @@ pure-eval==0.2.2 # via stack-data py==1.11.0 # via pytest -pyarrow==12.0.1 +pyarrow==16.1.0 # via pins (setup.cfg) -pyasn1==0.5.0 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth -pycparser==2.21 +pycparser==2.22 # via cffi -pydantic==1.10.12 +pydantic==2.8.2 # via quartodoc -pygments==2.15.1 - # via ipython -pyjwt[crypto]==2.8.0 - # via msal -pyproject-hooks==1.0.0 - # via build +pydantic-core==2.20.1 + # via pydantic +pygments==2.18.0 + # via + # ipython + # rich +pyjwt==2.8.0 + # via + # msal + # pyjwt +pyproject-hooks==1.1.0 + # via + # build + # pip-tools pytest==7.1.3 # via # pins (setup.cfg) # pytest-dotenv # pytest-parallel -pytest-cases==3.6.14 +pytest-cases==3.8.5 # via pins (setup.cfg) pytest-dotenv==0.5.2 # via pins (setup.cfg) pytest-parallel==0.1.1 # via pins (setup.cfg) -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # jupyter-client # pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via pytest-dotenv -pytz==2023.3 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # pins (setup.cfg) # quartodoc -pyzmq==25.1.0 +pyzmq==26.0.3 # via # ipykernel # jupyter-client -quartodoc==0.4.1 +quartodoc==0.7.5 # via pins (setup.cfg) -referencing==0.30.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # azure-core # azure-datalake-store @@ -321,40 +340,41 @@ requests==2.31.0 # google-cloud-storage # msal # pins (setup.cfg) + # quartodoc # requests-oauthlib -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via google-auth-oauthlib -rpds-py==0.9.2 +rich==13.7.1 + # via plum-dispatch +rpds-py==0.19.0 # via # jsonschema # referencing rsa==4.9 # via google-auth -s3fs==2023.6.0 +s3fs==2024.6.1 # via pins (setup.cfg) six==1.16.0 # via # asttokens # azure-core - # azure-identity - # google-auth # isodate # python-dateutil -sphobjinv==2.3.1 +sphobjinv==2.3.1.1 # via quartodoc -stack-data==0.6.2 +stack-data==0.6.3 # via ipython tabulate==0.9.0 # via quartodoc -tblib==2.0.0 +tblib==3.0.0 # via pytest-parallel tomli==2.0.1 # via pytest -tornado==6.3.2 +tornado==6.4.1 # via # ipykernel # jupyter-client -traitlets==5.9.0 +traitlets==5.14.3 # via # comm # ipykernel @@ -364,32 +384,33 @@ traitlets==5.9.0 # matplotlib-inline # nbclient # nbformat -typing-extensions==4.7.1 +typing-extensions==4.12.2 # via # azure-core + # azure-identity # azure-storage-blob # pydantic + # pydantic-core # quartodoc -tzdata==2023.3 +tzdata==2024.1 # via pandas -urllib3==1.26.16 +urllib3==2.2.2 # via # botocore - # google-auth # requests -watchdog==3.0.0 +watchdog==4.0.1 # via quartodoc -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit -wheel==0.41.0 +wheel==0.43.0 # via pip-tools -wrapt==1.15.0 +wrapt==1.16.0 # via aiobotocore -xxhash==3.2.0 +xxhash==3.4.1 # via pins (setup.cfg) -yarl==1.9.2 +yarl==1.9.4 # via aiohttp -zipp==3.16.2 +zipp==3.19.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 83973372fa4774a183d87bb01273e0fb6fec178c Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Tue, 16 Jul 2024 12:15:46 +1200 Subject: [PATCH 2/4] Fix typo (#238) --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8e5c344e..e6317a9a 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,5 +1,5 @@ # Who maintains pins -The pins-python package was is currently maintained by Isabel Zimmerman . [Posit Software, PBC](https://posit.co/products/open-source/) is a copyright holder and funder of this package. +The pins-python package is currently maintained by Isabel Zimmerman . [Posit Software, PBC](https://posit.co/products/open-source/) is a copyright holder and funder of this package. Several individuals in the community have taken an active role in helping to maintain this package and submit fixes. Those individuals are shown in the git changelog. From 41c37aea979d3f09e7c0ef4d8cae40d491b02440 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Tue, 16 Jul 2024 12:22:48 +1200 Subject: [PATCH 3/4] Add some extra information to help set up local pytest calls (#240) * Add some extra information to help set up local pytest calls * Revert bullet point style change --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9faeb122..127485a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,13 @@ pytest pins -m 'not fs_rsc' # run only local filesystem backend tests pytest pins -m 'fs_file' + +# run all tests except those for S3 and GCS +pytest pins -m 'not fs_s3 and not fs_gcs' + +# run all tests except those using data on GitHub +# n.b. doctests cannot have marks https://github.com/pytest-dev/pytest/issues/5794 +pytest pins -m 'not skip_on_github' -k 'not pins.boards.BoardManual' ``` There are two important details to note for testing: From 0aefeea2da4a43b8765ebb431abef314de43b8b3 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 05:18:13 +1200 Subject: [PATCH 4/4] Add comment linking why we use appdirs in setup.cfg (#245) --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 51a394a5..ff0c547d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,7 @@ install_requires = joblib>=0.12.0 importlib-metadata>=4.4 importlib-resources>=1.3 + # Using appdirs rather than platformdirs is deliberate, see https://github.com/rstudio/pins-python/pull/239 appdirs<2.0.0 humanize>=1.0.0 requests