From 1732cc49acfa82df5e8ae3be48e6a92a950b6cc7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 14 Oct 2024 18:29:54 +0500 Subject: [PATCH 1/3] Update Python versions. --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 24 ++++++++++----------- .readthedocs.yml | 2 +- docs/setup.rst | 2 +- docs/usage/scrapy-poet.rst | 2 -- setup.py | 2 +- tests/test_providers.py | 40 +---------------------------------- tox.ini | 20 +++++++++--------- 8 files changed, 27 insertions(+), 67 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 35aca3ed..820c29d6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5586cb21..39acefcb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,31 +17,31 @@ jobs: fail-fast: false matrix: include: - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-scrapy-2x0 - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-scrapy-2x1 - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-scrapy-2x3 - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-scrapy-2x4 - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-scrapy-2x5 - - python-version: '3.8' - toxenv: pinned-scrapy-2x6 - python-version: '3.9' + toxenv: pinned-scrapy-2x6 - python-version: '3.10' - python-version: '3.11' - python-version: '3.12' + - python-version: '3.13' - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-provider - - python-version: '3.12' + - python-version: '3.13' toxenv: provider - - python-version: '3.8' + - python-version: '3.9' toxenv: pinned-extra - - python-version: '3.12' + - python-version: '3.13' toxenv: extra steps: @@ -67,7 +67,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12"] + python-version: ["3.13"] tox-job: ["mypy", "linters", "twine-check", "docs"] steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index 1519565e..2c7ffd1e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ sphinx: build: os: ubuntu-22.04 tools: - python: "3.11" # Keep in sync with .github/workflows/test.yml + python: "3.13" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt diff --git a/docs/setup.rst b/docs/setup.rst index 58f9187f..1aea2bde 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -18,7 +18,7 @@ You need at least: - A :ref:`Zyte API ` subscription (there’s a :ref:`free trial `). -- Python 3.8+ +- Python 3.9+ - Scrapy 2.0.1+ diff --git a/docs/usage/scrapy-poet.rst b/docs/usage/scrapy-poet.rst index 9f3e3b8d..25527831 100644 --- a/docs/usage/scrapy-poet.rst +++ b/docs/usage/scrapy-poet.rst @@ -39,8 +39,6 @@ Dependency annotations ``ZyteApiProvider`` understands and makes use of some dependency annotations. -.. note:: Dependency annotations require Python 3.9+. - Item annotations ---------------- diff --git a/setup.py b/setup.py index 4f339562..a158db11 100644 --- a/setup.py +++ b/setup.py @@ -43,10 +43,10 @@ def get_version(): "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], ) diff --git a/tests/test_providers.py b/tests/test_providers.py index c5a935be..6ef746d3 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,5 +1,5 @@ -import sys from collections import defaultdict +from typing import Annotated import pytest @@ -259,13 +259,8 @@ async def test_provider_params_remove_unused_options(mockserver): ) -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_extractfrom(mockserver): - from typing import Annotated - @attrs.define class AnnotatedProductPage(BasePage): product: Annotated[Product, ExtractFrom.httpResponseBody] @@ -295,13 +290,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage): # type: ) -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_extractfrom_double(mockserver, caplog): - from typing import Annotated - @attrs.define class AnnotatedProductPage(BasePage): product: Annotated[Product, ExtractFrom.httpResponseBody] @@ -322,13 +312,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage): # type: assert "Multiple different extractFrom specified for product" in caplog.text -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_extractfrom_override(mockserver): - from typing import Annotated - @attrs.define class AnnotatedProductPage(BasePage): product: Annotated[Product, ExtractFrom.httpResponseBody] @@ -359,13 +344,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage): # type: ) -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_geolocation(mockserver): - from typing import Annotated - @attrs.define class GeoProductPage(BasePage): product: Product @@ -385,9 +365,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage): # type: ignore assert item["product"].name == "Product name (country DE)" -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_geolocation_unannotated(mockserver, caplog): @attrs.define @@ -414,9 +391,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage): # type: ignore } -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @pytest.mark.parametrize( "annotation", [ @@ -428,8 +402,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage): # type: ignore ) @ensureDeferred async def test_provider_custom_attrs(mockserver, annotation): - from typing import Annotated - @attrs.define class CustomAttrsPage(BasePage): product: Product @@ -468,13 +440,8 @@ def parse_(self, response: DummyResponse, page: CustomAttrsPage): # type: ignor ) -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_custom_attrs_values(mockserver): - from typing import Annotated - @attrs.define class CustomAttrsPage(BasePage): product: Product @@ -1086,13 +1053,8 @@ def parse_(self, response: DummyResponse, screenshot: Screenshot): assert item["screenshot"].body == b"screenshot-body-contents" -@pytest.mark.skipif( - sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" -) @ensureDeferred async def test_provider_actions(mockserver, caplog): - from typing import Annotated - @attrs.define class ActionProductPage(BasePage): product: Product diff --git a/tox.ini b/tox.ini index ee6e4999..b171a34b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,mypy,linters,twine-check,docs +envlist = py39,py310,py311,py312,py313,mypy,linters,twine-check,docs [testenv] deps = @@ -37,35 +37,35 @@ deps = # Earliest supported Scrapy version. [testenv:pinned-scrapy-2x0] -basepython=python3.8 +basepython=python3.9 deps = {[pinned-pre-scrapy-2x5]deps} scrapy==2.0.1 # Scrapy version introducing Response.ip_address. [testenv:pinned-scrapy-2x1] -basepython=python3.8 +basepython=python3.9 deps = {[pinned-pre-scrapy-2x5]deps} scrapy==2.1.0 # Latest Scrapy version since 2.0.1 not requiring to install the reactor early. [testenv:pinned-scrapy-2x3] -basepython=python3.8 +basepython=python3.9 deps = {[pinned-pre-scrapy-2x5]deps} scrapy==2.3.0 # First Scrapy version requiring to install the reactor early. [testenv:pinned-scrapy-2x4] -basepython=python3.8 +basepython=python3.9 deps = {[pinned-pre-scrapy-2x5]deps} scrapy==2.4.0 # Scrapy version introducing Response.protocol. [testenv:pinned-scrapy-2x5] -basepython=python3.8 +basepython=python3.9 deps = {[pinned]deps} scrapy==2.5.0 @@ -73,7 +73,7 @@ deps = # First Scrapy version since 2.4.0 where installing the reactor earlier is not # necessary. [testenv:pinned-scrapy-2x6] -basepython=python3.8 +basepython=python3.9 deps = {[pinned]deps} scrapy==2.6.0 @@ -82,7 +82,7 @@ deps = extras = provider [testenv:pinned-provider] -basepython=python3.8 +basepython=python3.9 extras = provider deps = # scrapy-poet >= 0.4.0 depends on scrapy >= 2.6.0 @@ -93,14 +93,14 @@ deps = zyte-common-items==0.24.0 [testenv:pinned-extra] -basepython=python3.8 +basepython=python3.9 deps = {[testenv:pinned-scrapy-2x0]deps} scrapy-crawlera==1.1.0 scrapy-zyte-smartproxy==2.0.0 [testenv:extra] -basepython=python3.12 +basepython=python3.13 deps = {[testenv]deps} scrapy-crawlera From 33e82368bb043713350bb5ea53b2bbd1f69468ec Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 14 Oct 2024 18:35:28 +0500 Subject: [PATCH 2/3] Bump tool versions. --- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 2 +- tox.ini | 9 +++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4323286..8e20ecbd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,19 +4,19 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - flake8-print - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.19.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.2.0 + - black==24.10.0 diff --git a/pyproject.toml b/pyproject.toml index 63f6744d..19622e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -target-version = ["py38", "py39", "py310", "py311"] +target-version = ["py39", "py310", "py311", "py312", "py313"] [tool.isort] profile = "black" diff --git a/tox.ini b/tox.ini index b171a34b..21c19341 100644 --- a/tox.ini +++ b/tox.ini @@ -109,8 +109,8 @@ deps = [testenv:mypy] extras = provider deps = - mypy==1.8.0 - types-setuptools + mypy==1.11.2 + pytest commands = mypy scrapy_zyte_api tests @@ -120,9 +120,10 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twine-check] deps = - twine + twine==5.1.1 + build==1.2.2 commands = - python setup.py sdist + python -m build --sdist twine check dist/* [testenv:docs] From 43f8b0fb9d78003075216e2bac17835b2b337659 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 15 Oct 2024 16:13:02 +0500 Subject: [PATCH 3/3] Roll back RTD Python --- .github/workflows/test.yml | 2 +- .readthedocs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39acefcb..89b756ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13"] + python-version: ["3.12"] # Keep in sync with .readthedocs.yml tox-job: ["mypy", "linters", "twine-check", "docs"] steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index 2c7ffd1e..9acdc482 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ sphinx: build: os: ubuntu-22.04 tools: - python: "3.13" # Keep in sync with .github/workflows/test.yml + python: "3.12" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt