Add a test for multiple data types extraction. (#233)

scrapy-plugins · Dec 9, 2024 · c78d304 · c78d304
1 parent 5fbf0bc
commit c78d304
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 0 deletions.
diff --git a/tests/mockserver.py b/tests/mockserver.py
@@ -241,6 +241,13 @@ def render_POST(self, request):
                     },
                 }
 
+            if request_data.get("productNavigation") is True:
+                response_data["productNavigation"] = {
+                    "url": response_data["url"],
+                    "name": "Product navigation",
+                    "pageNumber": 0,
+                }
+
         return json.dumps(response_data).encode()
 
 

diff --git a/tests/test_providers.py b/tests/test_providers.py
@@ -31,6 +31,7 @@
     CustomAttributes,
     CustomAttributesValues,
     Product,
+    ProductNavigation,
 )
 from zyte_common_items.fields import auto_field
 
@@ -68,6 +69,13 @@ class ProductPage(BasePage):
     product: Product
 
 
+@attrs.define
+class ProductNavigationPage(BasePage):
+    html: BrowserHtml
+    response: BrowserResponse
+    product_nav: ProductNavigation
+
+
 class ZyteAPISpider(Spider):
     url: str
 
@@ -1723,3 +1731,50 @@ def parse(self, response: DummyResponse, product: Product):
 
     # Reset rules
     default_registry.__init__()  # type: ignore[misc]
+
+
+class ZyteAPIMultipleSpider(Spider):
+    url: str
+
+    def start_requests(self):
+        yield Request(self.url, callback=self.parse_)
+
+    def parse_(
+        self,
+        response: DummyResponse,
+        page: ProductPage,
+        nav_page: ProductNavigationPage,
+    ):
+        yield {
+            "html": page.html,
+            "response_html": page.response.html,
+            "product": page.product,
+            "productNavigation": nav_page.product_nav,
+        }
+
+
+@ensureDeferred
+async def test_multiple_types(mockserver):
+    settings = create_scrapy_settings()
+    settings["ZYTE_API_URL"] = mockserver.urljoin("/")
+    settings["SCRAPY_POET_PROVIDERS"] = {ZyteApiProvider: 0}
+    item, url, _ = await crawl_single_item(
+        ZyteAPIMultipleSpider, HtmlResource, settings
+    )
+    assert item["html"] == "<html><body>Hello<h1>World!</h1></body></html>"
+    assert item["response_html"] == "<html><body>Hello<h1>World!</h1></body></html>"
+    assert item["product"] == Product.from_dict(
+        dict(
+            url=url,
+            name="Product name",
+            price="10",
+            currency="USD",
+        )
+    )
+    assert item["productNavigation"] == ProductNavigation.from_dict(
+        dict(
+            url=url,
+            name="Product navigation",
+            pageNumber=0,
+        )
+    )