Fix passing headers in a wrong format to scrapy.responsetypes.respons…

…etypes.
scrapy-plugins · Nov 4, 2024 · 86c5b67 · 86c5b67
1 parent a8fa49f
commit 86c5b67
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/scrapy_zyte_api/responses.py b/scrapy_zyte_api/responses.py
@@ -194,8 +194,11 @@ def _process_response(
         return ZyteAPITextResponse.from_api_response(api_response, request=request)
 
     if api_response.get("httpResponseHeaders") and api_response.get("httpResponseBody"):
+        scrapy_headers: Dict[bytes, bytes] = {}
+        for header in cast(List[Dict[str, str]], api_response["httpResponseHeaders"]):
+            scrapy_headers[header["name"].encode()] = header["value"].encode()
         response_cls = responsetypes.from_args(
-            headers=cast(List[Dict[str, str]], api_response["httpResponseHeaders"]),
+            headers=scrapy_headers,
             url=cast(str, api_response["url"]),
             # FIXME: update this when python-zyte-api supports base64 decoding
             body=b64decode(api_response["httpResponseBody"]),  # type: ignore

diff --git a/tests/test_responses.py b/tests/test_responses.py
@@ -64,7 +64,7 @@ def raw_api_response_browser():
         "echoData": {"some_value": "here"},
         "httpResponseHeaders": [
             {"name": "Content-Type", "value": "text/html"},
-            {"name": "Content-Length", "value": len(PAGE_CONTENT)},
+            {"name": "Content-Length", "value": str(len(PAGE_CONTENT))},
         ],
         "statusCode": 200,
         "experimental": {
@@ -80,7 +80,7 @@ def raw_api_response_body():
         "echoData": {"some_value": "here"},
         "httpResponseHeaders": [
             {"name": "Content-Type", "value": "text/html"},
-            {"name": "Content-Length", "value": len(PAGE_CONTENT)},
+            {"name": "Content-Length", "value": str(len(PAGE_CONTENT))},
         ],
         "statusCode": 200,
         "experimental": {
@@ -97,7 +97,7 @@ def raw_api_response_mixed():
         "echoData": {"some_value": "here"},
         "httpResponseHeaders": [
             {"name": "Content-Type", "value": "text/html"},
-            {"name": "Content-Length", "value": len(PAGE_CONTENT_2)},
+            {"name": "Content-Length", "value": str(len(PAGE_CONTENT_2))},
         ],
         "statusCode": 200,
         "experimental": {
@@ -200,7 +200,7 @@ def test_non_utf8_response():
         "browserHtml": content,
         "httpResponseHeaders": [
             {"name": "Content-Type", "value": "text/html; charset=iso-8859-1"},
-            {"name": "Content-Length", "value": len(content)},
+            {"name": "Content-Length", "value": str(len(content))},
         ],
     }