From 45a9ead39e25e5d65b5332590814e9853258aec3 Mon Sep 17 00:00:00 2001 From: Lyonnet Date: Tue, 17 Dec 2024 16:00:14 +0800 Subject: [PATCH] Update --- .github/ISSUE_TEMPLATE/bug_report.md | 3 +- curl_cffi/requests/options.py | 13 +------- curl_cffi/requests/session.py | 28 +++------------- curl_cffi/requests/websockets.py | 50 +++++++++++++++------------- pyproject.toml | 2 +- 5 files changed, 35 insertions(+), 61 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b8381c4..aa891e4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,7 +6,8 @@ labels: bug --- -Please check the following items before reporting a bug, otherwise it may be closed immediately. +Please check the following items and answer all the questions when reporting a bug, +otherwise it will be closed immediately. - [ ] **This is NOT a site-related "bugs"**, e.g. some site blocks me when using curl_cffi, UNLESS it has been verified that the reason is missing pieces in the impersonation. diff --git a/curl_cffi/requests/options.py b/curl_cffi/requests/options.py index f816fa0..968335e 100644 --- a/curl_cffi/requests/options.py +++ b/curl_cffi/requests/options.py @@ -7,18 +7,7 @@ from collections import Counter from io import BytesIO from json import dumps -from typing import ( - TYPE_CHECKING, - Any, - Callable, - cast, - Dict, - List, - Literal, - Optional, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, Union, cast from urllib.parse import ParseResult, parse_qsl, quote, unquote, urlencode, urljoin, urlparse from .. import CurlHttpVersion, CurlOpt, CurlSslVersion diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 930bb8c..69bc30c 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -22,18 +22,15 @@ Union, cast, ) -from urllib.parse import ParseResult, parse_qsl, quote, urlencode, urlparse +from urllib.parse import urlparse -from .. import AsyncCurl, Curl, CurlError, CurlHttpVersion, CurlInfo, CurlOpt, CurlSslVersion -from ..curl import CURL_WRITEFUNC_ERROR, CurlMime +from ..aio import AsyncCurl +from ..const import CurlHttpVersion, CurlInfo, CurlOpt +from ..curl import Curl, CurlError, CurlMime from .cookies import Cookies, CookieTypes, CurlMorsel from .exceptions import RequestException, SessionClosed, code2error from .headers import Headers, HeaderTypes -from .impersonate import ( - BrowserTypeLiteral, - ExtraFingerprints, - ExtraFpDict, -) +from .impersonate import BrowserTypeLiteral, ExtraFingerprints, ExtraFpDict from .models import Response from .options import set_curl_options, update_url_params from .websockets import AsyncWebSocket @@ -99,21 +96,6 @@ def _is_absolute_url(url: str) -> bool: SAFE_CHARS = set("!#$%&'()*+,/:;=?@[]~") -def _quote_path_and_params(url: str, quote_str: str = ""): - safe = "".join(SAFE_CHARS - set(quote_str)) - parsed_url = urlparse(url) - parsed_get_args = parse_qsl(parsed_url.query, keep_blank_values=True) - encoded_get_args = urlencode(parsed_get_args, doseq=True, safe=safe) - return ParseResult( - parsed_url.scheme, - parsed_url.netloc, - quote(parsed_url.path, safe=safe), - parsed_url.params, - encoded_get_args, - parsed_url.fragment, - ).geturl() - - def _peek_queue(q: queue.Queue, default=None): try: return q.queue[0] diff --git a/curl_cffi/requests/websockets.py b/curl_cffi/requests/websockets.py index a5afab1..3837916 100644 --- a/curl_cffi/requests/websockets.py +++ b/curl_cffi/requests/websockets.py @@ -161,7 +161,7 @@ def __init__( on_error: Optional[ON_ERROR_T] = None, ): """ - Parameters: + Args: autoclose: whether to close the WebSocket after receiving a close frame. skip_utf8_validation: whether to skip UTF-8 validation for text frames in run_forever(). debug: print extra curl debug info. @@ -242,7 +242,7 @@ def connect( libcurl automatically handles pings and pongs. ref: https://curl.se/libcurl/c/libcurl-ws.html - Parameters: + Args: url: url for the requests. params: query string for the requests. headers: headers to send. @@ -339,7 +339,7 @@ def recv(self) -> Tuple[bytes, int]: """ Receive a frame as bytes. - libcurl split frames into fragments, so we have to collect all the chunks for + libcurl splits frames into fragments, so we have to collect all the chunks for a frame. """ chunks = [] @@ -375,7 +375,7 @@ def recv_str(self) -> str: def recv_json(self, *, loads: Callable[[str], T] = loads) -> T: """Receive a JSON frame. - Parameters: + args: loads: JSON decoder, default is json.loads. """ data = self.recv_str() @@ -384,7 +384,7 @@ def recv_json(self, *, loads: Callable[[str], T] = loads) -> T: def send(self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag.BINARY): """Send a data frame. - Parameters: + Args: payload: data to send. flags: flags for the frame. """ @@ -399,7 +399,7 @@ def send(self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag.BINARY def send_binary(self, payload: bytes): """Send a binary frame. - Parameters: + Args: payload: binary data to send. """ return self.send(payload, CurlWsFlag.BINARY) @@ -407,7 +407,7 @@ def send_binary(self, payload: bytes): def send_bytes(self, payload: bytes): """Send a binary frame. Same as :meth:`send_binary`. - Parameters: + Args: payload: binary data to send. """ return self.send(payload, CurlWsFlag.BINARY) @@ -415,7 +415,7 @@ def send_bytes(self, payload: bytes): def send_str(self, payload: str): """Send a text frame. - Parameters: + Args: payload: text data to send. """ return self.send(payload, CurlWsFlag.TEXT) @@ -423,7 +423,7 @@ def send_str(self, payload: str): def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps): """Send a JSON frame. - Parameters: + Args: payload: data to send. dumps: JSON encoder, default is json.dumps. """ @@ -432,7 +432,7 @@ def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps): def ping(self, payload: Union[str, bytes]): """Send a ping frame. - Parameters: + Args: payload: data to send. """ return self.send(payload, CurlWsFlag.PING) @@ -497,7 +497,7 @@ def run_forever(self, url: str, **kwargs): def close(self, code: int = WsCloseCode.OK, message: bytes = b""): """Close the connection. - Parameters: + Args: code: close code. message: close reason. """ @@ -542,7 +542,7 @@ async def __anext__(self) -> bytes: async def recv_fragment(self, *, timeout: Optional[float] = None) -> Tuple[bytes, CurlWsFrame]: """Receive a single frame as bytes. - Parameters: + Args: timeout: how many seconds to wait before giving up. """ if self.closed: @@ -559,7 +559,9 @@ async def recv_fragment(self, *, timeout: Optional[float] = None) -> Tuple[bytes raise WebSocketTimeout("WebSocket recv_fragment() timed out") if frame.flags & CurlWsFlag.CLOSE: try: - code, message = self._close_code, self._close_reason = self._unpack_close_frame(chunk) + code, message = self._close_code, self._close_reason = self._unpack_close_frame( + chunk + ) except WebSocketError as e: # Follow the spec to close the connection # Errors do not respect autoclose @@ -575,10 +577,10 @@ async def recv(self, *, timeout: Optional[float] = None) -> Tuple[bytes, int]: """ Receive a frame as bytes. - libcurl split frames into fragments, so we have to collect all the chunks for + libcurl splits frames into fragments, so we have to collect all the chunks for a frame. - Parameters: + Args: timeout: how many seconds to wait before giving up. """ loop = self.loop @@ -608,7 +610,7 @@ async def recv(self, *, timeout: Optional[float] = None) -> Tuple[bytes, int]: async def recv_str(self, *, timeout: Optional[float] = None) -> str: """Receive a text frame. - Parameters: + Args: timeout: how many seconds to wait before giving up. """ data, flags = await self.recv(timeout=timeout) @@ -621,7 +623,7 @@ async def recv_json( ) -> T: """Receive a JSON frame. - Parameters: + Args: loads: JSON decoder, default is json.loads. timeout: how many seconds to wait before giving up. """ @@ -631,7 +633,7 @@ async def recv_json( async def send(self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag.BINARY): """Send a data frame. - Parameters: + Args: payload: data to send. flags: flags for the frame. """ @@ -649,7 +651,7 @@ async def send(self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag. async def send_binary(self, payload: bytes): """Send a binary frame. - Parameters: + Args: payload: binary data to send. """ return await self.send(payload, CurlWsFlag.BINARY) @@ -657,7 +659,7 @@ async def send_binary(self, payload: bytes): async def send_bytes(self, payload: bytes): """Send a binary frame. Same as :meth:`send_binary`. - Parameters: + Args: payload: binary data to send. """ return await self.send(payload, CurlWsFlag.BINARY) @@ -665,7 +667,7 @@ async def send_bytes(self, payload: bytes): async def send_str(self, payload: str): """Send a text frame. - Parameters: + Args: payload: text data to send. """ return await self.send(payload, CurlWsFlag.TEXT) @@ -673,7 +675,7 @@ async def send_str(self, payload: str): async def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps): """Send a JSON frame. - Parameters: + Args: payload: data to send. dumps: JSON encoder, default is json.dumps. """ @@ -682,7 +684,7 @@ async def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps): async def ping(self, payload: Union[str, bytes]): """Send a ping frame. - Parameters: + Args: payload: data to send. """ return await self.send(payload, CurlWsFlag.PING) @@ -690,7 +692,7 @@ async def ping(self, payload: Union[str, bytes]): async def close(self, code: int = WsCloseCode.OK, message: bytes = b""): """Close the connection. - Parameters: + Args: code: close code. message: close reason. """ diff --git a/pyproject.toml b/pyproject.toml index 6d68086..34dc4bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.8.0b7" +version = "0.8.1b1" authors = [{ name = "Lyonnet", email = "infinitesheldon@gmail.com" }] description = "libcurl ffi bindings for Python, with impersonation support." license = { file = "LICENSE" }