From 47b41589ac437fc73819230914ebcf1d6d5e93fc Mon Sep 17 00:00:00 2001 From: Sergey Korolev Date: Sat, 20 Mar 2021 19:00:49 +0300 Subject: [PATCH 1/3] feat: support HTTPS --- insales/connection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/insales/connection.py b/insales/connection.py index a6c49d8..2446c6b 100644 --- a/insales/connection.py +++ b/insales/connection.py @@ -10,11 +10,11 @@ # Python 3 from urllib import parse as urlparse from urllib.parse import urlencode - from http.client import HTTPConnection, HTTPException + from http.client import HTTPConnection, HTTPSConnection, HTTPException except ImportError: # Python 2 import urlparse - from httplib import HTTPConnection, HTTPException + from httplib import HTTPConnection, HTTPSConnection, HTTPException from urllib import urlencode @@ -26,11 +26,13 @@ def __init__(self, msg, code=None): class Connection(object): def __init__(self, account, api_key, password, + secure=False, retry_on_503=False, retry_on_socket_error=False, retry_timeout=1, response_timeout=10): self.account = account self.api_key = api_key self.password = password + self.secure = secure self.retry_on_503 = retry_on_503 self.retry_on_socket_error = retry_on_socket_error self.retry_timeout = retry_timeout @@ -47,8 +49,11 @@ def request(self, method, endpoint, qargs={}, data=None): done = False while not done: try: - conn = HTTPConnection('%s.myinsales.ru:80' % self.account, - timeout=self.response_timeout) + host = '%s.myinsales.ru' % self.account + if self.secure: + conn = HTTPSConnection(host, timeout=self.response_timeout) + else: + conn = HTTPConnection(host, timeout=self.response_timeout) conn.request(method, path, headers=headers, body=data) resp = conn.getresponse() body = resp.read() From 7227ee4d41b781e1eae4a5955c218602834594e0 Mon Sep 17 00:00:00 2001 From: Sergey Korolev Date: Sat, 20 Mar 2021 19:01:13 +0300 Subject: [PATCH 2/3] tweak: decode error messages --- insales/connection.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/insales/connection.py b/insales/connection.py index 2446c6b..16b2b32 100644 --- a/insales/connection.py +++ b/insales/connection.py @@ -72,8 +72,15 @@ def request(self, method, endpoint, qargs={}, data=None): if 200 <= resp.status < 300: return body else: - raise ApiError("%s request to %s returned: %s\n%s" % - (method, path, resp.status, body), resp.status) + raise ApiError( + "{} request to {} returned: {}\n{}".format( + method, + path, + resp.status, + body.decode('utf-8') if type(body) == bytes else body + ), + resp.status + ) def format_path(self, endpoint, qargs): for key, val in qargs.items(): From 0a7010a89518308b4e54c02e37712b3b92a99a7a Mon Sep 17 00:00:00 2001 From: Sergey Korolev Date: Sat, 20 Mar 2021 19:02:40 +0300 Subject: [PATCH 3/3] feat: support Retry-After, API-Usage-Limit --- insales/connection.py | 76 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/insales/connection.py b/insales/connection.py index 16b2b32..b354d25 100644 --- a/insales/connection.py +++ b/insales/connection.py @@ -2,6 +2,7 @@ import datetime import time +import threading import socket from base64 import b64encode @@ -18,6 +19,13 @@ from urllib import urlencode +insales_lock = threading.Lock() + + +def throttle_fn(curr: int, limit: int): + return 0.1 + (1.0 - (limit - curr)/limit)**16 * 8.5 + + class ApiError(Exception): def __init__(self, msg, code=None): super(ApiError, self).__init__(msg) @@ -28,15 +36,65 @@ class Connection(object): def __init__(self, account, api_key, password, secure=False, retry_on_503=False, retry_on_socket_error=False, - retry_timeout=1, response_timeout=10): + retry_timeout=1, response_timeout=10, + throttle=False): self.account = account self.api_key = api_key self.password = password self.secure = secure self.retry_on_503 = retry_on_503 self.retry_on_socket_error = retry_on_socket_error - self.retry_timeout = retry_timeout + self.retry_timeout = datetime.timedelta(seconds=retry_timeout) self.response_timeout = response_timeout + self.last_req_time = datetime.datetime.now() + self.retry_after = datetime.datetime.now() + self.max_wait_time = datetime.timedelta(seconds=60) + self.throttle = throttle != False + self.throttle_fn = throttle if callable(throttle) else throttle_fn + + def get_retry_after(self): + return self.retry_after + + def _set_retry_after(self, delta): + with insales_lock: + self.retry_after = self.last_req_time + min( + delta, self.max_wait_time + ) + + def _increase_retry_after(self, delta): + with insales_lock: + self.retry_after = max(self.last_req_time, self.retry_after) + min( + delta, self.max_wait_time + ) + + def _apply_retry_timeout(self): + self._set_retry_after(self.retry_timeout) + + def _handle_retry_after_header(self, header): + try: + self._set_retry_after(datetime.timedelta(seconds=int(header))) + except ValueError: + self._apply_retry_timeout() + + def _apply_usage_limit(self, header): + if header is None: + return + + try: + curr_str, limit_str = header.split("/") + delta = datetime.timedelta(seconds=self.throttle_fn( + int(curr_str), + int(limit_str), + )) + self._increase_retry_after(delta) + except ValueError: + pass + + def _wait_until_retry_after(self): + delta = self.retry_after - datetime.datetime.now() + if delta.total_seconds() > 0: + time.sleep(delta.total_seconds()) + def request(self, method, endpoint, qargs={}, data=None): path = self.format_path(endpoint, qargs) @@ -48,24 +106,34 @@ def request(self, method, endpoint, qargs={}, data=None): done = False while not done: + self._wait_until_retry_after() try: host = '%s.myinsales.ru' % self.account if self.secure: conn = HTTPSConnection(host, timeout=self.response_timeout) else: conn = HTTPConnection(host, timeout=self.response_timeout) + with insales_lock: + self.last_req_time = datetime.datetime.now() conn.request(method, path, headers=headers, body=data) resp = conn.getresponse() body = resp.read() except (socket.gaierror, socket.timeout, HTTPException): if self.retry_on_socket_error: - time.sleep(self.retry_timeout) + self._apply_retry_timeout() continue else: raise + if self.throttle: + self._apply_usage_limit(resp.getheader('API-Usage-Limit')) + if resp.status == 503 and self.retry_on_503: - time.sleep(self.retry_timeout) + retry_after_header = resp.getheader('Retry-After') + if retry_after_header: + self._handle_retry_after_header(retry_after_header) + else: + self._apply_retry_timeout() else: done = True