diff --git a/docs/extra-features.rst b/docs/extra-features.rst deleted file mode 100644 index af4c9a3c3..000000000 --- a/docs/extra-features.rst +++ /dev/null @@ -1,10 +0,0 @@ -Extra features -============== - -MWDB provides additional features such as: - -* User registration features (hosting public services with vetting) -* Rate limiting -* Family statistics - -These will be documented in further releases. diff --git a/docs/index.rst b/docs/index.rst index 58e5602ed..d35fe8531 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,13 +22,13 @@ Features setup-and-configuration user-guide/index integration-guide - extra-features developer-guide remotes-guide karton-guide oauth-guide rich-attributes-guide prometheus-guide + rate-limits Indices and tables ================== diff --git a/docs/prometheus-guide.rst b/docs/prometheus-guide.rst index 0feedd600..2257a8ebf 100644 --- a/docs/prometheus-guide.rst +++ b/docs/prometheus-guide.rst @@ -5,6 +5,10 @@ Prometheus metrics MWDB allows to enable Prometheus metrics to grab information about API usage by users. +.. warning:: + + This feature requires Redis database to be configured. + Available metrics: - ``mwdb_api_requests (method, endpoint, user, status_code)`` that tracks usage of specific endpoints by users and status codes. diff --git a/docs/rate-limits.rst b/docs/rate-limits.rst new file mode 100644 index 000000000..7d395f53c --- /dev/null +++ b/docs/rate-limits.rst @@ -0,0 +1,72 @@ +Rate limit configuration +======================== + +.. versionadded:: 2.7.0 + +MWDB allows you to set a rate limit that gives more control over API usage by users. + +.. warning:: + + This feature requires Redis database to be configured. + +Global rate-limit configuration +------------------------------- + +To enable rate limiting, set ``enable_rate_limit`` option in `mwdb.ini`` or ``MWDB_ENABLE_RATE_LIMIT`` environment variable to ``1``. + +mwdb-core comes with hardcoded default limits that are applied depending on HTTP method. The default values are as below: + +* GET method: 1000/10second 2000/minute 6000/5minute 10000/15minute +* POST method: 100/10second 1000/minute 3000/5minute 6000/15minute +* PUT method: 100/10second 1000/minute 3000/5minute 6000/15minute +* DELETE method: 100/10second 1000/minute 3000/5minute 6000/15minute + +User can override these limits for individual endpoints by placing new limits in ``mwdb.ini`` - in section ``[mwdb_limiter]``. +Each line in ``[mwdb_limiter]`` section should have a pattern - ``_ = limit_values_space_separated``. + +Example rate-limit records in mwdb.ini file are as below + +.. code-block:: + + [mwdb_limiter] + file_get = 100/10second + textblob_post = 10/second 1000/minute 3000/15minute + attributedefinition_delete = 10/minute 100/hour + +Above records establish request rate limits for endpoints: + +* GET /api/file to value: 100 per 10 seconds +* POST /api/blob to values: 10 per second, 1000 per minute and 3000 per 15 minutes +* DELETE /api/attribute/ to values: 10 per minute and 100 per hour + +Other endpoints are limited by default limits. + +Limiter configuration follows the same rules as other configuration fields and can be set using environment variables e.g. +``MWDB_LIMITER_TEXTBLOB_POST="10/second 1000/minute 3000/15minute``. + +.. note:: + + Complete list of possible rate-limit parameters is placed in ``mwdb-core\mwdb\core\templates\mwdb.ini.tmpl`` file - section ``mwdb_limiter``. + + If your MWDB instance uses standalone installation and MWDB backend is behind reverse proxy, make sure that use_x_forwarded_for is set to 1 + and your reverse proxy correctly sets X-Forwarded-For header with real remote IP. + +Group-based rate limit configuration +------------------------------------ + +.. versionadded:: 2.14.0 + +mwdb-core in version v2.14.0 extends the limit key syntax and allows you to set custom rate limits for a specific group of users. + +Complete key format is: + +* ``(group_)_()_()`` - to set limits for members of group +* ``(unauthenticated)_()_()`` - to set limits only for unauthenticated users + +where any key in parentheses may be excluded to have a more generic key. + +Examples: + +* 10/second limit for members of group called "limited_users": ``group_limited_users = 10/second`` +* 10/second or 30/minute for members of "limited_users" but only for POST requests : ``group_limited_users_post = 10/second 30/minute`` +* 1/second for unauthenticated users and for all requests: ``unauthenticated = 1/second`` diff --git a/docs/setup-and-configuration.rst b/docs/setup-and-configuration.rst index decd7093a..7990be833 100644 --- a/docs/setup-and-configuration.rst +++ b/docs/setup-and-configuration.rst @@ -346,49 +346,6 @@ Registration feature settings: * ``recaptcha_site_key`` (string) - ReCAPTCHA site key. If not set - ReCAPTCHA won't be required for registration. * ``recaptcha_secret`` (string) - ReCAPTCHA secret key. If not set - ReCAPTCHA won't be required for registration. - -Rate limit configuration ------------------------- - -.. versionadded:: 2.7.0 - -mwdb-core service has implemented rate limiting feature. Each limit for HTTP method can contain a few conditions (space separated). - -Default limits were applied for HTTP methods. The default values are as below: - - -* GET method: 1000/10second 2000/minute 6000/5minute 10000/15minute -* POST method: 100/10second 1000/minute 3000/5minute 6000/15minute -* PUT method: 100/10second 1000/minute 3000/5minute 6000/15minute -* DELETE method: 100/10second 1000/minute 3000/5minute 6000/15minute - -User can override these limits for individual endpoints by placing new limits in ``mwdb.ini`` - in section ``[mwdb_limiter]``. -Each line in ``[mwdb_limiter]`` section should have a pattern - ``_ = limit_values_space_separated``. - -Example rate-limit records in mwdb.ini file are as below - -.. code-block:: - - [mwdb_limiter] - file_get = 100/10second - textblob_post = 10/second 1000/minute 3000/15minute - attributedefinition_delete = 10/minute 100/hour - -Above records establish request rate limits for endpoints: - -* GET /api/file to value: 100 per 10 seconds -* POST /api/blob to values: 10 per second, 1000 per minute and 3000 per 15 minutes -* DELETE /api/attribute/ to values: 10 per minute and 100 per hour - -Other endpoints are limited by default limits. - -.. note:: - - Complete list of possible rate-limit parameters is placed in ``mwdb-core\mwdb\core\templates\mwdb.ini.tmpl`` file - section ``mwdb_limiter``. - - If your MWDB instance uses standalone installation and MWDB backend is behind reverse proxy, make sure that use_x_forwarded_for is set to 1 - and your reverse proxy correctly sets X-Forwarded-For header with real remote IP. - Using MWDB in Kubernetes environment ------------------------------------ diff --git a/mwdb/core/rate_limit.py b/mwdb/core/rate_limit.py index 164043f19..42b1da551 100644 --- a/mwdb/core/rate_limit.py +++ b/mwdb/core/rate_limit.py @@ -1,5 +1,5 @@ import time -from typing import Optional +from typing import Iterator, List, Optional, Tuple from flask import g, request from limits import parse @@ -30,6 +30,53 @@ def get_limit_from_config(key) -> Optional[str]: return app_config.get_key("mwdb_limiter", key) or DEFAULT_RATE_LIMITS.get(key) +def get_limit_keys_for_request() -> List[Tuple[str, ...]]: + """ + Finds suitable limit keys for current request + """ + # Split blueprint name and resource name from endpoint + if request.endpoint: + _, resource_name = request.endpoint.split(".", 2) + else: + resource_name = None + + method = request.method.lower() + user_group_keys = ( + [f"group_{group}" for group in g.auth_user.group_names] + if g.auth_user is not None + else ["unauthenticated"] + ) + + # Limit keys from most specific to the least specific + if resource_name is not None: + resource_limit_keys = [(resource_name, method), (resource_name,), (method,)] + else: + resource_limit_keys = [(method,)] + + return ( + [ + (user_group, *resource_limit_key_items) + for user_group in user_group_keys + for resource_limit_key_items in resource_limit_keys + ] + + [(user_group,) for user_group in user_group_keys] + + resource_limit_keys + ) + + +def get_limits_for_request() -> Iterator[Tuple[Tuple[str, ...], List[str]]]: + """ + Finds suitable limits for current request + """ + limit_keys = get_limit_keys_for_request() + for limit_key in limit_keys: + # Get limit values for key + limit_values = get_limit_from_config("_".join(limit_key)) + if not limit_values: + continue + yield limit_key, limit_values.split(" ") + + def apply_rate_limit_for_request() -> bool: """ Raises TooManyRequests if current user has exceeded the rate limit @@ -49,24 +96,12 @@ def apply_rate_limit_for_request() -> bool: Capabilities.unlimited_requests ): return False - # Split blueprint name and resource name from endpoint - if request.endpoint: - _, resource_name = request.endpoint.split(".", 2) - else: - resource_name = "None" - method = request.method.lower() + limits = get_limits_for_request() user = g.auth_user.login if g.auth_user is not None else request.remote_addr - # Limit keys from most specific to the least specific - limit_keys = [[resource_name, method], [resource_name], [method]] - for limit_key in limit_keys: - # Get limit values for key - limit_values = get_limit_from_config("_".join(limit_key)) - if not limit_values: - continue - # limits has parse_many, but we're separating values using space - for limit_value in limit_values.split(" "): + for limit_key, limit_values in limits: + for limit_value in limit_values: limit_item = parse(limit_value) - identifiers = [user, *limit_key] + identifiers = (user, *limit_key) if not limiter.hit(limit_item, *identifiers): reset_time = limiter.get_window_stats( limit_item, *identifiers @@ -79,5 +114,5 @@ def apply_rate_limit_for_request() -> bool: raise TooManyRequests( retry_after=retry_after, description=f"Request limit: {limit_value} for " - f"{method} method was exceeded!", + f"{'_'.join(limit_key)} was exceeded!", )