Limit the size of manifests/signatures during sync

Adds limit to the size of manifests and signatures as a safeguard
to avoid DDoS attack during sync operations.
To also prevent this during image upload, this commit configures a
`client_max_body_size` for manifests and signatures Nginx endpoints.

closes: pulp#532

CHANGES/532.feature

@@ -0,0 +1,2 @@
+Added a limit to the size of manifests and signatures during sync tasks and
+updated the Nginx snippet to also limit the size of the body for these endpoints.

pulp_container/app/tasks/sync_stages.py

@@ -15,6 +15,7 @@
     MEDIA_TYPE,
     SIGNATURE_API_EXTENSION_VERSION,
     SIGNATURE_HEADER,
+    SIGNATURE_PAYLOAD_MAX_SIZE,
     SIGNATURE_SOURCE,
     SIGNATURE_TYPE,
     V2_ACCEPT_HEADERS,
@@ -35,8 +36,11 @@
     calculate_digest,
     filter_resources,
     get_content_data,
+    is_signature_size_valid,
 )
 
+from pulp_container.app.exceptions import ManifestSignatureInvalid
+
 log = logging.getLogger(__name__)
 
 
@@ -545,6 +549,14 @@ async def create_signatures(self, man_dc, signature_source):
                         "Error: {} {}".format(signature_url, exc.status, exc.message)
                     )
 
+                if not is_signature_size_valid(signature_download_result.path):
+                    log.info(
+                        "Signature size is not valid, the max allowed size is {}.".format(
+                            SIGNATURE_PAYLOAD_MAX_SIZE
+                        )
+                    )
+                    raise ManifestSignatureInvalid(digest=man_digest_reformatted)
+
                 with open(signature_download_result.path, "rb") as f:
                     signature_raw = f.read()
 
@@ -566,7 +578,12 @@ async def create_signatures(self, man_dc, signature_source):
             # signature extensions endpoint does not like any unnecessary headers to be sent
             await signatures_downloader.run(extra_data={"headers": {}})
             with open(signatures_downloader.path) as signatures_fd:
-                api_extension_signatures = json.loads(signatures_fd.read())
+                try:
+                    api_extension_signatures = json.loads(
+                        signatures_fd.read(SIGNATURE_PAYLOAD_MAX_SIZE)
+                    )
+                except json.decoder.JSONDecodeError:
+                    raise ManifestSignatureInvalid(digest=man_dc.content.digest)
             for signature in api_extension_signatures.get("signatures", []):
                 if (
                     signature.get("schemaVersion") == SIGNATURE_API_EXTENSION_VERSION

pulp_container/app/utils.py

@@ -13,13 +13,16 @@
 from django.core.files.storage import default_storage as storage
 from django.db import IntegrityError
 from functools import partial
+from pathlib import Path
 from rest_framework.exceptions import Throttled
 
 from pulpcore.plugin.models import Artifact, Task
 
 from pulp_container.constants import (
     MANIFEST_MEDIA_TYPES,
+    MANIFEST_PAYLOAD_MAX_SIZE,
     MEDIA_TYPE,
+    SIGNATURE_PAYLOAD_MAX_SIZE,
 )
 from pulp_container.app.exceptions import ManifestInvalid
 from pulp_container.app.json_schemas import (
@@ -221,6 +224,22 @@ def validate_manifest(content_data, media_type, digest):
             reason=f'{".".join(map(str, error.path))}: {error.message}', digest=digest
         )
 
+    manifests = content_data.get("manifests", None)
+    if manifests and not _is_manifest_size_valid(manifests):
+        raise ManifestInvalid(
+            reason="Manifest size is not valid, the max allowed size is {}.".format(
+                MANIFEST_PAYLOAD_MAX_SIZE
+            ),
+            digest=digest,
+        )
+
+
+def _is_manifest_size_valid(manifests):
+    for manifest in manifests:
+        if manifest.get("size") > MANIFEST_PAYLOAD_MAX_SIZE:
+            return False
+    return True
+
 
 def calculate_digest(manifest):
     """
@@ -342,3 +361,9 @@ def filter_resources(element_list, include_patterns, exclude_patterns):
     if exclude_patterns:
         element_list = filter(partial(exclude, patterns=exclude_patterns), element_list)
     return list(element_list)
+
+
+def is_signature_size_valid(file_path):
+    if Path(file_path).stat().st_size > SIGNATURE_PAYLOAD_MAX_SIZE:
+        return False
+    return True

pulp_container/app/webserver_snippets/nginx.conf

@@ -38,3 +38,25 @@ location /token/ {
     proxy_redirect off;
     proxy_pass http://pulp-api;
 }
+
+location ~* /v2/.*/manifests/.*$ {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+    # we don't want nginx trying to do something clever with
+    # redirects, we set the Host: header above already.
+    proxy_redirect off;
+    proxy_pass http://pulp-api;
+    client_max_body_size 4m;
+}
+
+location ~* /extensions/v2/.*/signatures/.*$ {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+    # we don't want nginx trying to do something clever with
+    # redirects, we set the Host: header above already.
+    proxy_redirect off;
+    proxy_pass http://pulp-api;
+    client_max_body_size 4m;
+}

pulp_container/constants.py

@@ -69,5 +69,6 @@
 
 MEGABYTE = 1_000_000
 SIGNATURE_PAYLOAD_MAX_SIZE = 4 * MEGABYTE
+MANIFEST_PAYLOAD_MAX_SIZE = 4 * MEGABYTE
 
 SIGNATURE_API_EXTENSION_VERSION = 2