Add validation for uploaded manifests

closes #854 closes #853 closes #672
pulp · Jul 4, 2022 · d24891b · d24891b
1 parent ada3f58
commit d24891b
Show file tree

Hide file tree

Showing 8 changed files with 305 additions and 45 deletions.
diff --git a/CHANGES/672.feature b/CHANGES/672.feature
@@ -0,0 +1 @@
+Added validation for uploaded manifest JSON content.
diff --git a/CHANGES/853.bugfix b/CHANGES/853.bugfix
@@ -0,0 +1,2 @@
+Fixed internal server errors raised when a podman client (<4.0) used invalid content types for
+manifest lists.
diff --git a/CHANGES/854.bugfix b/CHANGES/854.bugfix
@@ -0,0 +1 @@
+Fixed a misleading error message raised when a user provided an invalid manifest list.
diff --git a/pulp_container/app/json_schemas.py b/pulp_container/app/json_schemas.py
@@ -1,54 +1,251 @@
-SIGNATURE_SCHEMA = """{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://example.com/product.schema.json",
+def get_descriptor_schema(
+    allowed_media_types, additional_properties=None, additional_required=None
+):
+    """Return a concrete descriptor schema for manifests."""
+    properties = {
+        "mediaType": {"type": "string", "enum": allowed_media_types},
+        "size": {"type": "number"},
+        "digest": {"type": "string"},
+        "annotations": {"type": "object", "additionalProperties": True},
+        "urls": {"type": "array", "items": {"type": "string"}},
+    }
+
+    if additional_properties:
+        properties.update(additional_properties)
+
+    required = ["mediaType", "size", "digest"]
+    if additional_required:
+        required.extend(additional_required)
+
+    return {"type": "object", "properties": properties, "required": required}
+
+
+OCI_INDEX_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.oci.image.index.v1+json"],
+        },
+        "manifests": {
+            "type": "array",
+            "items": get_descriptor_schema(
+                allowed_media_types=[
+                    "application/vnd.oci.image.manifest.v1+json",
+                    "application/vnd.oci.image.index.v1+json",
+                ],
+                additional_properties={
+                    "platform": {
+                        "type": "object",
+                        "properties": {
+                            "architecture": {"type": "string"},
+                            "os": {"type": "string"},
+                            "os.version": {"type": "string"},
+                            "os.features": {"type": "array", "items": {"type": "string"}},
+                            "variant": {"type": "string"},
+                            "features": {"type": "array", "items": {"type": "string"}},
+                        },
+                        "required": ["architecture", "os"],
+                    },
+                },
+                additional_required=["platform"],
+            ),
+        },
+        "annotations": {"type": "object", "additionalProperties": True},
+    },
+    "required": ["schemaVersion", "manifests"],
+}
+
+OCI_MANIFEST_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.oci.image.manifest.v1+json"],
+        },
+        "config": get_descriptor_schema(["application/vnd.oci.image.config.v1+json"]),
+        "layers": {
+            "type": "array",
+            "items": get_descriptor_schema(
+                [
+                    "application/vnd.oci.image.layer.v1.tar",
+                    "application/vnd.oci.image.layer.v1.tar+gzip",
+                    "application/vnd.oci.image.layer.v1.tar+zstd",
+                    "application/vnd.oci.image.layer.nondistributable.v1.tar",
+                    "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+                ]
+            ),
+        },
+    },
+    "required": ["schemaVersion", "config", "layers"],
+}
+
+DOCKER_MANIFEST_LIST_V2_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.docker.distribution.manifest.list.v2+json"],
+        },
+        "manifests": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "mediaType": {
+                        "type": "string",
+                        "enum": [
+                            "application/vnd.docker.distribution.manifest.v2+json",
+                            "application/vnd.docker.distribution.manifest.v1+json",
+                        ],
+                    },
+                    "size": {"type": "number"},
+                    "digest": {"type": "string"},
+                    "platform": {
+                        "type": "object",
+                        "properties": {
+                            "architecture": {"type": "string"},
+                            "os": {"type": "string"},
+                            "os.version": {"type": "string"},
+                            "os.features": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                            },
+                            "variant": {"type": "string"},
+                            "features": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                            },
+                        },
+                        "required": ["architecture", "os"],
+                    },
+                },
+                "required": ["mediaType", "size", "digest", "platform"],
+            },
+        },
+    },
+    "required": ["schemaVersion", "mediaType", "manifests"],
+}
+
+DOCKER_MANIFEST_V2_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.docker.distribution.manifest.v2+json"],
+        },
+        "config": {
+            "type": "object",
+            "properties": {
+                "mediaType": {
+                    "type": "string",
+                    "enum": ["application/vnd.docker.container.image.v1+json"],
+                },
+                "size": {"type": "number"},
+                "digest": {"type": "string"},
+            },
+            "required": ["mediaType", "size", "digest"],
+        },
+        "layers": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "mediaType:": {
+                        "type": "string",
+                        "enum": [
+                            "application/vnd.docker.image.rootfs.diff.tar.gzip",
+                            "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+                        ],
+                    },
+                    "size": {"type": "number"},
+                    "digest": {"type": "string"},
+                },
+                "required": ["mediaType", "size", "digest"],
+            },
+        },
+    },
+    "required": ["schemaVersion", "mediaType", "config", "layers"],
+}
+
+DOCKER_MANIFEST_V1_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "signatures": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "protected": {"type": "string"},
+                    "header": {
+                        "type": "object",
+                        "properties": {"alg": {"type": "string"}, "jwk": {"type": "object"}},
+                        "required": ["alg", "jwk"],
+                    },
+                    "signature": {"type": "string"},
+                },
+                "required": ["protected", "header", "signature"],
+            },
+        },
+        "tag": {"type": "string"},
+        "name": {"type": "string"},
+        "history": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {"v1Compatibility": {"type": "string"}},
+                "required": ["v1Compatibility"],
+            },
+        },
+        "fsLayers": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {"blobSum": {"type": "string"}},
+                "required": ["blobSum"],
+            },
+        },
+    },
+    "required": ["tag", "name", "fsLayers", "history"],
+}
+
+SIGNATURE_SCHEMA = {
     "title": "Atomic Container Signature",
     "description": "JSON Schema Validation for the Signature Payload",
     "type": "object",
     "properties": {
         "critical": {
             "type": "object",
             "properties": {
-                "type": {
-                    "type": "string",
-                    "const": "atomic container signature"
-                },
+                "type": {"type": "string", "const": "atomic container signature"},
                 "image": {
                     "type": "object",
-                    "properties": {
-                        "docker-manifest-digest": {
-                            "type": "string"
-                        }
-                    },
+                    "properties": {"docker-manifest-digest": {"type": "string"}},
                     "required": ["docker-manifest-digest"],
-                    "additionalProperties": false
+                    "additionalProperties": False,
                 },
                 "identity": {
                     "type": "object",
-                    "properties": {
-                        "docker-reference": {
-                            "type": "string"
-                        }
-                    },
+                    "properties": {"docker-reference": {"type": "string"}},
                     "required": ["docker-reference"],
-                    "additionalProperties": false
-                }
+                    "additionalProperties": False,
+                },
             },
             "required": ["type", "image", "identity"],
-            "additionalProperties": false
+            "additionalProperties": False,
         },
         "optional": {
             "type": "object",
             "properties": {
-                "creator": {
-                    "type": "string"
-                },
-                "timestamp": {
-                    "type": "number",
-                    "minimum": 0
-                }
-            }
-        }
+                "creator": {"type": "string"},
+                "timestamp": {"type": "number", "minimum": 0},
+            },
+        },
     },
     "required": ["critical", "optional"],
-    "additionalProperties": false
-}"""
+    "additionalProperties": False,
+}
diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py
@@ -23,6 +23,7 @@
 from django.shortcuts import get_object_or_404
 
 from django.conf import settings
+from jsonschema import validate, ValidationError as SchemaValidationError
 
 from pulpcore.plugin.models import Artifact, ContentArtifact, Task, UploadChunk
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
@@ -71,7 +72,11 @@
     RegistryPermission,
     TokenPermission,
 )
-from pulp_container.app.utils import extract_data_from_signature, has_task_completed
+from pulp_container.app.utils import (
+    determine_schema,
+    extract_data_from_signature,
+    has_task_completed,
+)
 from pulp_container.constants import (
     EMPTY_BLOB,
     SIGNATURE_API_EXTENSION_VERSION,
@@ -867,14 +872,6 @@ def put(self, request, path, pk=None):
         """
         Responds with the actual manifest
         """
-        # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded
-        # before) and the specified repository has not been created yet, create the repository
-        # without raising an error
-        create_new_repo = request.content_type in (
-            models.MEDIA_TYPE.MANIFEST_LIST,
-            models.MEDIA_TYPE.INDEX_OCI,
-        )
-        _, repository = self.get_dr_push(request, path, create=create_new_repo)
         # iterate over all the layers and create
         chunk = request.META["wsgi.input"]
         artifact = self.receive_artifact(chunk)
@@ -895,6 +892,20 @@ def put(self, request, path, pk=None):
 
         content_data = json.loads(raw_data)
 
+        try:
+            self.validate_manifest(content_data, request.content_type)
+        except SchemaValidationError:
+            raise ManifestInvalid(digest=manifest_digest)
+
+        # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded
+        # before) and the specified repository has not been created yet, create the repository
+        # without raising an error
+        create_new_repo = request.content_type in (
+            models.MEDIA_TYPE.MANIFEST_LIST,
+            models.MEDIA_TYPE.INDEX_OCI,
+        )
+        _, repository = self.get_dr_push(request, path, create=create_new_repo)
+
         if request.content_type in (
             models.MEDIA_TYPE.MANIFEST_LIST,
             models.MEDIA_TYPE.INDEX_OCI,
@@ -1019,6 +1030,15 @@ def put(self, request, path, pk=None):
         if has_task_completed(dispatched_task):
             return ManifestResponse(manifest, path, request, status=201)
 
+    def validate_manifest(self, content_data, content_type):
+        """Validate JSON data (manifest) according to its declared content type (e.g., list)."""
+        try:
+            schema = determine_schema(content_type)
+        except ValueError:
+            raise ValidationError()
+        else:
+            validate(content_data, schema)
+
     def _save_manifest(self, artifact, manifest_digest, content_type, config_blob=None):
         manifest = models.Manifest(
             digest=manifest_digest,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Added validation for uploaded manifest JSON content.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Fixed internal server errors raised when a podman client (<4.0) used invalid content types for
		manifest lists.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fixed a misleading error message raised when a user provided an invalid manifest list.