Add validation for uploaded manifests

closes pulp#854
closes pulp#853
closes pulp#672

CHANGES/672.feature

@@ -0,0 +1 @@
+Added validation for uploaded manifest JSON content.

CHANGES/853.bugfix

@@ -0,0 +1,2 @@
+Fixed internal server errors raised when a podman client (<4.0) used invalid content types for
+manifest lists.

CHANGES/854.bugfix

@@ -0,0 +1 @@
+Fixed a misleading error message raised when a user provided an invalid manifest list.

pulp_container/app/json_schemas.py

@@ -1,54 +1,210 @@
-SIGNATURE_SCHEMA = """{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://example.com/product.schema.json",
+def get_descriptor_schema(
+    allowed_media_types, additional_properties=None, additional_required=None
+):
+    """Return a concrete descriptor schema for manifests."""
+    properties = {
+        "mediaType": {"type": "string", "enum": allowed_media_types},
+        "size": {"type": "number"},
+        "digest": {"type": "string"},
+        "annotations": {"type": "object", "additionalProperties": True},
+        "urls": {"type": "array", "items": {"type": "string"}},
+    }
+
+    if additional_properties:
+        properties.update(additional_properties)
+
+    required = ["mediaType", "size", "digest"]
+    if additional_required:
+        required.extend(additional_required)
+
+    return {"type": "object", "properties": properties, "required": required}
+
+
+OCI_INDEX_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.oci.image.index.v1+json"],
+        },
+        "manifests": {
+            "type": "array",
+            "items": get_descriptor_schema(
+                allowed_media_types=[
+                    "application/vnd.oci.image.manifest.v1+json",
+                    "application/vnd.oci.image.index.v1+json",
+                ],
+                additional_properties={
+                    "platform": {
+                        "type": "object",
+                        "properties": {
+                            "architecture": {"type": "string"},
+                            "os": {"type": "string"},
+                            "os.version": {"type": "string"},
+                            "os.features": {"type": "array", "items": {"type": "string"}},
+                            "variant": {"type": "string"},
+                            "features": {"type": "array", "items": {"type": "string"}},
+                        },
+                        "required": ["architecture", "os"],
+                    },
+                },
+                additional_required=["platform"],
+            ),
+        },
+        "annotations": {"type": "object", "additionalProperties": True},
+    },
+    "required": ["schemaVersion", "manifests"],
+}
+
+OCI_MANIFEST_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.oci.image.manifest.v1+json"],
+        },
+        "config": get_descriptor_schema(["application/vnd.oci.image.config.v1+json"]),
+        "layers": {
+            "type": "array",
+            "items": get_descriptor_schema(
+                [
+                    "application/vnd.oci.image.layer.v1.tar",
+                    "application/vnd.oci.image.layer.v1.tar+gzip",
+                    "application/vnd.oci.image.layer.v1.tar+zstd",
+                    "application/vnd.oci.image.layer.nondistributable.v1.tar",
+                    "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+                ]
+            ),
+        },
+    },
+    "required": ["schemaVersion", "config", "layers"],
+}
+
+DOCKER_MANIFEST_LIST_V2_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.docker.distribution.manifest.list.v2+json"],
+        },
+        "manifests": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "mediaType": {
+                        "type": "string",
+                        "enum": [
+                            "application/vnd.docker.distribution.manifest.v2+json",
+                            "application/vnd.docker.distribution.manifest.v1+json",
+                        ],
+                    },
+                    "size": {"type": "number"},
+                    "digest": {"type": "string"},
+                    "platform": {
+                        "type": "object",
+                        "properties": {
+                            "architecture": {"type": "string"},
+                            "os": {"type": "string"},
+                            "os.version": {"type": "string"},
+                            "os.features": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                            },
+                            "variant": {"type": "string"},
+                            "features": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                            },
+                        },
+                        "required": ["architecture", "os"],
+                    },
+                },
+                "required": ["mediaType", "size", "digest", "platform"],
+            },
+        },
+    },
+    "required": ["schemaVersion", "mediaType", "manifests"],
+}
+
+DOCKER_MANIFEST_V2_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2},
+        "mediaType": {
+            "type": "string",
+            "enum": ["application/vnd.docker.distribution.manifest.v2+json"],
+        },
+        "config": {
+            "type": "object",
+            "properties": {
+                "mediaType": {
+                    "type": "string",
+                    "enum": ["application/vnd.docker.container.image.v1+json"],
+                },
+                "size": {"type": "number"},
+                "digest": {"type": "string"},
+            },
+            "required": ["mediaType", "size", "digest"],
+        },
+        "layers": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "mediaType:": {
+                        "type": "string",
+                        "enum": [
+                            "application/vnd.docker.image.rootfs.diff.tar.gzip",
+                            "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+                        ],
+                    },
+                    "size": {"type": "number"},
+                    "digest": {"type": "string"},
+                },
+                "required": ["mediaType", "size", "digest"],
+            },
+        },
+    },
+    "required": ["schemaVersion", "mediaType", "config", "layers"],
+}
+
+SIGNATURE_SCHEMA = {
     "title": "Atomic Container Signature",
     "description": "JSON Schema Validation for the Signature Payload",
     "type": "object",
     "properties": {
         "critical": {
             "type": "object",
             "properties": {
-                "type": {
-                    "type": "string",
-                    "const": "atomic container signature"
-                },
+                "type": {"type": "string", "const": "atomic container signature"},
                 "image": {
                     "type": "object",
-                    "properties": {
-                        "docker-manifest-digest": {
-                            "type": "string"
-                        }
-                    },
+                    "properties": {"docker-manifest-digest": {"type": "string"}},
                     "required": ["docker-manifest-digest"],
-                    "additionalProperties": false
+                    "additionalProperties": False,
                 },
                 "identity": {
                     "type": "object",
-                    "properties": {
-                        "docker-reference": {
-                            "type": "string"
-                        }
-                    },
+                    "properties": {"docker-reference": {"type": "string"}},
                     "required": ["docker-reference"],
-                    "additionalProperties": false
-                }
+                    "additionalProperties": False,
+                },
             },
             "required": ["type", "image", "identity"],
-            "additionalProperties": false
+            "additionalProperties": False,
         },
         "optional": {
             "type": "object",
             "properties": {
-                "creator": {
-                    "type": "string"
-                },
-                "timestamp": {
-                    "type": "number",
-                    "minimum": 0
-                }
-            }
-        }
+                "creator": {"type": "string"},
+                "timestamp": {"type": "number", "minimum": 0},
+            },
+        },
     },
     "required": ["critical", "optional"],
-    "additionalProperties": false
-}"""
+    "additionalProperties": False,
+}

pulp_container/app/registry_api.py

@@ -23,6 +23,7 @@
 from django.shortcuts import get_object_or_404
 
 from django.conf import settings
+from jsonschema import validate, ValidationError as SchemaValidationError
 
 from pulpcore.plugin.models import Artifact, ContentArtifact, Task, UploadChunk
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
@@ -59,6 +60,12 @@
     ManifestInvalid,
     ManifestSignatureInvalid,
 )
+from pulp_container.app.json_schemas import (
+    OCI_INDEX_SCHEMA,
+    OCI_MANIFEST_SCHEMA,
+    DOCKER_MANIFEST_LIST_V2_SCHEMA,
+    DOCKER_MANIFEST_V2_SCHEMA,
+)
 from pulp_container.app.redirects import (
     FileStorageRedirects,
     S3StorageRedirects,
@@ -836,14 +843,6 @@ def put(self, request, path, pk=None):
         """
         Responds with the actual manifest
         """
-        # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded
-        # before) and the specified repository has not been created yet, create the repository
-        # without raising an error
-        create_new_repo = request.content_type in (
-            models.MEDIA_TYPE.MANIFEST_LIST,
-            models.MEDIA_TYPE.INDEX_OCI,
-        )
-        _, repository = self.get_dr_push(request, path, create=create_new_repo)
         # iterate over all the layers and create
         chunk = request.META["wsgi.input"]
         artifact = self.receive_artifact(chunk)
@@ -864,6 +863,20 @@ def put(self, request, path, pk=None):
 
         content_data = json.loads(raw_data)
 
+        try:
+            self.validate_manifest(content_data, request.content_type)
+        except SchemaValidationError:
+            raise ManifestInvalid(digest=manifest_digest)
+
+        # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded
+        # before) and the specified repository has not been created yet, create the repository
+        # without raising an error
+        create_new_repo = request.content_type in (
+            models.MEDIA_TYPE.MANIFEST_LIST,
+            models.MEDIA_TYPE.INDEX_OCI,
+        )
+        _, repository = self.get_dr_push(request, path, create=create_new_repo)
+
         if request.content_type in (
             models.MEDIA_TYPE.MANIFEST_LIST,
             models.MEDIA_TYPE.INDEX_OCI,
@@ -980,6 +993,28 @@ def put(self, request, path, pk=None):
         if has_task_completed(dispatched_task):
             return ManifestResponse(manifest, path, request, status=201)
 
+    def validate_manifest(self, content_data, content_type):
+        """Validate JSON data (manifest) according to its declared content type (e.g., list)."""
+        try:
+            schema = self.determine_schema(content_type)
+        except ValueError:
+            raise ValidationError()
+        else:
+            validate(content_data, schema)
+
+    def determine_schema(self, content_type):
+        """Return a JSON schema based on the specified content type."""
+        if content_type == models.MEDIA_TYPE.MANIFEST_V2:
+            return DOCKER_MANIFEST_V2_SCHEMA
+        elif content_type == models.MEDIA_TYPE.MANIFEST_OCI:
+            return OCI_MANIFEST_SCHEMA
+        elif content_type == models.MEDIA_TYPE.MANIFEST_LIST:
+            return DOCKER_MANIFEST_LIST_V2_SCHEMA
+        elif content_type == models.MEDIA_TYPE.INDEX_OCI:
+            return OCI_INDEX_SCHEMA
+        else:
+            raise ValueError()
+
     def _save_manifest(self, artifact, manifest_digest, content_type, config_blob=None):
         manifest = models.Manifest(
             digest=manifest_digest,

pulp_container/app/utils.py

@@ -11,7 +11,7 @@
 from pulp_container.app.json_schemas import SIGNATURE_SCHEMA
 
 
-validator = Draft7Validator(json.loads(SIGNATURE_SCHEMA))
+validator = Draft7Validator(SIGNATURE_SCHEMA)
 
 log = logging.getLogger(__name__)
 

pulp_container/tests/unit/test_json_schemas.py

@@ -6,7 +6,7 @@
 
 from pulp_container.app.json_schemas import SIGNATURE_SCHEMA
 
-validator = Draft7Validator(json.loads(SIGNATURE_SCHEMA))
+validator = Draft7Validator(SIGNATURE_SCHEMA)
 
 
 class TestSignatureJsonSchema(TestCase):