From c84ab08ab387a3b45ef179ad751a3c40ff2e252e Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Thu, 5 Jan 2023 20:37:00 +0100 Subject: [PATCH] Added support for cosign and well-known types. closes #1165 closes #1166 closes #1167 closes #1232 closes #1233 closes #464 --- CHANGES/1165.feature | 1 + CHANGES/1166.feature | 1 + CHANGES/1167.feature | 1 + CHANGES/1232.feature | 1 + CHANGES/1233.feature | 2 + CHANGES/464.feature | 1 + docs/workflows/cosign-support.rst | 102 ++++++++++++++++++ docs/workflows/helm-support.rst | 83 ++++++++++++++ docs/workflows/index.rst | 14 ++- docs/workflows/oci-artifacts.rst | 54 ++++++++++ docs/workflows/sign-images.rst | 27 ++--- docs/workflows/sync.rst | 6 +- pulp_container/app/__init__.py | 12 +++ pulp_container/app/json_schemas.py | 27 ++--- pulp_container/app/schema_convert.py | 6 +- pulp_container/app/settings.py | 44 ++++++++ pulp_container/app/tasks/sync_stages.py | 25 +++-- pulp_container/app/utils.py | 4 +- pulp_container/constants.py | 25 +++++ .../functional/api/test_sync_signatures.py | 26 ++--- pulp_container/tests/unit/test_convert.py | 14 +-- 21 files changed, 412 insertions(+), 64 deletions(-) create mode 100644 CHANGES/1165.feature create mode 100644 CHANGES/1166.feature create mode 100644 CHANGES/1167.feature create mode 100644 CHANGES/1232.feature create mode 100644 CHANGES/1233.feature create mode 100644 CHANGES/464.feature create mode 100644 docs/workflows/cosign-support.rst create mode 100644 docs/workflows/helm-support.rst create mode 100644 docs/workflows/oci-artifacts.rst diff --git a/CHANGES/1165.feature b/CHANGES/1165.feature new file mode 100644 index 000000000..ba2291d93 --- /dev/null +++ b/CHANGES/1165.feature @@ -0,0 +1 @@ +Added support to serve cosign signatures, SBOMs, and attestations. diff --git a/CHANGES/1166.feature b/CHANGES/1166.feature new file mode 100644 index 000000000..7878d9b76 --- /dev/null +++ b/CHANGES/1166.feature @@ -0,0 +1 @@ +Added support to mirror cosign signatures, SBOMs and attestations. diff --git a/CHANGES/1167.feature b/CHANGES/1167.feature new file mode 100644 index 000000000..9c38f0711 --- /dev/null +++ b/CHANGES/1167.feature @@ -0,0 +1 @@ +Added suport to push cosign signatures, attestations or SBOMs to Pulp Registry. diff --git a/CHANGES/1232.feature b/CHANGES/1232.feature new file mode 100644 index 000000000..a9eab871e --- /dev/null +++ b/CHANGES/1232.feature @@ -0,0 +1 @@ +Enabled Pulp registry to support by default some well-known OCI types. diff --git a/CHANGES/1233.feature b/CHANGES/1233.feature new file mode 100644 index 000000000..89e9629dc --- /dev/null +++ b/CHANGES/1233.feature @@ -0,0 +1,2 @@ +Added ``ADDITIONAL_OCI_ARTIFACT_TYPES`` setting to make the list of supported OCI artifact types +configurable. diff --git a/CHANGES/464.feature b/CHANGES/464.feature new file mode 100644 index 000000000..7c41a817d --- /dev/null +++ b/CHANGES/464.feature @@ -0,0 +1 @@ +Added OCI artifact support for Helm charts. diff --git a/docs/workflows/cosign-support.rst b/docs/workflows/cosign-support.rst new file mode 100644 index 000000000..9607df50f --- /dev/null +++ b/docs/workflows/cosign-support.rst @@ -0,0 +1,102 @@ +.. _cosign-support: + +Mirror cosign signatures +======================== + +Being an OCI compliant registry Pulp Container registry can natively mirror cosign signatures +wich are stored as an OCI image:: + + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:f35028aa1563f37ccbaa0b32c57777ffbd8e9e3d81d739fec0022995e58a375a", + "size": 153 + }, + "layers": [ + { + "mediaType": "application/vnd.dev.cosign.simplesigning.v1+json", + "digest": "sha256:d3370bd32b32aba43de2b45bb4a2de2fb5c95fd2edbe738acbc3bc595b80c456", + "size": 305, + "annotations": { + "dev.cosignproject.cosign/signature": "MEUCIBWDnTKhbf5x3mSuEHWkv3ixloIFXeDpfXipF9szqrd5AiEA+UU5J84gQ9JnmT6QZAXiPXqSoDVW0CXQYssGh63e9Ro=" + } + } + ] + } + + +During the syncronization task, Pulp will automatically mirror cosign signatures or atomic +signatures (accessible via signatures extentions API). + + +Sign and push cosign signatures +=============================== + +Pulp Container registry can host cosign signature which can be pushed via cosign or podman clients: + +Cosign:: + + + # This command creates an ECDSA-P256 key pair (a private and a public key). + cosign generate-key-pair + cosign sign --key cosign.key pulp-registry/ipanova/cosign-test:latest + +or via Podman:: + + podman push pulp-registry/ipanova/cosign-test:latest --sign-by-sigstore-private-key cosign.key + +.. warning:: + To use this with images hosted on image registries, the relevant registry or repository must have + the use-sigstore-attachments option enabled in containers-registries.d(5). This specifies whether + sigstore image attachments (signatures, attestations and the like) are going to be read/written + along with the image. If disabled, the images are treated as if no attachments exist; attempts to + write attachments fail. + +As a result of this operation ``ipanova/cosign-test:latest`` image is signed and its +cosign signature is stored in the registry as an OCI image. Cosign uses a fixed naming convention +to decide the name for a separate image, at which we can store the signature. The tag name resolved +to a fixed digest of the image/or manifest list which is being signed in a form of ``sha256-12345.sig`` + +The payload of the signature will be store as an image layer:: + + { + "critical": { + "identity": { + "docker-reference": "pulp-registry/ipanova/cosigned:latest" + }, + "image": { + "docker-manifest-digest": "sha256:81cd171c4eda75046c31d6ed26f1241bbfa9326640613430be780ea931b02c24" + }, + "type": "cosign container image signature" + }, + "optional": { + "creator": "containers/image 5.23.1", + "timestamp": 1673006074 + } + } + + +.. note: + Besides cosign signature Pulp Container Registry can mirror and host SBOMs and attestations. + + +Cosign signature verification +============================= + +Signature verification can be done via cosign or podman clients:: + + cosign verify --key cosign.pub pulp-registry/ipanova/cosign-test:latest + +When using podman client the policy.json file should be properly configured per specs. +A new requirement type ``sigstoreSigned`` has been introduced: + +https://github.com/containers/image/blob/main/docs/containers-policy.json.5.md#sigstoresigned + +.. warning:: + To use this with images hosted on image registries, the relevant registry or repository must have + the use-sigstore-attachments option enabled in containers-registries.d(5). This specifies whether + sigstore image attachments (signatures, attestations and the like) are going to be read/written + along with the image. If disabled, the images are treated as if no attachments exist; attempts to + write attachments fail. diff --git a/docs/workflows/helm-support.rst b/docs/workflows/helm-support.rst new file mode 100644 index 000000000..1f2daf325 --- /dev/null +++ b/docs/workflows/helm-support.rst @@ -0,0 +1,83 @@ +.. _helm-support: + +Using Helm charts with Pulp Container +===================================== + + +Push and Host +------------- + +Use the following **example** to download and push an etherpad chart from the Red Hat community repository. + +Add a chart repository:: + + $ helm repo add redhat-cop https://redhat-cop.github.io/helm-charts + +Update the information of available charts locally from the chart repository:: + + $ helm repo update + +Download a chart from a repository:: + + $ helm pull redhat-cop/etherpad --version=0.0.4 --untar + +Package the chart into a chart archive:: + + $ helm package ./etherpad + Successfully packaged chart and saved it to: /home/vagrant/devel/pulp_container/etherpad-0.0.4.tgz + +Log in to your Pulp container registry using helm registry login:: + + $ helm registry login pulp3-source-fedora36.puffy.example.com + +Push the chart to your Pulp Container registry using the helm push command:: + + $ helm push etherpad-0.0.4.tgz oci://pulp3-source-fedora36.puffy.example.com + Pushed: pulp3-source-fedora36.puffy.example.com/etherpad:0.0.4 + Digest: sha256:a6667ff2a0e2bd7aa4813db9ac854b5124ff1c458d170b70c2d2375325f2451b + +Ensure that the push worked by deleting the local copy, and then pulling the chart from the repository:: + + $ rm -rf etherpad-0.0.4.tgz + + $ helm pull oci://pulp3-source-fedora36.puffy.example.com/etherpad --version 0.0.4 + Pulled: pulp3-source-fedora36.puffy.example.com/etherpad:0.0.4 + Digest: sha256:4f627399685880daf30cf77b6026dc129034d68c7676c7e07020b70cf7130902 + +The chart can then be installed using the helm install command:: + + $ helm install etherpad-0.0.4.tgz + +Alternatively, charts can be installed directly from the registry without needing to download locally. +Use the helm install command and reference the registry location:: + + $ helm install oci://pulp3-source-fedora36.puffy.example.com/helm/etherpad --version=0.0.4 + + + +Mirror +------ + +Being an OCI compliant registry Pulp Container registry can natively mirror helm charts +wich are stored as an OCI image:: + + { + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.cncf.helm.config.v1+json", + "digest": "sha256:8ec7c0f2f6860037c19b54c3cfbab48d9b4b21b485a93d87b64690fdb68c2111", + "size": 117 + }, + "layers": [ + { + "mediaType": "application/vnd.cncf.helm.chart.content.v1.tar+gzip", + "digest": "sha256:1b251d38cfe948dfc0a5745b7af5ca574ecb61e52aed10b19039db39af6e1617", + "size": 2487 + }, + { + "mediaType": "application/vnd.cncf.helm.chart.provenance.v1.prov", + "digest": "sha256:3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "size": 643 + } + ] + } diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst index 300938ca5..35fc20da5 100644 --- a/docs/workflows/index.rst +++ b/docs/workflows/index.rst @@ -60,11 +60,21 @@ Basic Workflows push import-export -Managing Signatures -------------------- +Managing Atomic Signatures +-------------------------- .. toctree:: :maxdepth: 2 sign-images verify-images + +OCI artifact support +-------------------- + +.. toctree:: + :maxdepth: 2 + + cosign-support + helm-support + oci-artifacts diff --git a/docs/workflows/oci-artifacts.rst b/docs/workflows/oci-artifacts.rst new file mode 100644 index 000000000..4bb7bded4 --- /dev/null +++ b/docs/workflows/oci-artifacts.rst @@ -0,0 +1,54 @@ +Managing additional OCI media types +=================================== + +.. _default-oci-types: + +By default the following list of media types is enabled in the Container Registry:: + + * OCI images + * Helm + * Cosign, SBOMs, attestations + * Source containers + * Singularity + * Conftest policies + * WASM + +For any other OCI media type that is not supported by default, you can add them to the +ADDITIONAL_OCI_ARTIFACT_TYPES settings using the following format:: + + ADDITIONAL_OCI_ARTIFACT_TYPES = { + "": [ + "", + "", + ], + "": [ + "", + "", + ], + } + + +For example, you can add support for custom defined mediatype by adding the following to your +ADDITIONAL_OCI_ARTIFACT_TYPES setting:: + + ADDITIONAL_OCI_ARTIFACT_TYPES = { + "": [ + "", + "", + ], + "": [ + "", + "", + ], + "application/vnd.guardians.groot.config.v1+json": [ + "text/plain", + "application/vnd.guardians.groot.docs.layer.v1+tar", + ], + } + + .. note:: + +When adding OCI media types that are not configured by default, users will also need to manually add +support for the :ref:`Default oci types`. +The OCI image-spec types are supported by default and cannot be disabled, so users will not need +to add that to enable support. diff --git a/docs/workflows/sign-images.rst b/docs/workflows/sign-images.rst index 2b09e9cca..59c7abde1 100644 --- a/docs/workflows/sign-images.rst +++ b/docs/workflows/sign-images.rst @@ -12,10 +12,10 @@ The example below demonstrates how a manifest signing service can be created usi hardware cryptographic device. 2. Create a signing script that accepts a manifest path as the only argument. The script invokes - ``skopeo standalone-sign`` command that generates a container signature for the image manifest, - using the key specified via the ``PULP_SIGNING_KEY_FINGERPRINT`` environment variable. The script - should then print out a JSON structure with the following format. The path of the created - signature is a relative path inside the current working directory:: + ``skopeo standalone-sign`` command that generates an atomic container signature for the image + manifest, using the key specified via the ``PULP_SIGNING_KEY_FINGERPRINT`` environment variable. + The script should then print out a JSON structure with the following format. The path of the + created signature is a relative path inside the current working directory:: {"signature_path": "signature"} @@ -30,7 +30,7 @@ The example below demonstrates how a manifest signing service can be created usi IMAGE_REFERENCE="$REFERENCE" SIGNATURE_PATH="$SIG_PATH" - # Create container signature + # Create atomic container signature skopeo standalone-sign $MANIFEST_PATH $IMAGE_REFERENCE $FINGEPRINT --output $SIGNATURE_PATH # Check the exit status STATUS=$? @@ -80,9 +80,12 @@ The example below demonstrates how a manifest signing service can be created usi Afterwards, users are able to sign selected content by the provided script. + .. warning:: + Underlying singing facility with sign anything what represents an OCI container image. + Sign Images Pushed to the Registry -================================== +---------------------------------- Given that an image is pushed to the Pulp Registry via ``podman/docker push`` or via the standard DockerRegistry v2 push API, a repository is created containing it:: @@ -140,7 +143,7 @@ operation or it can be explictly specified in the following manner:: "worker": "/pulp/api/v3/workers/eb65c2d9-31b2-47dc-847e-dad0e744c539/" } -Upon task complection, a signature is created and added to the repository:: +Upon task complection, an atomic container signature is created and added to the repository:: $ http GET https://pulp.example.com/pulp/api/v3/repositories/container/container-push/3b279a32-b313-44bf-ad41-4359a92cae24/versions/10/ { @@ -202,7 +205,7 @@ Upon task complection, a signature is created and added to the repository:: Sign Images Mirrored into the Registry -====================================== +-------------------------------------- It is possible to sign content that was synchronized from remote registries. If the content was synced together with signatures, upon signing task completion new signatures will be @@ -313,8 +316,8 @@ This API exposes an endpoint for reading and writing image signatures. Users sho sigstore section in the `registries.d file `_ accordingly to benefit from the API. -Reading Image Signatures ------------------------- +Reading Signatures +------------------ To read existing signatures, issue the following GET request:: @@ -323,8 +326,8 @@ To read existing signatures, issue the following GET request:: Signatures are retrieved by container clients automatically if the policy requires so. The policy is defined in the file ``/etc/containers/policy.json``. -Writing Image Signatures ------------------------- +Writing Signatures +------------------ To add a new signature to an image, execute the following PUT request:: diff --git a/docs/workflows/sync.rst b/docs/workflows/sync.rst index ce1ac1b05..b86351a40 100644 --- a/docs/workflows/sync.rst +++ b/docs/workflows/sync.rst @@ -72,9 +72,9 @@ the ``sigstore`` field when creating a Remote. .. note:: Some registries provide docker API extensions for ``atomic container signature`` type only, or - have ``cosign`` type signatures that are stored as a separate OCI artifact in a registry. - Pulp will automatically sync signatures provided via the docker API extension. At the moment, - `cosign` signatures are not supported. + have ``cosign`` type signatures that are stored as a separate OCI image in a registry. + Pulp will automatically sync signatures provided via the docker API extension or cosign + signatures stored as an OCI image. Reference: `Container Remote Usage <../restapi.html#tag/Remotes:-Container>`_ diff --git a/pulp_container/app/__init__.py b/pulp_container/app/__init__.py index 3e1c93095..4ef1a0de0 100644 --- a/pulp_container/app/__init__.py +++ b/pulp_container/app/__init__.py @@ -8,3 +8,15 @@ class PulpContainerPluginAppConfig(PulpPluginAppConfig): label = "container" version = "2.16.0.dev" python_package_name = "pulp-container" + + def ready(self): + super().ready() + self.register_registry_types() + + def register_registry_types(self): + # circular import avoidance + from pulp_container import constants + from django.conf import settings + + for media_type, layer_types in settings.ADDITIONAL_OCI_ARTIFACT_TYPES.items(): + constants.register_well_known_types(media_type, layer_types) diff --git a/pulp_container/app/json_schemas.py b/pulp_container/app/json_schemas.py index 185030cfd..e6bddc2b5 100644 --- a/pulp_container/app/json_schemas.py +++ b/pulp_container/app/json_schemas.py @@ -1,10 +1,17 @@ -from pulp_container.constants import BLOB_CONTENT_TYPE, MEDIA_TYPE, SIGNATURE_TYPE +from pulp_container.constants import ( + ALLOWED_ARTIFACT_TYPES, + ALLOWED_BLOB_CONTENT_TYPES, + BLOB_CONTENT_TYPE, + MEDIA_TYPE, + SIGNATURE_TYPE, +) def get_descriptor_schema( allowed_media_types, additional_properties=None, additional_required=None ): """Return a concrete descriptor schema for manifests.""" + properties = { "mediaType": {"type": "string", "enum": allowed_media_types}, "size": {"type": "number"}, @@ -62,6 +69,7 @@ def get_descriptor_schema( "required": ["schemaVersion", "manifests"], } + OCI_MANIFEST_SCHEMA = { "type": "object", "properties": { @@ -70,24 +78,16 @@ def get_descriptor_schema( "type": "string", "enum": [MEDIA_TYPE.MANIFEST_OCI], }, - "config": get_descriptor_schema([MEDIA_TYPE.CONFIG_BLOB_OCI]), + "config": get_descriptor_schema(ALLOWED_ARTIFACT_TYPES), "layers": { "type": "array", - "items": get_descriptor_schema( - [ - MEDIA_TYPE.REGULAR_BLOB_OCI_TAR, - MEDIA_TYPE.REGULAR_BLOB_OCI_TAR_GZIP, - MEDIA_TYPE.REGULAR_BLOB_OCI_TAR_ZSTD, - MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR, - MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR_GZIP, - MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR_ZSTD, - ] - ), + "items": get_descriptor_schema(ALLOWED_BLOB_CONTENT_TYPES), }, }, "required": ["schemaVersion", "config", "layers"], } + DOCKER_MANIFEST_LIST_V2_SCHEMA = { "type": "object", "properties": { @@ -136,6 +136,7 @@ def get_descriptor_schema( "required": ["schemaVersion", "mediaType", "manifests"], } + DOCKER_MANIFEST_V2_SCHEMA = { "type": "object", "properties": { @@ -178,6 +179,7 @@ def get_descriptor_schema( "required": ["schemaVersion", "mediaType", "config", "layers"], } + DOCKER_MANIFEST_V1_SCHEMA = { "type": "object", "properties": { @@ -219,6 +221,7 @@ def get_descriptor_schema( "required": ["tag", "name", "fsLayers", "history"], } + SIGNATURE_SCHEMA = { "title": "Atomic Container Signature", "description": "JSON Schema Validation for the Signature Payload", diff --git a/pulp_container/app/schema_convert.py b/pulp_container/app/schema_convert.py index 581f5e1c2..c75e2c984 100644 --- a/pulp_container/app/schema_convert.py +++ b/pulp_container/app/schema_convert.py @@ -94,7 +94,7 @@ class Schema2toSchema1Converter: and call convert() to obtain the signed manifest, as a JSON-encoded string. """ - EMPTY_LAYER = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + EMPTY_BLOB = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" def __init__(self, manifest, config_layer, name, tag): """ @@ -171,7 +171,7 @@ def _compute_fs_layers(self): curr_uncompressed_dig = next(diff_ids) for curr_hist in config_layer_history: if curr_hist.get("empty_layer"): - layer_id = self.EMPTY_LAYER + layer_id = self.EMPTY_BLOB uncompressed_dig = None else: layer_id = curr_compressed_dig @@ -180,7 +180,7 @@ def _compute_fs_layers(self): curr_compressed_dig = next(layers)["digest"] curr_uncompressed_dig = next(diff_ids) except StopIteration: - curr_compressed_dig = self.EMPTY_LAYER + curr_compressed_dig = self.EMPTY_BLOB curr_uncompressed_dig = None fs_layers.append(FS_Layer(layer_id, uncompressed_dig, curr_hist)) return fs_layers diff --git a/pulp_container/app/settings.py b/pulp_container/app/settings.py index a1e413b24..3060606d6 100644 --- a/pulp_container/app/settings.py +++ b/pulp_container/app/settings.py @@ -2,3 +2,47 @@ "dynaconf_merge_unique": True, "reusable_conditions": ["pulp_container.app.global_access_conditions"], } + +ADDITIONAL_OCI_ARTIFACT_TYPES = { + "application/vnd.oci.image.config.v1+json": [ + # cosign signing and attestations + "application/vnd.dev.cosign.simplesigning.v1+json", + "application/vnd.dsse.envelope.v1+json", + # cosign SBOMS spdx and cyclonedx + "text/spdx", + "text/spdx+xml", + "text/spdx+json", + "application/vnd.cyclonedx", + "application/vnd.cyclonedx+xml", + "application/vnd.cyclonedx+json", + # syft SBOMS + "application/vnd.syft+json", + # cosign in-toto attestations + "application/vnd.in-toto+json", + ], + # helm + "application/vnd.cncf.helm.config.v1+json": [ + "application/tar+gzip", + "application/vnd.cncf.helm.chart.content.v1.tar+gzip", + "application/vnd.cncf.helm.chart.provenance.v1.prov", + ], + # source containers + "application/vnd.oci.source.image.config.v1+json": [ + "application/vnd.oci.image.layer.v1.tar+gzip", + ], + # conftest policies + "application/vnd.cncf.openpolicyagent.config.v1+json": [ + "application/vnd.cncf.openpolicyagent.policy.layer.v1+rego", + "application/vnd.cncf.openpolicyagent.data.layer.v1+json", + "application/vnd.cncf.openpolicyagent.manifest.layer.v1+json", + "application/vnd.cncf.openpolicyagent.rego.layer.v1+rego", + ], + # singularity + "application/vnd.sylabs.sif.config.v1+json": [ + "application/vnd.sylabs.sif.layer.v1.sif", + ], + # wasm + "application/vnd.wasm.config.v1+json": [ + "application/vnd.wasm.content.layer.v1+wasm", + ], +} diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 1d9e37844..546b59977 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -133,15 +133,9 @@ async def run(self): to_download = [] BATCH_SIZE = 500 + # it can be wether a separate sigstore location or registry with extended signatures API signature_source = await self.get_signature_source() - if signature_source is None and self.signed_only: - raise ValueError( - "It is requested to sync only signed content but no sigstore URL is " - "provided. Please configure a `sigstore` on your Remote or set " - "`signed_only` to `False` for your sync request." - ) - async with ProgressReport( message="Downloading tag list", code="sync.downloading.tag_list", total=1 ) as pb: @@ -176,6 +170,23 @@ async def run(self): digest = response.artifact_attributes["sha256"] + # Look for cosign signatures + # cosign signature has a tag convention 'sha256-1234.sig' + if self.signed_only and not signature_source: + if ( + not (tag_name.endswith(".sig") and tag_name.startswith("sha256-")) + and f"sha256-{digest}.sig" not in tag_list + ): + # skip this tag, there is no corresponding signature + log.info( + "The unsigned image {digest} can't be synced " + "due to a requirement to sync signed content " + "only.".format(digest=digest) + ) + # Count the skipped tagks as parsed too. + await pb_parsed_tags.aincrement() + continue + media_type = determine_media_type(content_data, response) validate_manifest(content_data, media_type, digest) diff --git a/pulp_container/app/utils.py b/pulp_container/app/utils.py index 1daf6ff93..fe6c6e71a 100644 --- a/pulp_container/app/utils.py +++ b/pulp_container/app/utils.py @@ -10,7 +10,7 @@ from pulpcore.plugin.models import Task -from pulp_container.constants import MANIFEST_MEDIA_TYPES, MEDIA_TYPE +from pulp_container.constants import ALLOWED_ARTIFACT_TYPES, MANIFEST_MEDIA_TYPES, MEDIA_TYPE from pulp_container.app.exceptions import ManifestInvalid from pulp_container.app.json_schemas import ( OCI_INDEX_SCHEMA, @@ -172,7 +172,7 @@ def determine_media_type_from_json(content_data): else: if config := content_data.get("config"): config_media_type = config.get("mediaType") - if config_media_type == MEDIA_TYPE.CONFIG_BLOB_OCI: + if config_media_type in ALLOWED_ARTIFACT_TYPES: return MEDIA_TYPE.MANIFEST_OCI else: return MEDIA_TYPE.MANIFEST_V2 diff --git a/pulp_container/constants.py b/pulp_container/constants.py index 333a8f669..c67519af2 100644 --- a/pulp_container/constants.py +++ b/pulp_container/constants.py @@ -30,6 +30,18 @@ LIST=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI], ) +OCI_BLOB_DISTRIBUTABLE_MEDIA_TYPE = [ + MEDIA_TYPE.REGULAR_BLOB_OCI_TAR, + MEDIA_TYPE.REGULAR_BLOB_OCI_TAR_GZIP, + MEDIA_TYPE.REGULAR_BLOB_OCI_TAR_ZSTD, +] +OCI_BLOB_NON_DISTRIBUTABLE_MEDIA_TYPE = [ + MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR, + MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR_GZIP, + MEDIA_TYPE.FOREIGN_BLOB_OCI_TAR_ZSTD, +] +OCI_BLOB_MEDIA_TYPE = OCI_BLOB_DISTRIBUTABLE_MEDIA_TYPE + OCI_BLOB_NON_DISTRIBUTABLE_MEDIA_TYPE + EMPTY_BLOB = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" BLOB_CONTENT_TYPE = "application/octet-stream" SIGNATURE_TYPE = SimpleNamespace( @@ -43,3 +55,16 @@ SIGNATURE_PAYLOAD_MAX_SIZE = 4 * MEGABYTE SIGNATURE_API_EXTENSION_VERSION = 2 + + +ALLOWED_ARTIFACT_TYPES = [MEDIA_TYPE.CONFIG_BLOB_OCI] +ALLOWED_BLOB_CONTENT_TYPES = OCI_BLOB_MEDIA_TYPE + + +def register_well_known_types(artifact_config_type, artifact_layer_types): + if artifact_config_type not in ALLOWED_ARTIFACT_TYPES: + ALLOWED_ARTIFACT_TYPES.append(artifact_config_type) + + for layer_type in artifact_layer_types: + if layer_type not in ALLOWED_BLOB_CONTENT_TYPES: + ALLOWED_BLOB_CONTENT_TYPES.append(layer_type) diff --git a/pulp_container/tests/functional/api/test_sync_signatures.py b/pulp_container/tests/functional/api/test_sync_signatures.py index 6995ccbac..2c1166eab 100644 --- a/pulp_container/tests/functional/api/test_sync_signatures.py +++ b/pulp_container/tests/functional/api/test_sync_signatures.py @@ -113,23 +113,17 @@ def test_sync_signed_images_from_sigstore( assert len(manifest_signatures) == 6 +@pytest.mark.parametrize( + "synced_repository", [{"sigstore": None, "signed_only": True}], indirect=True +) def test_sync_images_without_sigstore_requiring_signatures( - container_remote_api, container_repository_api, gen_object_with_cleanup + container_signature_api, container_tag_api, synced_repository ): """Sync a repository with no sigstore but with the signed_only option enabled.""" - data = gen_container_remote( - url=REDHAT_REGISTRY_V2, - upstream_name=DEPRECATED_REPOSITORY_NAME, - policy="on_demand", - include_tags=[MANIFEST_LIST_TAG, IMAGE_MANIFEST_TAG], - ) - remote = gen_object_with_cleanup(container_remote_api, data) - - data = ContainerContainerRepository(**gen_repo()) - repository = gen_object_with_cleanup(container_repository_api, data) - - data = ContainerRepositorySyncURL(remote=remote.pulp_href, signed_only=True) - response = container_repository_api.sync(repository.pulp_href, data) + signatures = container_signature_api.list( + repository_version=synced_repository.latest_version_href + ).results + assert len(signatures) == 0 - with pytest.raises(PulpTaskError): - monitor_task(response.task) + tags = container_tag_api.list(repository_version=synced_repository.latest_version_href).results + assert len(tags) == 0 diff --git a/pulp_container/tests/unit/test_convert.py b/pulp_container/tests/unit/test_convert.py index 611e1531d..fa4b4bdd3 100644 --- a/pulp_container/tests/unit/test_convert.py +++ b/pulp_container/tests/unit/test_convert.py @@ -14,13 +14,13 @@ class Test(TestCase): def test_convert(self): """Test schema converter on a known manifest""" cnv = schema_convert.Schema2toSchema1Converter( - MANIFEST, CONFIG_LAYER, "test-repo", "tes-tag" + MANIFEST, CONFIG_BLOB, "test-repo", "tes-tag" ) converted_mf, signed_mf = cnv.convert() compare_manifests(converted_mf, signed_mf) validate_signature(signed_mf) - empty = dict(blobSum=cnv.EMPTY_LAYER) + empty = dict(blobSum=cnv.EMPTY_BLOB) assert [ dict(blobSum="sha256:layer1"), empty, @@ -34,7 +34,7 @@ def test_manifest_with_foreign_layers_conversion(self): """Test if the conversion of a manifest with foreign layers fails gracefully""" try: schema_convert.Schema2toSchema1Converter( - MANIFEST_WITH_FOREIGN_LAYERS, CONFIG_LAYER, "test-repo", "tes-tag" + MANIFEST_WITH_FOREIGN_BLOBS, CONFIG_BLOB, "test-repo", "tes-tag" ) except ValueError: pass @@ -44,10 +44,10 @@ def test_manifest_with_foreign_layers_conversion(self): def test_compute_layers(self): """Test that computing the layers produces the expected data""" cnv = schema_convert.Schema2toSchema1Converter( - MANIFEST, CONFIG_LAYER, "test-repo", "tes-tag" + MANIFEST, CONFIG_BLOB, "test-repo", "tes-tag" ) cnv.compute_layers() - empty = dict(blobSum=cnv.EMPTY_LAYER) + empty = dict(blobSum=cnv.EMPTY_BLOB) assert [ dict(blobSum="sha256:layer1"), empty, @@ -118,7 +118,7 @@ def validate_signature(signed_mf): MANIFEST = dict(schemaVersion=2, layers=[dict(digest="sha256:base"), dict(digest="sha256:layer1")]) -MANIFEST_WITH_FOREIGN_LAYERS = dict( +MANIFEST_WITH_FOREIGN_BLOBS = dict( schemaVersion=2, layers=[ dict( @@ -129,7 +129,7 @@ def validate_signature(signed_mf): ], ) -CONFIG_LAYER = dict( +CONFIG_BLOB = dict( architecture="amd64", author="Mihai Ibanescu ", config=dict(Hostname="decafbad", Cmd=["/bin/bash"]),