Added an ability to sync image signatures from a provided sigstore.

CHANGES/498.feature

@@ -0,0 +1 @@
+Added ability to sync manifest signatures from a sigstore.

pulp_container/app/downloaders.py

@@ -1,14 +1,16 @@
 from gettext import gettext as _
-from logging import getLogger
-from urllib import parse
+
 import aiohttp
 import asyncio
 import json
 import re
 
 from aiohttp.client_exceptions import ClientResponseError
+from logging import getLogger
+from multidict import MultiDict
+from urllib import parse
 
-from pulpcore.plugin.download import HttpDownloader
+from pulpcore.plugin.download import DownloaderFactory, HttpDownloader
 
 
 log = getLogger(__name__)
@@ -161,3 +163,66 @@ def auth_header(token, basic_auth):
         elif basic_auth is not None:
             return {"Authorization": basic_auth}
         return {}
+
+
+class NoAuthDownloaderFactory(DownloaderFactory):
+    """
+    A downloader factory without any preset auth configuration, TLS or basic auth.
+    """
+
+    def _make_aiohttp_session_from_remote(self):
+        """
+        Same as DownloaderFactory._make_aiohttp_session_from_remote, excluding TLS configuration.
+
+        Returns:
+            :class:`aiohttp.ClientSession`
+
+        """
+        tcp_conn_opts = {"force_close": True}
+
+        headers = MultiDict({"User-Agent": NoAuthDownloaderFactory.user_agent()})
+        if self._remote.headers is not None:
+            for header_dict in self._remote.headers:
+                user_agent_header = header_dict.pop("User-Agent", None)
+                if user_agent_header:
+                    headers["User-Agent"] = f"{headers['User-Agent']}, {user_agent_header}"
+                headers.extend(header_dict)
+
+        conn = aiohttp.TCPConnector(**tcp_conn_opts)
+        total = self._remote.total_timeout
+        sock_connect = self._remote.sock_connect_timeout
+        sock_read = self._remote.sock_read_timeout
+        connect = self._remote.connect_timeout
+
+        timeout = aiohttp.ClientTimeout(
+            total=total, sock_connect=sock_connect, sock_read=sock_read, connect=connect
+        )
+        return aiohttp.ClientSession(connector=conn, timeout=timeout, headers=headers)
+
+    def _http_or_https(self, download_class, url, **kwargs):
+        """
+        Same as DownloaderFactory._http_or_https, excluding the basic auth credentials.
+
+        Args:
+            download_class (:class:`~pulpcore.plugin.download.BaseDownloader`): The download
+                class to be instantiated.
+            url (str): The download URL.
+            kwargs (dict): All kwargs are passed along to the downloader. At a minimum, these
+                include the :class:`~pulpcore.plugin.download.BaseDownloader` parameters.
+
+        Returns:
+            :class:`~pulpcore.plugin.download.HttpDownloader`: A downloader that
+            is configured with the remote settings.
+
+        """
+        options = {"session": self._session}
+        if self._remote.proxy_url:
+            options["proxy"] = self._remote.proxy_url
+            if self._remote.proxy_username and self._remote.proxy_password:
+                options["proxy_auth"] = aiohttp.BasicAuth(
+                    login=self._remote.proxy_username, password=self._remote.proxy_password
+                )
+
+        kwargs["throttler"] = self._remote.download_throttler if self._remote.rate_limit else None
+
+        return download_class(url, **options, **kwargs)

pulp_container/app/migrations/0022_containerremote_sigstore.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.9 on 2021-11-29 21:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0021_manifestsignature'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='containerremote',
+            name='sigstore',
+            field=models.TextField(null=True),
+        ),
+    ]

pulp_container/app/models.py

@@ -205,8 +205,8 @@ class ManifestSignature(Content):
     A signature for a manifest.
 
     Fields:
-        name (models.CharField): A signature name in the 'manifest_digest@random_name' format
-        digest (models.CharField): A signature sha256 digest.
+        name (models.CharField): A signature name in the 'manifest_digest@random_name' format.
+        digest (models.CharField): A signature sha256 digest prepended with its algorithm `sha256:`.
         type (models.CharField): A signature type as specified in signature metadata. Currently
                                  it's only "atomic container signature".
         key_id (models.CharField): A key id identified by gpg (last 8 bytes of the fingerprint).
@@ -320,26 +320,28 @@ class ContainerRemote(Remote, AutoAddObjPermsMixin, AutoDeleteObjPermsMixin):
         upstream_name (models.CharField): The name of the image at the remote.
         include_foreign_layers (models.BooleanField): Foreign layers in the remote
             are included. They are not included by default.
+        include_tags (fields.ArrayField): List of tags to include during sync.
+        exclude_tags (fields.ArrayField): List of tags to exclude during sync.
+        sigstore (models.TextField): The URL to a sigstore where signatures of container images
+            should be synced from.
     """
 
     upstream_name = models.CharField(max_length=255, db_index=True)
     include_foreign_layers = models.BooleanField(default=False)
     include_tags = fields.ArrayField(models.CharField(max_length=255, null=True), null=True)
     exclude_tags = fields.ArrayField(models.CharField(max_length=255, null=True), null=True)
+    sigstore = models.TextField(null=True)
 
     TYPE = "container"
     ACCESS_POLICY_VIEWSET_NAME = "remotes/container/container"
 
     @property
     def download_factory(self):
         """
-        Return the DownloaderFactory which can be used to generate asyncio capable downloaders.
+        Downloader Factory that maps to custom downloaders which support registry auth.
 
         Upon first access, the DownloaderFactory is instantiated and saved internally.
 
-        Plugin writers are expected to override when additional configuration of the
-        DownloaderFactory is needed.
-
         Returns:
             DownloadFactory: The instantiated DownloaderFactory to be used by
                 get_downloader()
@@ -357,6 +359,26 @@ def download_factory(self):
             )
             return self._download_factory
 
+    @property
+    def noauth_download_factory(self):
+        """
+        Downloader Factory that doesn't use Basic Auth or TLS settings from a remote.
+
+        Some supplementary data, e.g. signatures, might be available via unprotected resources.
+
+        Upon first access, the NoAuthDownloaderFactory is instantiated and saved internally.
+
+        Returns:
+            DownloadFactory: The instantiated NoAuthDownloaderFactory to be used by
+                get_noauth_downloader().
+
+        """
+        try:
+            return self._noauth_download_factory
+        except AttributeError:
+            self._noauth_download_factory = downloaders.NoAuthDownloaderFactory(self)
+            return self._noauth_download_factory
+
     def get_downloader(self, remote_artifact=None, url=None, **kwargs):
         """
         Get a downloader from either a RemoteArtifact or URL that is configured with this Remote.
@@ -382,6 +404,35 @@ def get_downloader(self, remote_artifact=None, url=None, **kwargs):
         kwargs["remote"] = self
         return super().get_downloader(remote_artifact=remote_artifact, url=url, **kwargs)
 
+    def get_noauth_downloader(self, remote_artifact=None, url=None, **kwargs):
+        """
+        Get a no-auth downloader from either a RemoteArtifact or URL that is provided.
+
+        This method accepts either `remote_artifact` or `url` but not both. At least one is
+        required. If neither or both are passed a ValueError is raised.
+
+        Args:
+            remote_artifact (:class:`~pulpcore.app.models.RemoteArtifact`): The RemoteArtifact to
+                download.
+            url (str): The URL to download.
+            kwargs (dict): This accepts the parameters of
+                :class:`~pulpcore.plugin.download.BaseDownloader`.
+
+        Raises:
+            ValueError: If neither remote_artifact and url are passed, or if both are passed.
+
+        Returns:
+            subclass of :class:`~pulpcore.plugin.download.BaseDownloader`: A downloader that
+            is configured with the remote settings.
+
+        """
+        return super().get_downloader(
+            remote_artifact=remote_artifact,
+            url=url,
+            download_factory=self.noauth_download_factory,
+            **kwargs,
+        )
+
     @property
     def namespaced_upstream_name(self):
         """

pulp_container/app/serializers.py

@@ -4,7 +4,7 @@
 
 from django.db import IntegrityError
 from django.core.exceptions import ObjectDoesNotExist
-
+from django.core.validators import URLValidator
 from rest_framework import serializers
 
 from pulpcore.plugin.models import (
@@ -15,14 +15,15 @@
 from pulpcore.plugin.serializers import (
     ContentGuardSerializer,
     DetailRelatedField,
+    DistributionSerializer,
     IdentityField,
     ModelSerializer,
     NestedRelatedField,
     NoArtifactContentSerializer,
     RelatedField,
     RemoteSerializer,
     RepositorySerializer,
-    DistributionSerializer,
+    RepositorySyncURLSerializer,
     RepositoryVersionRelatedField,
     SingleArtifactContentSerializer,
     validate_unknown_fields,
@@ -247,8 +248,19 @@ class ContainerRemoteSerializer(RemoteSerializer):
         default=Remote.IMMEDIATE,
     )
 
+    sigstore = serializers.CharField(
+        required=False,
+        help_text=_("A URL to a sigstore to download image signatures from"),
+        validators=[URLValidator(schemes=["http", "https"])],
+    )
+
     class Meta:
-        fields = RemoteSerializer.Meta.fields + ("upstream_name", "include_tags", "exclude_tags")
+        fields = RemoteSerializer.Meta.fields + (
+            "upstream_name",
+            "include_tags",
+            "exclude_tags",
+            "sigstore",
+        )
         model = models.ContainerRemote
 
 
@@ -261,7 +273,7 @@ class ContainerDistributionSerializer(DistributionSerializer):
         source="base_path",
         read_only=True,
         help_text=_(
-            "The Registry hostame/name/ to use with docker pull command defined by "
+            "The Registry hostname/name/ to use with docker pull command defined by "
             "this distribution."
         ),
     )
@@ -696,3 +708,17 @@ class Meta:
             "tag",
             "artifacts",
         )
+
+
+class ContainerRepositorySyncURLSerializer(RepositorySyncURLSerializer):
+    """
+    Serializer for Container Sync.
+    """
+
+    signed_only = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=_(
+            "If ``True``, only signed content will be synced. Signatures are not verified."
+        ),
+    )