Initial source package and source indices support

fixes pulp#409
pulp#409
(was https://pulp.plan.io/issues/8775)

Allows the addition of Debian Source Packages, DSC files and any
associate source files (.orig.tar.gz, .debian.tar.xz, ...), as well as
synchronizing with a remote's source indices (sync_sources=True).
Publishing a distribution will make available DSC files, sources and
source indices in the repository, compliant with the Debian
rempository format.

The source files referenced in a dsc_file must be uploaded as
artifacts before the dsc_file so a typical workflow might be

--
apt-get source bc

http --form post $BASE/pulp/api/v3/artifacts/ \
  file@"/tmp/bc_1.07.1-2build1.debian.tar.xz"
http --form post $BASE/pulp/api/v3/artifacts/ \
  file@"/tmp/bc_1.07.1.orig.tar.gz"
http --form post $BASE/pulp/api/v3/artifacts/ \
  file@"/tmp/bc_1.07.1-2build1.dsc"
http --form post $BASE/pulp/api/v3/content/deb/source_packages/ \
  artifact=/pulp/api/v3/artifacts/3094de7e-da61-4cf6-ae75-a35b74472d9a/
--

Attempting to create the source_packages content from a DSC file
artifact before its source files are present as artifacts will result
validation errors.

Once source_packages content has been created it can be inspected via
its pulp_href or using the source_packages endpoint

http get $BASE/pulp/api/v3/content/deb/source_packages/

Synchronizing with a remote with "sync_sources=True" will synchronize
the Source Indicies files inspecting each contained paragraph to
download all associated DSC files and referced source files (note:
source file download may be deferred if remote policy="on_demand"),
performing a high level of data validation and creating all required
associations. As with uploading inspecting contents after a sync is
complete can be done with the same endpoints as above.

To remain spec compliant with https://wiki.debian.org/DebianRepository
"md5" must be present in the ALLOWED_CONTENT_CHECKSUMS. In all cases
the use of md5 is supplemental so security concerns around this
addition, at least for use with this feature, should be minimal.

CHANGES/409.feature

@@ -0,0 +1 @@
+Add support for handling source packages.

pulp_deb/app/migrations/0023_sourceindex_sourcepackage_sourcepackagereleasecomponent.py

@@ -0,0 +1,82 @@
+# Generated by Django 3.2.18 on 2023-05-31 20:04
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0107_distribution_hidden'),
+        ('deb', '0022_alter_aptdistribution_distribution_ptr_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SourcePackage',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='deb_sourcepackage', serialize=False, to='core.content')),
+                ('relative_path', models.TextField()),
+                ('format', models.TextField()),
+                ('source', models.TextField()),
+                ('binary', models.TextField(null=True)),
+                ('architecture', models.TextField(null=True)),
+                ('version', models.TextField()),
+                ('maintainer', models.TextField()),
+                ('uploaders', models.TextField(null=True)),
+                ('homepage', models.TextField(null=True)),
+                ('vcs_browser', models.TextField(null=True)),
+                ('vcs_arch', models.TextField(null=True)),
+                ('vcs_bzr', models.TextField(null=True)),
+                ('vcs_cvs', models.TextField(null=True)),
+                ('vcs_darcs', models.TextField(null=True)),
+                ('vcs_git', models.TextField(null=True)),
+                ('vcs_hg', models.TextField(null=True)),
+                ('vcs_mtn', models.TextField(null=True)),
+                ('vcs_snv', models.TextField(null=True)),
+                ('testsuite', models.TextField(null=True)),
+                ('dgit', models.TextField(null=True)),
+                ('standards_version', models.TextField()),
+                ('build_depends', models.TextField(null=True)),
+                ('build_depends_indep', models.TextField(null=True)),
+                ('build_depends_arch', models.TextField(null=True)),
+                ('build_conflicts', models.TextField(null=True)),
+                ('build_conflicts_indep', models.TextField(null=True)),
+                ('build_conflicts_arch', models.TextField(null=True)),
+                ('package_list', models.TextField(null=True)),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+            },
+            bases=('core.content',),
+        ),
+        migrations.CreateModel(
+            name='SourcePackageReleaseComponent',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='deb_sourcepackagereleasecomponent', serialize=False, to='core.content')),
+                ('release_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_sourcepackagereleasecomponent', to='deb.releasecomponent')),
+                ('source_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_sourcepackagereleasecomponent', to='deb.sourcepackage')),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+                'unique_together': {('source_package', 'release_component')},
+            },
+            bases=('core.content',),
+        ),
+        migrations.CreateModel(
+            name='SourceIndex',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='deb_sourceindex', serialize=False, to='core.content')),
+                ('component', models.CharField(max_length=255)),
+                ('relative_path', models.TextField()),
+                ('sha256', models.CharField(max_length=255)),
+                ('release', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_sourceindex', to='deb.releasefile')),
+            ],
+            options={
+                'verbose_name_plural': 'SourceIndices',
+                'default_related_name': '%(app_label)s_%(model_name)s',
+                'unique_together': {('relative_path', 'sha256')},
+            },
+            bases=('core.content',),
+        ),
+    ]

pulp_deb/app/models/__init__.py

@@ -6,6 +6,7 @@
     GenericContent,
     InstallerPackage,
     Package,
+    SourcePackage,
 )
 
 from .content.metadata import (
@@ -16,9 +17,10 @@
     ReleaseArchitecture,
     ReleaseComponent,
     PackageReleaseComponent,
+    SourcePackageReleaseComponent,
 )
 
-from .content.verbatim_metadata import ReleaseFile, PackageIndex, InstallerFileIndex
+from .content.verbatim_metadata import ReleaseFile, PackageIndex, InstallerFileIndex, SourceIndex
 
 from .publication import AptDistribution, AptPublication, VerbatimPublication
 

pulp_deb/app/models/content/content.py

@@ -14,7 +14,6 @@
 
 from pulpcore.plugin.models import Content
 
-
 BOOL_CHOICES = [(True, "yes"), (False, "no")]
 
 
@@ -147,3 +146,172 @@ class GenericContent(Content):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = (("relative_path", "sha256"),)
+
+
+class SourcePackage(Content):
+    """
+    The Debian Source Package (dsc, orig.tar.gz, debian.tar.gz... files) content type.
+
+    This model must contain all information that is needed to
+    generate the corresponding paragraph in "Souces" indices files.
+    """
+
+    TYPE = "source_package"
+
+    SUFFIX = "dsc"
+
+    relative_path = models.TextField()
+    format = models.TextField()  # the format of the source package
+    source = models.TextField()  # source package nameformat
+    binary = models.TextField(null=True)  # lists binary packages which a source package can produce
+    architecture = models.TextField(null=True)  # all, i386, ...
+    version = models.TextField()  # The format is: [epoch:]upstream_version[-debian_revision]
+    maintainer = models.TextField()
+    uploaders = models.TextField(null=True)  # Names and emails of co-maintainers
+    homepage = models.TextField(null=True)
+    vcs_browser = models.TextField(null=True)
+    vcs_arch = models.TextField(null=True)
+    vcs_bzr = models.TextField(null=True)
+    vcs_cvs = models.TextField(null=True)
+    vcs_darcs = models.TextField(null=True)
+    vcs_git = models.TextField(null=True)
+    vcs_hg = models.TextField(null=True)
+    vcs_mtn = models.TextField(null=True)
+    vcs_snv = models.TextField(null=True)
+    testsuite = models.TextField(null=True)
+    dgit = models.TextField(null=True)
+    standards_version = models.TextField()  # most recent version of the standards the pkg complies
+    build_depends = models.TextField(null=True)
+    build_depends_indep = models.TextField(null=True)
+    build_depends_arch = models.TextField(null=True)
+    build_conflicts = models.TextField(null=True)
+    build_conflicts_indep = models.TextField(null=True)
+    build_conflicts_arch = models.TextField(null=True)
+    package_list = models.TextField(
+        null=True
+    )  # all the packages that can be built from the source package
+
+    def __init__(self, *args, **kwargs):
+        """Sanatize kwargs by removing multi-lists before contructing DscFile"""
+        for kw in ["files", "checksums_sha1", "checksums_sha256", "checksums_sha512"]:
+            if kw in kwargs:
+                kwargs.pop(kw)
+        super().__init__(*args, **kwargs)
+
+    def derived_dsc_filename(self):
+        """Print a nice name for the Dsc file."""
+        return "{}_{}.{}".format(self.source, self.version, self.SUFFIX)
+
+    def derived_dir(self, component=""):
+        """Assemble full dir in pool directory."""
+        sourcename = self.source
+        prefix = sourcename[0]
+        return os.path.join(
+            "pool",
+            component,
+            prefix,
+            sourcename,
+        )
+
+    def derived_path(self, name, component=""):
+        """Assemble filename in pool directory."""
+        return os.path.join(self.derived_dir(component), name)
+
+    @property
+    def checksums_sha1(self):
+        """Generate 'Checksums-Sha1' list from content artifacts."""
+        contents = []
+        for content_artifact in self.contentartifact_set.all():
+            if content_artifact:
+                if content_artifact.artifact:
+                    sha1 = content_artifact.artifact.sha1
+                    size = content_artifact.artifact.size
+                else:
+                    remote_artifact = content_artifact.remoteartifact_set.first()
+                    sha1 = remote_artifact.sha1
+                    size = remote_artifact.size
+                # Sha1 is optional so filter out incomplete data
+                if sha1 is not None:
+                    contents.append(
+                        {
+                            "name": os.path.basename(content_artifact.relative_path),
+                            "sha1": sha1,
+                            "size": size,
+                        }
+                    )
+        return contents
+
+    @property
+    def checksums_sha256(self):
+        """Generate 'Checksums-Sha256' list from content artifacts."""
+        contents = []
+        for content_artifact in self.contentartifact_set.all():
+            if content_artifact:
+                if content_artifact.artifact:
+                    sha256 = content_artifact.artifact.sha256
+                    size = content_artifact.artifact.size
+                else:
+                    remote_artifact = content_artifact.remoteartifact_set.first()
+                    sha256 = remote_artifact.sha256
+                    size = remote_artifact.size
+                # Sha256 is required so better to not filter out incomplete data
+                contents.append(
+                    {
+                        "name": os.path.basename(content_artifact.relative_path),
+                        "sha256": sha256,
+                        "size": size,
+                    }
+                )
+        return contents
+
+    @property
+    def checksums_sha512(self):
+        """Generate 'Checksums-Sha512' list from content artifacts."""
+        contents = []
+        for content_artifact in self.contentartifact_set.all():
+            if content_artifact:
+                if content_artifact.artifact:
+                    sha512 = content_artifact.artifact.sha512
+                    size = content_artifact.artifact.size
+                else:
+                    remote_artifact = content_artifact.remoteartifact_set.first()
+                    sha512 = remote_artifact.sha512
+                    size = remote_artifact.size
+                # Sha512 is optional so filter out incomplete data
+                if sha512 is not None:
+                    contents.append(
+                        {
+                            "name": os.path.basename(content_artifact.relative_path),
+                            "sha512": sha512,
+                            "size": size,
+                        }
+                    )
+        return contents
+
+    @property
+    def files(self):
+        """Generate 'Files' list from content artifacts."""
+        contents = []
+        for content_artifact in self.contentartifact_set.all():
+            if content_artifact:
+                if content_artifact.artifact:
+                    md5 = content_artifact.artifact.md5
+                    size = content_artifact.artifact.size
+                else:
+                    remote_artifact = content_artifact.remoteartifact_set.first()
+                    md5 = remote_artifact.md5
+                    size = remote_artifact.size
+                # md5 is required so better to not filter out incomplete data
+                contents.append(
+                    {
+                        "name": os.path.basename(content_artifact.relative_path),
+                        "md5sum": md5,
+                        "size": size,
+                    }
+                )
+        return contents
+
+    repo_key_fields = ("source", "version")
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"

pulp_deb/app/models/content/structure_content.py

@@ -16,7 +16,7 @@
 
 from pulpcore.plugin.models import Content
 
-from pulp_deb.app.models import Package
+from pulp_deb.app.models import Package, SourcePackage
 
 
 BOOL_CHOICES = [(True, "yes"), (False, "no")]
@@ -113,3 +113,21 @@ class PackageReleaseComponent(Content):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = (("package", "release_component"),)
+
+
+class SourcePackageReleaseComponent(Content):
+    """
+    The SourcePackageReleaseComponent.
+
+    This is the join table that decides, which Source Package (in which RepositoryVersions) belong
+    to which ReleaseComponents.
+    """
+
+    TYPE = "source_package_release_component"
+
+    source_package = models.ForeignKey(SourcePackage, on_delete=models.CASCADE)
+    release_component = models.ForeignKey(ReleaseComponent, on_delete=models.CASCADE)
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+        unique_together = (("source_package", "release_component"),)

pulp_deb/app/models/content/verbatim_metadata.py

@@ -115,3 +115,33 @@ def main_artifact(self):
         Retrieve the uncompressed SHA256SUMS artifact.
         """
         return self._artifacts.get(sha256=self.sha256)
+
+
+class SourceIndex(Content):
+    """
+    The "SourceIndex" content type.
+
+    This model represents the Sources file for a specific
+    component.
+    It's artifacts should include all (non-)compressed versions
+    of the upstream Sources file.
+    """
+
+    TYPE = "source_index"
+
+    release = models.ForeignKey(ReleaseFile, on_delete=models.CASCADE)
+    component = models.CharField(max_length=255)
+    relative_path = models.TextField()
+    sha256 = models.CharField(max_length=255)
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+        verbose_name_plural = "SourceIndices"
+        unique_together = (("relative_path", "sha256"),)
+
+    @property
+    def main_artifact(self):
+        """
+        Retrieve teh uncompressed SourceIndex artifact.
+        """
+        return self._artifacts.get(sha256=self.sha256)

pulp_deb/app/models/repository.py

@@ -14,6 +14,9 @@
     ReleaseArchitecture,
     ReleaseComponent,
     ReleaseFile,
+    SourceIndex,
+    SourcePackage,
+    SourcePackageReleaseComponent,
 )
 
 
@@ -34,6 +37,9 @@ class AptRepository(Repository):
         ReleaseArchitecture,
         ReleaseComponent,
         ReleaseFile,
+        SourceIndex,
+        SourcePackage,
+        SourcePackageReleaseComponent,
     ]
     REMOTE_TYPES = [
         AptRemote,

pulp_deb/app/serializers/__init__.py

@@ -13,6 +13,10 @@
     ReleaseArchitectureSerializer,
     ReleaseComponentSerializer,
     ReleaseFileSerializer,
+    SourceIndexSerializer,
+    DscFile822Serializer,
+    SourcePackageSerializer,
+    SourcePackageReleaseComponentSerializer,
 )
 
 from .publication_serializers import (