Merge pull request #295 from masselstine/main

Initial source file and source indices support

CHANGES/409.feature

@@ -0,0 +1 @@
+Add support for handling source packages.

pulp_deb/app/migrations/0028_sourcepackage_sourcepackagereleasecomponent_and_more.py

@@ -0,0 +1,82 @@
+# Generated by Django 4.2.6 on 2023-10-06 15:36
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core', '0107_distribution_hidden'),
+ ('deb', '0027_alter_aptpublication_structured'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='SourcePackage',
+ fields=[
+ ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content')),
+ ('relative_path', models.TextField()),
+ ('format', models.TextField()),
+ ('source', models.TextField()),
+ ('binary', models.TextField(null=True)),
+ ('architecture', models.TextField(null=True)),
+ ('version', models.TextField()),
+ ('maintainer', models.TextField()),
+ ('uploaders', models.TextField(null=True)),
+ ('homepage', models.TextField(null=True)),
+ ('vcs_browser', models.TextField(null=True)),
+ ('vcs_arch', models.TextField(null=True)),
+ ('vcs_bzr', models.TextField(null=True)),
+ ('vcs_cvs', models.TextField(null=True)),
+ ('vcs_darcs', models.TextField(null=True)),
+ ('vcs_git', models.TextField(null=True)),
+ ('vcs_hg', models.TextField(null=True)),
+ ('vcs_mtn', models.TextField(null=True)),
+ ('vcs_snv', models.TextField(null=True)),
+ ('testsuite', models.TextField(null=True)),
+ ('dgit', models.TextField(null=True)),
+ ('standards_version', models.TextField()),
+ ('build_depends', models.TextField(null=True)),
+ ('build_depends_indep', models.TextField(null=True)),
+ ('build_depends_arch', models.TextField(null=True)),
+ ('build_conflicts', models.TextField(null=True)),
+ ('build_conflicts_indep', models.TextField(null=True)),
+ ('build_conflicts_arch', models.TextField(null=True)),
+ ('package_list', models.TextField(null=True)),
+ ],
+ options={
+ 'default_related_name': '%(app_label)s_%(model_name)s',
+ },
+ bases=('core.content',),
+ ),
+ migrations.CreateModel(
+ name='SourcePackageReleaseComponent',
+ fields=[
+ ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content')),
+ ('release_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='deb.releasecomponent')),
+ ('source_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='deb.sourcepackage')),
+ ],
+ options={
+ 'default_related_name': '%(app_label)s_%(model_name)s',
+ 'unique_together': {('source_package', 'release_component')},
+ },
+ bases=('core.content',),
+ ),
+ migrations.CreateModel(
+ name='SourceIndex',
+ fields=[
+ ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content')),
+ ('component', models.CharField(max_length=255)),
+ ('relative_path', models.TextField()),
+ ('sha256', models.CharField(max_length=255)),
+ ('release', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='deb.releasefile')),
+ ],
+ options={
+ 'verbose_name_plural': 'SourceIndices',
+ 'default_related_name': '%(app_label)s_%(model_name)s',
+ 'unique_together': {('relative_path', 'sha256')},
+ },
+ bases=('core.content',),
+ ),
+ ]

pulp_deb/app/models/__init__.py

@@ -6,6 +6,7 @@
  GenericContent,
  InstallerPackage,
  Package,
+ SourcePackage,
 )
 
 from .signing_service import AptReleaseSigningService
@@ -18,9 +19,10 @@
  ReleaseArchitecture,
  ReleaseComponent,
  PackageReleaseComponent,
+ SourcePackageReleaseComponent,
 )
 
-from .content.verbatim_metadata import ReleaseFile, PackageIndex, InstallerFileIndex
+from .content.verbatim_metadata import ReleaseFile, PackageIndex, InstallerFileIndex, SourceIndex
 
 from .publication import AptDistribution, AptPublication, VerbatimPublication
 

pulp_deb/app/models/content/content.py

@@ -14,7 +14,6 @@
 
 from pulpcore.plugin.models import Content
 
-
 BOOL_CHOICES = [(True, "yes"), (False, "no")]
 
 
@@ -147,3 +146,172 @@ class GenericContent(Content):
  class Meta:
  default_related_name = "%(app_label)s_%(model_name)s"
  unique_together = (("relative_path", "sha256"),)
+
+
+class SourcePackage(Content):
+ """
+ The Debian Source Package (dsc, orig.tar.gz, debian.tar.gz... files) content type.
+
+ This model must contain all information that is needed to
+ generate the corresponding paragraph in "Souces" indices files.
+ """
+
+ TYPE = "source_package"
+
+ SUFFIX = "dsc"
+
+ relative_path = models.TextField()
+ format = models.TextField() # the format of the source package
+ source = models.TextField() # source package nameformat
+ binary = models.TextField(null=True) # lists binary packages which a source package can produce
+ architecture = models.TextField(null=True) # all, i386, ...
+ version = models.TextField() # The format is: [epoch:]upstream_version[-debian_revision]
+ maintainer = models.TextField()
+ uploaders = models.TextField(null=True) # Names and emails of co-maintainers
+ homepage = models.TextField(null=True)
+ vcs_browser = models.TextField(null=True)
+ vcs_arch = models.TextField(null=True)
+ vcs_bzr = models.TextField(null=True)
+ vcs_cvs = models.TextField(null=True)
+ vcs_darcs = models.TextField(null=True)
+ vcs_git = models.TextField(null=True)
+ vcs_hg = models.TextField(null=True)
+ vcs_mtn = models.TextField(null=True)
+ vcs_snv = models.TextField(null=True)
+ testsuite = models.TextField(null=True)
+ dgit = models.TextField(null=True)
+ standards_version = models.TextField() # most recent version of the standards the pkg complies
+ build_depends = models.TextField(null=True)
+ build_depends_indep = models.TextField(null=True)
+ build_depends_arch = models.TextField(null=True)
+ build_conflicts = models.TextField(null=True)
+ build_conflicts_indep = models.TextField(null=True)
+ build_conflicts_arch = models.TextField(null=True)
+ package_list = models.TextField(
+ null=True
+ ) # all the packages that can be built from the source package
+
+ def __init__(self, *args, **kwargs):
+ """Sanatize kwargs by removing multi-lists before contructing DscFile"""
+ for kw in ["files", "checksums_sha1", "checksums_sha256", "checksums_sha512"]:
+ if kw in kwargs:
+ kwargs.pop(kw)
+ super().__init__(*args, **kwargs)
+
+ def derived_dsc_filename(self):
+ """Print a nice name for the Dsc file."""
+ return "{}_{}.{}".format(self.source, self.version, self.SUFFIX)
+
+ def derived_dir(self, component=""):
+ """Assemble full dir in pool directory."""
+ sourcename = self.source
+ prefix = sourcename[0]
+ return os.path.join(
+ "pool",
+ component,
+ prefix,
+ sourcename,
+ )
+
+ def derived_path(self, name, component=""):
+ """Assemble filename in pool directory."""
+ return os.path.join(self.derived_dir(component), name)
+
+ @property
+ def checksums_sha1(self):
+ """Generate 'Checksums-Sha1' list from content artifacts."""
+ contents = []
+ for content_artifact in self.contentartifact_set.all():
+ if content_artifact:
+ if content_artifact.artifact:
+ sha1 = content_artifact.artifact.sha1
+ size = content_artifact.artifact.size
+ else:
+ remote_artifact = content_artifact.remoteartifact_set.first()
+ sha1 = remote_artifact.sha1
+ size = remote_artifact.size
+ # Sha1 is optional so filter out incomplete data
+ if sha1 is not None:
+ contents.append(
+ {
+ "name": os.path.basename(content_artifact.relative_path),
+ "sha1": sha1,
+ "size": size,
+ }
+ )
+ return contents
+
+ @property
+ def checksums_sha256(self):
+ """Generate 'Checksums-Sha256' list from content artifacts."""
+ contents = []
+ for content_artifact in self.contentartifact_set.all():
+ if content_artifact:
+ if content_artifact.artifact:
+ sha256 = content_artifact.artifact.sha256
+ size = content_artifact.artifact.size
+ else:
+ remote_artifact = content_artifact.remoteartifact_set.first()
+ sha256 = remote_artifact.sha256
+ size = remote_artifact.size
+ # Sha256 is required so better to not filter out incomplete data
+ contents.append(
+ {
+ "name": os.path.basename(content_artifact.relative_path),
+ "sha256": sha256,
+ "size": size,
+ }
+ )
+ return contents
+
+ @property
+ def checksums_sha512(self):
+ """Generate 'Checksums-Sha512' list from content artifacts."""
+ contents = []
+ for content_artifact in self.contentartifact_set.all():
+ if content_artifact:
+ if content_artifact.artifact:
+ sha512 = content_artifact.artifact.sha512
+ size = content_artifact.artifact.size
+ else:
+ remote_artifact = content_artifact.remoteartifact_set.first()
+ sha512 = remote_artifact.sha512
+ size = remote_artifact.size
+ # Sha512 is optional so filter out incomplete data
+ if sha512 is not None:
+ contents.append(
+ {
+ "name": os.path.basename(content_artifact.relative_path),
+ "sha512": sha512,
+ "size": size,
+ }
+ )
+ return contents
+
+ @property
+ def files(self):
+ """Generate 'Files' list from content artifacts."""
+ contents = []
+ for content_artifact in self.contentartifact_set.all():
+ if content_artifact:
+ if content_artifact.artifact:
+ md5 = content_artifact.artifact.md5
+ size = content_artifact.artifact.size
+ else:
+ remote_artifact = content_artifact.remoteartifact_set.first()
+ md5 = remote_artifact.md5
+ size = remote_artifact.size
+ # md5 is required so better to not filter out incomplete data
+ contents.append(
+ {
+ "name": os.path.basename(content_artifact.relative_path),
+ "md5sum": md5,
+ "size": size,
+ }
+ )
+ return contents
+
+ repo_key_fields = ("source", "version")
+
+ class Meta:
+ default_related_name = "%(app_label)s_%(model_name)s"

pulp_deb/app/models/content/structure_content.py

@@ -16,7 +16,7 @@
 
 from pulpcore.plugin.models import Content
 
-from pulp_deb.app.models import Package
+from pulp_deb.app.models import Package, SourcePackage
 
 
 BOOL_CHOICES = [(True, "yes"), (False, "no")]
@@ -101,3 +101,21 @@ class PackageReleaseComponent(Content):
  class Meta:
  default_related_name = "%(app_label)s_%(model_name)s"
  unique_together = (("package", "release_component"),)
+
+
+class SourcePackageReleaseComponent(Content):
+ """
+ The SourcePackageReleaseComponent.
+
+ This is the join table that decides, which Source Package (in which RepositoryVersions) belong
+ to which ReleaseComponents.
+ """
+
+ TYPE = "source_package_release_component"
+
+ source_package = models.ForeignKey(SourcePackage, on_delete=models.CASCADE)
+ release_component = models.ForeignKey(ReleaseComponent, on_delete=models.CASCADE)
+
+ class Meta:
+ default_related_name = "%(app_label)s_%(model_name)s"
+ unique_together = (("source_package", "release_component"),)

pulp_deb/app/models/content/verbatim_metadata.py

@@ -115,3 +115,33 @@ def main_artifact(self):
  Retrieve the uncompressed SHA256SUMS artifact.
  """
  return self._artifacts.get(sha256=self.sha256)
+
+
+class SourceIndex(Content):
+ """
+ The "SourceIndex" content type.
+
+ This model represents the Sources file for a specific
+ component.
+ It's artifacts should include all (non-)compressed versions
+ of the upstream Sources file.
+ """
+
+ TYPE = "source_index"
+
+ release = models.ForeignKey(ReleaseFile, on_delete=models.CASCADE)
+ component = models.CharField(max_length=255)
+ relative_path = models.TextField()
+ sha256 = models.CharField(max_length=255)
+
+ class Meta:
+ default_related_name = "%(app_label)s_%(model_name)s"
+ verbose_name_plural = "SourceIndices"
+ unique_together = (("relative_path", "sha256"),)
+
+ @property
+ def main_artifact(self):
+ """
+ Retrieve teh uncompressed SourceIndex artifact.
+ """
+ return self._artifacts.get(sha256=self.sha256)

pulp_deb/app/models/repository.py

@@ -15,6 +15,9 @@
  ReleaseArchitecture,
  ReleaseComponent,
  ReleaseFile,
+ SourceIndex,
+ SourcePackage,
+ SourcePackageReleaseComponent,
 )
 
 
@@ -35,6 +38,9 @@ class AptRepository(Repository):
  ReleaseArchitecture,
  ReleaseComponent,
  ReleaseFile,
+ SourceIndex,
+ SourcePackage,
+ SourcePackageReleaseComponent,
  ]
  REMOTE_TYPES = [
  AptRemote,

pulp_deb/app/serializers/__init__.py

@@ -13,6 +13,10 @@
  ReleaseArchitectureSerializer,
  ReleaseComponentSerializer,
  ReleaseFileSerializer,
+ SourceIndexSerializer,
+ DscFile822Serializer,
+ SourcePackageSerializer,
+ SourcePackageReleaseComponentSerializer,
 )
 
 from .publication_serializers import (