Migrate Debian content types for structured publishing

fixes #7865
https://pulp.plan.io/issues/7865

CHANGES/7865.feature

@@ -0,0 +1 @@
+Added the ability to migrate additional Debian content types needed for structured publishing.

README.md

@@ -5,7 +5,7 @@ Supported plugins:
  - Pulp 2 ISO can be migrated into Pulp 3 File.
  - Pulp 2 Docker can be migrated into Pulp 3 Container.
  - Pulp 2 RPM can be migrated into Pulp 3 RPM.
- - Pulp 2 Deb can be migrated into Pulp 3 Deb (tech preview, packages only).
+ - Pulp 2 Deb can be migrated into Pulp 3 Deb (tech preview).
 
 
 For more information, please see the [documentation](https://pulp-2to3-migration.readthedocs.io/en/latest/).

docs/workflows.rst

@@ -139,11 +139,11 @@ The current level of Debian support
 -----------------------------------
 
 .. important::
-   The migration plugins Debian support is still undergoing active development, and should be considered to have "tech preview" status.
+   The migration plugins Debian support has not yet undergone large scale robustness testing, and should be considered to have "tech preview" status.
 
 The migration plugin can be used to create a ``deb`` type migration plan to migrate any Pulp 2 APT repositories into a Pulp 3 instance.
-Only package type content units will be migrated (as well as published using the Pulp 3 Debian plugins APT publisher using simple mode only).
-This creates usable repositories, however, any repository structure information like what packages were organized into which components will not be migrated.
+Both simple and structured content can be migrated making the ``deb`` migration feature complete.
 
-The plan is to add the ability to migrate releases and component type content units in a second step.
-Users who migrate their packages now, may be able to migrate those additional content units once that functionality is added, but should be prepared to start over with a clean Pulp 3 instance if necessary.
+.. note::
+   Since the Pulp 2 version of the ``pulp_deb`` plugin had nothing equivalent to verbatim publications, it is not possible to migrate content for Pulp 3 verbatim publications.
+   Only the APT publisher (in both simple and structured mode) is supported for ``pulp_deb`` content that was migrated from Pulp 2.

pulp_2to3_migration/app/migrations/0022_add_structured_deb_types.py

@@ -0,0 +1,89 @@
+# Generated by Django 2.2.17 on 2021-02-08 15:40
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_lifecycle.mixins
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0021_pulp2content_add_subid'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Pulp2DebReleaseArchitecture',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('architecture', models.TextField()),
+                ('distribution', models.TextField()),
+                ('codename', models.TextField()),
+                ('suite', models.TextField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_release_architecture_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'deb_release_architecture_detail_model',
+                'unique_together': {('architecture', 'distribution', 'codename', 'suite')},
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='Pulp2DebRelease',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('distribution', models.TextField()),
+                ('codename', models.TextField()),
+                ('suite', models.TextField(null=True)),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_release_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'deb_release_detail_model',
+                'unique_together': {('codename', 'suite', 'distribution')},
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='Pulp2DebComponentPackage',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('package_relative_path', models.TextField()),
+                ('package_sha256', models.TextField()),
+                ('component', models.TextField()),
+                ('distribution', models.TextField()),
+                ('codename', models.TextField()),
+                ('suite', models.TextField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_component_package_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'deb_component_package_detail_model',
+                'unique_together': {('package_relative_path', 'package_sha256', 'component', 'distribution', 'codename', 'suite')},
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='Pulp2DebComponent',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('distribution', models.TextField()),
+                ('codename', models.TextField()),
+                ('component', models.TextField()),
+                ('suite', models.TextField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_component_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'deb_component_detail_model',
+                'unique_together': {('component', 'codename', 'suite', 'distribution')},
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+    ]

pulp_2to3_migration/app/plugin/deb/migrator.py

@@ -1,5 +1,7 @@
 import asyncio
 
+from collections import OrderedDict
+
 from pulpcore.plugin.stages import (
     ArtifactSaver,
     ContentSaver,
@@ -35,13 +37,19 @@ class DebMigrator(Pulp2to3PluginMigrator):
     pulp2_plugin = 'deb'
     pulp2_content_models = {
         'deb': pulp2_models.DebPackage,
+        'deb_release': pulp2_models.DebRelease,
+        'deb_component': pulp2_models.DebComponent,
     }
     pulp2_collection = 'units_deb'
     pulp3_plugin = 'pulp_deb'
     pulp3_repository = pulp3_models.AptRepository
-    content_models = {
-        'deb': pulp_2to3_models.Pulp2DebPackage,
-    }
+    content_models = OrderedDict([
+        ('deb_release', pulp_2to3_models.Pulp2DebRelease),
+        ('deb', pulp_2to3_models.Pulp2DebPackage),
+        ('deb_component', pulp_2to3_models.Pulp2DebComponent),
+        ('deb_component2', pulp_2to3_models.Pulp2DebComponentPackage),
+        ('deb_component3', pulp_2to3_models.Pulp2DebReleaseArchitecture),
+    ])
     importer_migrators = {
         'deb_importer': repository.DebImporter,
     }
@@ -50,6 +58,14 @@ class DebMigrator(Pulp2to3PluginMigrator):
     }
     future_types = {
         'deb': pulp_2to3_models.Pulp2DebPackage,
+        'deb_release': pulp_2to3_models.Pulp2DebRelease,
+        'deb_component': pulp_2to3_models.Pulp2DebComponent,
+    }
+    artifactless_types = {
+        'deb_release': pulp_2to3_models.Pulp2DebRelease,
+        'deb_component': pulp_2to3_models.Pulp2DebComponent,
+        'deb_component2': pulp_2to3_models.Pulp2DebComponentPackage,
+        'deb_component3': pulp_2to3_models.Pulp2DebReleaseArchitecture,
     }
 
     @classmethod