Epic 11.5 - Import APIs

.github/workflows/unit-test-shards.json

@@ -237,6 +237,7 @@
       "cms/djangoapps/api/",
       "cms/djangoapps/cms_user_tasks/",
       "cms/djangoapps/course_creators/",
+      "cms/djangoapps/import_from_modulestore/",
       "cms/djangoapps/export_course_metadata/",
       "cms/djangoapps/maintenance/",
       "cms/djangoapps/models/",

cms/djangoapps/import_from_modulestore/README.rst

@@ -0,0 +1,32 @@
+========================
+Course to Library Import
+========================
+
+The new Django application `import_from_modulestore` is designed to
+automate the process of importing course OLX content from Modulestore to
+Content Libraries. The application allows users to easily and quickly
+migrate existing course content, minimizing the manual work and potential
+errors associated with manual migration.
+The new app makes the import process automated and easy to manage.
+
+The main problems solved by the application:
+
+* Reducing the time to import course content.
+* Ensuring data integrity during the transfer.
+* Ability to choose which content to import before the final import.
+
+------------------------------
+Course to Library Import Usage
+------------------------------
+
+* Import course elements at the level of sections, subsections, units,
+  and xblocks into the Content Libraries.
+* Choose the structure of this import, whether it will be only xblocks
+  from a particular course or full sections/subsections/units.
+* Store the history of imports.
+* Synchronize the course content with the library content (when re-importing,
+  the blocks can be updated according to changes in the original course).
+* The new import mechanism ensures data integrity at the time of importing
+  by saving the course in StagedContent.
+
+

cms/djangoapps/import_from_modulestore/admin.py

@@ -0,0 +1,159 @@
+"""
+This module contains the admin configuration for the Import model.
+"""
+from django import forms
+from django.contrib import admin, messages
+from django.http import HttpResponseRedirect
+from django.template.response import TemplateResponse
+from django.utils.translation import gettext_lazy as _
+
+from opaque_keys.edx.keys import UsageKey
+from opaque_keys import InvalidKeyError
+
+from . import api
+from .data import ImportStatus
+from .models import Import, PublishableEntityImport, PublishableEntityMapping
+from .tasks import save_legacy_content_to_staged_content_task
+
+COMPOSITION_LEVEL_CHOICES = (
+    ('xblock', _('XBlock')),
+    ('vertical', _('Unit')),
+    ('sequential', _('Section')),
+    ('chapter', _('Chapter')),
+)
+
+
+def _validate_block_keys(model_admin, request, block_keys_to_import):
+    """
+    Validate the block keys to import.
+    """
+    block_keys_to_import = block_keys_to_import.split(',')
+    for block_key in block_keys_to_import:
+        try:
+            UsageKey.from_string(block_key)
+        except InvalidKeyError:
+            model_admin.message_user(
+                request,
+                _('Invalid block key: {block_key}').format(block_key=block_key),
+                level=messages.ERROR,
+            )
+            return False
+    return True
+
+
+class ImportActionForm(forms.Form):
+    """
+    Form for the CourseToLibraryImport action.
+    """
+
+    composition_level = forms.ChoiceField(
+        choices=COMPOSITION_LEVEL_CHOICES,
+        required=False,
+        label='Composition Level'
+    )
+    override = forms.BooleanField(
+        required=False,
+        label='Override Existing Content'
+    )
+    block_keys_to_import = forms.CharField(
+        widget=forms.Textarea(attrs={
+            'placeholder': 'Comma separated list of block keys to import.',
+            'rows': 4
+        }),
+        required=False,
+        label='Block Keys to Import'
+    )
+
+
+class ImportAdmin(admin.ModelAdmin):
+    """
+    Admin configuration for the Import model.
+    """
+
+    list_display = (
+        'uuid',
+        'created',
+        'status',
+        'source_key',
+        'target',
+    )
+    list_filter = (
+        'status',
+    )
+    search_fields = (
+        'source_key',
+        'target',
+    )
+
+    raw_id_fields = ('user',)
+    readonly_fields = ('status',)
+    actions = ['import_course_to_library_action']
+
+    def save_model(self, request, obj, form, change):
+        """
+        Launches the creation of Staged Content after creating a new import instance.
+        """
+        is_created = not getattr(obj, 'id', None)
+        super().save_model(request, obj, form, change)
+        if is_created:
+            save_legacy_content_to_staged_content_task.delay_on_commit(obj.uuid)
+
+    def import_course_to_library_action(self, request, queryset):
+        """
+        Import selected courses to the library.
+        """
+        form = ImportActionForm(request.POST or None)
+
+        if request.POST and 'apply' in request.POST:
+            if form.is_valid():
+                block_keys_string = form.cleaned_data['block_keys_to_import']
+                are_keys_valid = _validate_block_keys(self, request, block_keys_string)
+                if not are_keys_valid:
+                    return
+
+                target_key_string = block_keys_string.split(',') if block_keys_string else []
+                composition_level = form.cleaned_data['composition_level']
+                override = form.cleaned_data['override']
+
+                if not queryset.count() == queryset.filter(status=ImportStatus.READY).count():
+                    self.message_user(
+                        request,
+                        _('Only imports with status "Ready" can be imported to the library.'),
+                        level=messages.ERROR,
+                    )
+                    return
+
+                for obj in queryset:
+                    api.import_course_staged_content_to_library(
+                        usage_ids=target_key_string,
+                        import_uuid=str(obj.uuid),
+                        user_id=request.user.pk,
+                        composition_level=composition_level,
+                        override=override,
+                    )
+
+                self.message_user(
+                    request,
+                    _('Importing courses to library.'),
+                    level=messages.SUCCESS,
+                )
+
+                return HttpResponseRedirect(request.get_full_path())
+
+        return TemplateResponse(
+            request,
+            'admin/custom_course_import_form.html',
+            {
+                'form': form,
+                'queryset': queryset,
+                'action_name': 'import_course_to_library_action',
+                'title': _('Import Selected Courses to Library')
+            }
+        )
+
+    import_course_to_library_action.short_description = _('Import selected courses to library')
+
+
+admin.site.register(Import, ImportAdmin)
+admin.site.register(PublishableEntityImport)
+admin.site.register(PublishableEntityMapping)

cms/djangoapps/import_from_modulestore/api.py

@@ -0,0 +1,39 @@
+"""
+API for course to library import.
+"""
+from .models import Import as _Import
+from .tasks import import_course_staged_content_to_library_task, save_legacy_content_to_staged_content_task
+
+
+def import_course_staged_content_to_library(
+    usage_ids: list[str],
+    import_uuid: str,
+    user_id: int,
+    composition_level: str,
+    override: bool
+) -> None:
+    """
+    Import staged content to a library.
+    """
+    import_course_staged_content_to_library_task.apply_async(
+        kwargs={
+            'usage_ids': usage_ids,
+            'import_uuid': import_uuid,
+            'user_id': user_id,
+            'composition_level': composition_level,
+            'override': override,
+        },
+    )
+
+
+def create_import(source_key, user_id: int, learning_package_id: int) -> _Import:
+    """
+    Create a new import task to import a course to a library.
+    """
+    import_from_modulestore = _Import.objects.create(
+        source_key=source_key,
+        target_id=learning_package_id,
+        user_id=user_id,
+    )
+    save_legacy_content_to_staged_content_task.delay_on_commit(import_from_modulestore.uuid)
+    return import_from_modulestore

cms/djangoapps/import_from_modulestore/apps.py

@@ -0,0 +1,19 @@
+"""
+App for importing from the modulestore tools.
+"""
+
+from django.apps import AppConfig
+
+
+class ImportFromModulestoreConfig(AppConfig):
+    """
+    App for importing legacy content from the modulestore.
+    """
+
+    name = 'cms.djangoapps.import_from_modulestore'
+
+    def ready(self):
+        """
+        Connect handlers to signals.
+        """
+        from . import signals, tasks  # pylint: disable=unused-import, import-outside-toplevel

cms/djangoapps/import_from_modulestore/constants.py

@@ -0,0 +1,5 @@
+"""
+Constants for import_from_modulestore app
+"""
+
+IMPORT_FROM_MODULESTORE_PURPOSE = "import_from_modulestore"

cms/djangoapps/import_from_modulestore/data.py

@@ -0,0 +1,50 @@
+"""
+This module contains the data models for the import_from_modulestore app.
+"""
+from enum import Enum
+
+from django.db.models import TextChoices
+from django.utils.translation import gettext_lazy as _
+
+
+class ImportStatus(TextChoices):
+    """
+    The status of this course import.
+    """
+
+    # PENDING: The import has been created, but the OLX and related data are not yet in the library.
+    # It is not ready to be read.
+    PENDING = 'pending', _('Pending')
+    # READY: The content is staged and ready to be read.
+    READY = 'ready', _('Ready')
+    # IMPORTED: The content has been imported into the library.
+    IMPORTED = 'imported', _('Imported')
+    # CANCELED: The import was canceled before it was imported.
+    CANCELED = 'canceled', _('Canceled')
+    # ERROR: The content could not be imported.
+    ERROR = 'error', _('Error')
+
+
+class CompositionLevel(Enum):
+    """
+    Enumeration of composition levels for course content.
+
+    Defines the different levels of composition for course content,
+    including chapters, sequentials, verticals, and xblocks.
+    It also categorizes these levels into complicated and flat
+    levels for easier processing.
+    """
+
+    CHAPTER = 'chapter'
+    SEQUENTIAL = 'sequential'
+    VERTICAL = 'vertical'
+    XBLOCK = 'xblock'
+    COMPLICATED_LEVELS = [CHAPTER, SEQUENTIAL, VERTICAL]
+    FLAT_LEVELS = [XBLOCK]
+
+    @classmethod
+    def values(cls):
+        """
+        Returns all levels of composition levels.
+        """
+        return [composition_level.value for composition_level in cls]