diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 784f607f06ba..00215fde949c 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -237,6 +237,7 @@ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", "cms/djangoapps/course_creators/", + "cms/djangoapps/import_from_modulestore/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", "cms/djangoapps/models/", diff --git a/cms/djangoapps/import_from_modulestore/README.rst b/cms/djangoapps/import_from_modulestore/README.rst new file mode 100644 index 000000000000..47e4c77b2f7a --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/README.rst @@ -0,0 +1,32 @@ +======================== +Course to Library Import +======================== + +The new Django application `import_from_modulestore` is designed to +automate the process of importing course OLX content from Modulestore to +Content Libraries. The application allows users to easily and quickly +migrate existing course content, minimizing the manual work and potential +errors associated with manual migration. +The new app makes the import process automated and easy to manage. + +The main problems solved by the application: + +* Reducing the time to import course content. +* Ensuring data integrity during the transfer. +* Ability to choose which content to import before the final import. + +------------------------------ +Course to Library Import Usage +------------------------------ + +* Import course elements at the level of sections, subsections, units, + and xblocks into the Content Libraries. +* Choose the structure of this import, whether it will be only xblocks + from a particular course or full sections/subsections/units. +* Store the history of imports. +* Synchronize the course content with the library content (when re-importing, + the blocks can be updated according to changes in the original course). +* The new import mechanism ensures data integrity at the time of importing + by saving the course in StagedContent. + + diff --git a/cms/djangoapps/import_from_modulestore/__init__.py b/cms/djangoapps/import_from_modulestore/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py new file mode 100644 index 000000000000..dd01515dc69c --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/admin.py @@ -0,0 +1,159 @@ +""" +This module contains the admin configuration for the Import model. +""" +from django import forms +from django.contrib import admin, messages +from django.http import HttpResponseRedirect +from django.template.response import TemplateResponse +from django.utils.translation import gettext_lazy as _ + +from opaque_keys.edx.keys import UsageKey +from opaque_keys import InvalidKeyError + +from . import api +from .data import ImportStatus +from .models import Import, PublishableEntityImport, PublishableEntityMapping +from .tasks import save_legacy_content_to_staged_content_task + +COMPOSITION_LEVEL_CHOICES = ( + ('xblock', _('XBlock')), + ('vertical', _('Unit')), + ('sequential', _('Section')), + ('chapter', _('Chapter')), +) + + +def _validate_block_keys(model_admin, request, block_keys_to_import): + """ + Validate the block keys to import. + """ + block_keys_to_import = block_keys_to_import.split(',') + for block_key in block_keys_to_import: + try: + UsageKey.from_string(block_key) + except InvalidKeyError: + model_admin.message_user( + request, + _('Invalid block key: {block_key}').format(block_key=block_key), + level=messages.ERROR, + ) + return False + return True + + +class ImportActionForm(forms.Form): + """ + Form for the CourseToLibraryImport action. + """ + + composition_level = forms.ChoiceField( + choices=COMPOSITION_LEVEL_CHOICES, + required=False, + label='Composition Level' + ) + override = forms.BooleanField( + required=False, + label='Override Existing Content' + ) + block_keys_to_import = forms.CharField( + widget=forms.Textarea(attrs={ + 'placeholder': 'Comma separated list of block keys to import.', + 'rows': 4 + }), + required=False, + label='Block Keys to Import' + ) + + +class ImportAdmin(admin.ModelAdmin): + """ + Admin configuration for the Import model. + """ + + list_display = ( + 'uuid', + 'created', + 'status', + 'source_key', + 'target', + ) + list_filter = ( + 'status', + ) + search_fields = ( + 'source_key', + 'target', + ) + + raw_id_fields = ('user',) + readonly_fields = ('status',) + actions = ['import_course_to_library_action'] + + def save_model(self, request, obj, form, change): + """ + Launches the creation of Staged Content after creating a new import instance. + """ + is_created = not getattr(obj, 'id', None) + super().save_model(request, obj, form, change) + if is_created: + save_legacy_content_to_staged_content_task.delay_on_commit(obj.uuid) + + def import_course_to_library_action(self, request, queryset): + """ + Import selected courses to the library. + """ + form = ImportActionForm(request.POST or None) + + if request.POST and 'apply' in request.POST: + if form.is_valid(): + block_keys_string = form.cleaned_data['block_keys_to_import'] + are_keys_valid = _validate_block_keys(self, request, block_keys_string) + if not are_keys_valid: + return + + target_key_string = block_keys_string.split(',') if block_keys_string else [] + composition_level = form.cleaned_data['composition_level'] + override = form.cleaned_data['override'] + + if not queryset.count() == queryset.filter(status=ImportStatus.READY).count(): + self.message_user( + request, + _('Only imports with status "Ready" can be imported to the library.'), + level=messages.ERROR, + ) + return + + for obj in queryset: + api.import_course_staged_content_to_library( + usage_ids=target_key_string, + import_uuid=str(obj.uuid), + user_id=request.user.pk, + composition_level=composition_level, + override=override, + ) + + self.message_user( + request, + _('Importing courses to library.'), + level=messages.SUCCESS, + ) + + return HttpResponseRedirect(request.get_full_path()) + + return TemplateResponse( + request, + 'admin/custom_course_import_form.html', + { + 'form': form, + 'queryset': queryset, + 'action_name': 'import_course_to_library_action', + 'title': _('Import Selected Courses to Library') + } + ) + + import_course_to_library_action.short_description = _('Import selected courses to library') + + +admin.site.register(Import, ImportAdmin) +admin.site.register(PublishableEntityImport) +admin.site.register(PublishableEntityMapping) diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py new file mode 100644 index 000000000000..178545827c91 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/api.py @@ -0,0 +1,39 @@ +""" +API for course to library import. +""" +from .models import Import as _Import +from .tasks import import_course_staged_content_to_library_task, save_legacy_content_to_staged_content_task + + +def import_course_staged_content_to_library( + usage_ids: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library. + """ + import_course_staged_content_to_library_task.apply_async( + kwargs={ + 'usage_ids': usage_ids, + 'import_uuid': import_uuid, + 'user_id': user_id, + 'composition_level': composition_level, + 'override': override, + }, + ) + + +def create_import(source_key, user_id: int, learning_package_id: int) -> _Import: + """ + Create a new import task to import a course to a library. + """ + import_from_modulestore = _Import.objects.create( + source_key=source_key, + target_id=learning_package_id, + user_id=user_id, + ) + save_legacy_content_to_staged_content_task.delay_on_commit(import_from_modulestore.uuid) + return import_from_modulestore diff --git a/cms/djangoapps/import_from_modulestore/apps.py b/cms/djangoapps/import_from_modulestore/apps.py new file mode 100644 index 000000000000..ea100d20f689 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/apps.py @@ -0,0 +1,19 @@ +""" +App for importing from the modulestore tools. +""" + +from django.apps import AppConfig + + +class ImportFromModulestoreConfig(AppConfig): + """ + App for importing legacy content from the modulestore. + """ + + name = 'cms.djangoapps.import_from_modulestore' + + def ready(self): + """ + Connect handlers to signals. + """ + from . import signals, tasks # pylint: disable=unused-import, import-outside-toplevel diff --git a/cms/djangoapps/import_from_modulestore/constants.py b/cms/djangoapps/import_from_modulestore/constants.py new file mode 100644 index 000000000000..099acfbc2b7b --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/constants.py @@ -0,0 +1,5 @@ +""" +Constants for import_from_modulestore app +""" + +IMPORT_FROM_MODULESTORE_PURPOSE = "import_from_modulestore" diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py new file mode 100644 index 000000000000..69aa71e1becf --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/data.py @@ -0,0 +1,50 @@ +""" +This module contains the data models for the import_from_modulestore app. +""" +from enum import Enum + +from django.db.models import TextChoices +from django.utils.translation import gettext_lazy as _ + + +class ImportStatus(TextChoices): + """ + The status of this course import. + """ + + # PENDING: The import has been created, but the OLX and related data are not yet in the library. + # It is not ready to be read. + PENDING = 'pending', _('Pending') + # READY: The content is staged and ready to be read. + READY = 'ready', _('Ready') + # IMPORTED: The content has been imported into the library. + IMPORTED = 'imported', _('Imported') + # CANCELED: The import was canceled before it was imported. + CANCELED = 'canceled', _('Canceled') + # ERROR: The content could not be imported. + ERROR = 'error', _('Error') + + +class CompositionLevel(Enum): + """ + Enumeration of composition levels for course content. + + Defines the different levels of composition for course content, + including chapters, sequentials, verticals, and xblocks. + It also categorizes these levels into complicated and flat + levels for easier processing. + """ + + CHAPTER = 'chapter' + SEQUENTIAL = 'sequential' + VERTICAL = 'vertical' + XBLOCK = 'xblock' + COMPLICATED_LEVELS = [CHAPTER, SEQUENTIAL, VERTICAL] + FLAT_LEVELS = [XBLOCK] + + @classmethod + def values(cls): + """ + Returns all levels of composition levels. + """ + return [composition_level.value for composition_level in cls] diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py new file mode 100644 index 000000000000..1595e17da95c --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -0,0 +1,372 @@ +""" +Helper functions for importing course content into a library. +""" +from datetime import datetime, timezone +import logging +import mimetypes +import os +import secrets + +from django.db import transaction +from django.db.utils import IntegrityError +from lxml import etree + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import CourseLocator, LibraryUsageLocatorV2 +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import ContainerVersion + +from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.django import modulestore + +from .data import CompositionLevel +from .models import PublishableEntityMapping, PublishableEntityImport + + +log = logging.getLogger(__name__) +parser = etree.XMLParser(strip_cdata=False) + + +class ImportClient: + """ + Client for importing course content into a library. + + This class handles the import of course content from staged content into a + content library, creating the appropriate container hierarchy based on the + specified composition level. + """ + + CONTAINER_CREATORS_MAP = { + 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version + 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version + 'vertical': authoring_api.create_unit_and_version, + } + + CONTAINER_OVERRIDERS_MAP = { + 'chapter': authoring_api.create_next_unit_version, # TODO: replace with create_next_module_version + 'sequential': authoring_api.create_next_unit_version, # TODO: replace with create_next_section_version + 'vertical': authoring_api.create_next_unit_version, + } + + def __init__(self, import_event, block_usage_id_to_import, staged_content, composition_level, override=False): + self.import_event = import_event + self.block_usage_id_to_import = block_usage_id_to_import + self.staged_content = staged_content + self.composition_level = composition_level + self.override = override + + self.user_id = import_event.user_id + self.content_library = import_event.target.contentlibrary + self.library_key = self.content_library.library_key + self.parser = etree.XMLParser(strip_cdata=False) + + def import_from_staged_content(self): + """ + Import staged content into a library. + """ + node = etree.fromstring(self.staged_content.olx, parser=parser) + usage_key = UsageKey.from_string(self.block_usage_id_to_import) + block_to_import = get_block_to_import(node, usage_key) + if block_to_import is None: + return + + self._process_import(self.block_usage_id_to_import, block_to_import) + + def get_or_create_container(self, container_type, key, display_name): + """ + Create a container of the specified type. + + Creates a container (e.g., chapter, sequential, vertical) in the + content library. + """ + container_creator_func = self.CONTAINER_CREATORS_MAP.get(container_type) + container_override_func = self.CONTAINER_OVERRIDERS_MAP.get(container_type) + if not all((container_creator_func, container_override_func)): + raise ValueError(f"Unknown container type: {container_type}") + + container_version = self.content_library.learning_package.publishable_entities.filter(key=key).first() + if container_version and self.override: + container_version = container_override_func( + container_version.container, + title=display_name or f"New {container_type}", + components=[], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + elif not container_version: + _, container_version = container_creator_func( + self.import_event.target_id, + key=key or secrets.token_hex(16), + title=display_name or f"New {container_type}", + components=[], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + + return container_version + + def _process_import(self, usage_id, block_to_import): + """ + Process import of a block from staged content into a library. + + Imports a block and its children into the library based on the + composition level. It handles both simple and complicated blocks, creating + the necessary container hierarchy. + """ + usage_key = UsageKey.from_string(usage_id) + result = [] + + if block_to_import.tag not in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_simple_block(block_to_import, usage_key) + + for child in block_to_import.getchildren(): + child_usage_id = get_usage_id_from_staged_content(self.staged_content, child.get('url_name')) + if not child_usage_id: + continue + result.extend(self._import_child_block(child, child_usage_id)) + + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return [component for component in result if not isinstance(component, ContainerVersion)] + return result + + def _import_simple_block(self, block_to_import, usage_key) -> list: + """ + Import a simple block into the library. + + Creates a block in the library from the staged content block. + It returns a list containing the created component version. + """ + component_version = self._create_block_in_library(block_to_import, usage_key) + return [component_version] if component_version else [] + + def _import_child_block(self, child, child_usage_id): + """ + Import a child block into the library. + + Determines whether the child block is simple or complicated and + delegates the import process to the appropriate helper method. + """ + child_usage_key = UsageKey.from_string(child_usage_id) + if child.tag in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_complicated_child(child, child_usage_id) + else: + return self._import_simple_block(child, child_usage_key) + + def _import_complicated_child(self, child, child_usage_id): + """ + Import a complicated child block into the library + + Handles the import of complicated child blocks, including creating + containers and updating components. + Returns a list containing the created container version. + """ + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return self._process_import(child_usage_id, child) + + container_version = self.get_or_create_container( + child.tag, + child.get('url_name'), + child.get('display_name', child.tag) + ) + child_component_versions = self._process_import(child_usage_id, child) + self._update_container_components(container_version, child_component_versions) + return [container_version] + + def _update_container_components(self, container_version, component_versions): + """ + Update components of a container. + """ + return authoring_api.create_next_container_version( + container_pk=container_version.container.pk, + title=container_version.title, + publishable_entities_pks=[ + cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions + ], + entity_version_pks=[cv.pk for cv in component_versions], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + container_version_cls=container_version.__class__, + ) + + def _create_block_in_library(self, block_to_import, usage_key): + """ + Create a block in a library from a staged content block. + """ + now = datetime.now(tz=timezone.utc) + staged_content_files = content_staging_api.get_staged_content_static_files(self.staged_content.id) + + with transaction.atomic(): + component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) + does_component_exist = authoring_api.get_components( + self.import_event.target_id + ).filter(local_key=usage_key.block_id).exists() + + if does_component_exist: + if not self.override: + log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.") + return + else: + component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import)) + else: + # Create component (regardless of override path) + # FIXME check override logic + _, library_usage_key = api.validate_can_add_block_to_library( + self.library_key, + block_to_import.tag, + usage_key.block_id, + ) + authoring_api.create_component( + self.import_event.target_id, + component_type=component_type, + local_key=usage_key.block_id, + created=now, + created_by=self.import_event.user_id, + ) + component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) + + self._process_staged_content_files( + component_version, + staged_content_files, + usage_key, + block_to_import, + now, + ) + _create_publishable_entity_import(self.import_event, usage_key, component_version) + + return component_version + + def _handle_component_override(self, usage_key, new_content): + """ + Create new ComponentVersion for overridden component. + """ + component_version = None + component = self.import_event.target.component_set.filter(local_key=usage_key.block_id).first() + + if component: + library_usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] + lib_key=self.library_key, + block_type=component.component_type.name, + usage_id=component.local_key, + ) + component_version = api.set_library_block_olx(library_usage_key, new_content) + + return component_version + + def _process_staged_content_files( + self, + component_version, + staged_content_files, + usage_key, + block_to_import, + created_at, + ): + """ + Process staged content files for a component. + + Processes the staged content files for a component, creating the + necessary file content and associating it with the component version. + """ + block_olx = etree.tostring(block_to_import).decode('utf-8') + + for staged_content_file_data in staged_content_files: + original_filename = staged_content_file_data.filename + file_basename = os.path.basename(original_filename) + file_basename_no_ext, _ = os.path.splitext(file_basename) + + # Skip files not referenced in the block + if file_basename not in block_olx and file_basename_no_ext not in block_olx: + log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") + continue + + file_data = content_staging_api.get_staged_content_static_file_data( + self.staged_content.id, + original_filename, + ) + if not file_data: + log.error( + f"Staged content {self.staged_content.id} included referenced " + f"file {original_filename}, but no file data was found." + ) + continue + + filename = f"static/{file_basename}" + media_type_str, _ = mimetypes.guess_type(filename) + if not media_type_str: + media_type_str = "application/octet-stream" + + media_type = authoring_api.get_or_create_media_type(media_type_str) + content = authoring_api.get_or_create_file_content( + self.import_event.target_id, + media_type.id, + data=file_data, + created=created_at, + ) + + try: + authoring_api.create_component_version_content(component_version.pk, content.id, key=filename) + except IntegrityError: + pass # Content already exists + + +def _create_publishable_entity_import(import_event, usage_key, component_version) -> PublishableEntityImport: + """ + Creates relations between the imported component and source usage key and import event. + """ + publishable_entity_mapping, _ = _get_or_create_publishable_entity_mapping( + usage_key, + component_version.component + ) + return PublishableEntityImport.objects.create( + import_event=import_event, + result=publishable_entity_mapping, + resulting_draft=component_version.publishable_entity_version, + ) + + +def _get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]: + """ + Creates a mapping between the source usage key and the target publishable entity. + """ + return PublishableEntityMapping.objects.get_or_create( + source_usage_key=usage_key, + target_entity=component.publishable_entity, + target_package=component.learning_package + ) + + +def get_usage_id_from_staged_content(staged_content, block_id): + """ + Get the usage ID from a staged content by block ID. + """ + return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None) + + +def get_block_to_import(node, usage_key): + """ + Get the block to import from a node. + """ + + if node.get('url_name') == usage_key.block_id: + return node + + for child in node.getchildren(): + found = get_block_to_import(child, usage_key) + if found is not None: + return found + + +def get_items_to_import(import_event): + """ + Collect items to import from a course. + """ + items_to_import = [] + if isinstance(import_event.source_key, CourseLocator): + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or [] + ) + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "static_tab"}) or [] + ) + + return items_to_import diff --git a/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py b/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py new file mode 100644 index 000000000000..d759788fcc45 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py @@ -0,0 +1,81 @@ +# Generated by Django 4.2.18 on 2025-04-03 09:55 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import opaque_keys.edx.django.models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('content_staging', '0005_stagedcontent_version_num'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oel_publishing', '0003_containers'), + ] + + operations = [ + migrations.CreateModel( + name='Import', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('imported', 'Imported'), ('canceled', 'Canceled'), ('error', 'Error')], default='pending', max_length=100)), + ('source_key', opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, help_text='The modulestore course', max_length=255)), + ('target', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.learningpackage')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Import from modulestore', + 'verbose_name_plural': 'Imports from modulestore', + }, + ), + migrations.CreateModel( + name='PublishableEntityMapping', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key/ID of the thing that has been imported.', max_length=255)), + ('target_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishableentity')), + ('target_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), + ], + options={ + 'unique_together': {('source_usage_key', 'target_package')}, + }, + ), + migrations.CreateModel( + name='StagedContentForImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('import_event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='import_from_modulestore.import')), + ('staged_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='content_staging.stagedcontent')), + ], + options={ + 'unique_together': {('import_event', 'staged_content')}, + }, + ), + migrations.CreateModel( + name='PublishableEntityImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('import_event', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='import_from_modulestore.import')), + ('result', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='import_from_modulestore.publishableentitymapping')), + ('resulting_draft', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.publishableentityversion')), + ], + options={ + 'unique_together': {('import_event', 'result')}, + }, + ), + ] diff --git a/cms/djangoapps/import_from_modulestore/migrations/__init__.py b/cms/djangoapps/import_from_modulestore/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py new file mode 100644 index 000000000000..9d812f697eb3 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/models.py @@ -0,0 +1,175 @@ +""" +Models for the course to library import app. +""" + +import uuid as uuid_tools +from typing import Self, Optional + +from django.contrib.auth import get_user_model +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from model_utils.models import TimeStampedModel +from opaque_keys.edx.django.models import ( + LearningContextKeyField, + UsageKeyField, +) +from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity + +from .data import ImportStatus + +User = get_user_model() + + +# TODO: Rename app to import_from_modulestore + +class Import(TimeStampedModel): + """ + Represents the action of a user importing a modulestore-based course or legacy + library into a learning-core based learning package (today, that is always a content library). + """ + + uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True, db_index=True) + status = models.CharField(max_length=100, choices=ImportStatus.choices, default=ImportStatus.PENDING) + user = models.ForeignKey(User, on_delete=models.CASCADE) + + source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True) + target = models.ForeignKey(LearningPackage, models.SET_NULL, null=True) + + def __str__(self): + return f'{self.source_key} - {self.target}' + + def ready(self) -> None: + """ + Set import status to ready. + """ + self.status = ImportStatus.READY + self.save() + + def imported(self) -> None: + """ + Set import status to imported and clean related staged content. + """ + self.status = ImportStatus.IMPORTED + self.save() + self.clean_related_staged_content() + + def cancel(self) -> None: + """ + Cancel import action and delete related staged content. + """ + self.status = ImportStatus.CANCELED + self.save() + self.clean_related_staged_content() + + def error(self) -> None: + """ + Set import status to error and clean related staged + """ + self.status = ImportStatus.ERROR + self.save() + self.clean_related_staged_content() + + def clean_related_staged_content(self) -> None: + """ + Clean related staged content. + """ + for staged_content_for_import in self.staged_content_for_import.all(): + staged_content_for_import.staged_content.delete() + + class Meta: + verbose_name = _('Import from modulestore') + verbose_name_plural = _('Imports from modulestore') + + @classmethod + def get_by_uuid(cls, import_uuid: str) -> Self | None: + """ + Get an import task by its ID. + """ + return cls.objects.filter(uuid=import_uuid).first() + + @classmethod + def get_ready_by_uuid(cls, import_uuid: str) -> Self | None: + """ + Get an import task by its UUID. + """ + return cls.objects.filter(uuid=import_uuid, status=ImportStatus.READY).first() + + def get_staged_content_by_block_usage_id(self, block_usage_id: str) -> Optional["StagedContent"]: + """ + Get staged content by block usage ID. + """ + staged_content_for_import = self.staged_content_for_import.filter( + staged_content__tags__icontains=block_usage_id + ).first() + return getattr(staged_content_for_import, 'staged_content', None) + + +class PublishableEntityMapping(TimeStampedModel): + """ + Represents a mapping between a source usage key and a target publishable entity. + """ + + source_usage_key = UsageKeyField( + max_length=255, + help_text=_('Original usage key/ID of the thing that has been imported.'), + ) + target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE) + + class Meta: + unique_together = ('source_usage_key', 'target_package') + + def __str__(self): + return f'{self.source_usage_key} - {self.target_entity}' + + +class PublishableEntityImport(TimeStampedModel): + """ + Represents a container version that has been imported into a content library. + + This is a many-to-many relationship between a container version and a course to library import. + """ + + import_event = models.ForeignKey(Import, on_delete=models.SET_NULL, null=True, blank=True) + result = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True) + resulting_draft = models.OneToOneField( + to='oel_publishing.PublishableEntityVersion', + # a version can be pruned, which would set this to NULL, but not delete the + # entire import record + null=True, + on_delete=models.SET_NULL, + ) + + class Meta: + unique_together = ( + ('import_event', 'result'), + ) + + def __str__(self): + return f'{self.import_event} - {self.result}' + + +class StagedContentForImport(TimeStampedModel): + """ + Represents m2m relationship between an import and staged content created for that import. + """ + + import_event = models.ForeignKey( + Import, + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + staged_content = models.OneToOneField( + to='content_staging.StagedContent', + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + + class Meta: + unique_together = ( + ('import_event', 'staged_content'), + ) + + def __str__(self): + return f'{self.import_event} - {self.staged_content}' diff --git a/cms/djangoapps/import_from_modulestore/permissions.py b/cms/djangoapps/import_from_modulestore/permissions.py new file mode 100644 index 000000000000..0d30f8b6af58 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/permissions.py @@ -0,0 +1,17 @@ +""" +Permission classes for the import_from_modulestore app. +""" +from django.shortcuts import get_object_or_404 +from rest_framework import permissions + +from cms.djangoapps.import_from_modulestore.models import Import + + +class IsImportAuthor(permissions.BasePermission): + """ + Permission class to check if the user is the author of the import. + """ + + def has_permission(self, request, view): + import_event = get_object_or_404(Import, uuid=request.data.get('import_uuid')) + return import_event.user_id == request.user.pk diff --git a/cms/djangoapps/import_from_modulestore/signals.py b/cms/djangoapps/import_from_modulestore/signals.py new file mode 100644 index 000000000000..b0c6b48177ec --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/signals.py @@ -0,0 +1,25 @@ +""" +Signals for Import. +""" +from django.dispatch import receiver +from django.db.models.signals import post_save + +from .models import Import + + +@receiver(post_save, sender=Import) +def cancel_incomplete_imports(sender, instance, created, **kwargs): + """ + Cancel any incomplete imports that have the same target as the current import. + + When a new import is created, we want to cancel any other incomplete user imports that have the same target. + """ + if created: + incomplete_user_imports_with_same_target = Import.objects.filter( + user=instance.user, + target=instance.target, + source_key=instance.source_key, + staged_content_for_import__isnull=False + ).exclude(uuid=instance.uuid) + for incomplete_import in incomplete_user_imports_with_same_target: + incomplete_import.cancel() diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py new file mode 100644 index 000000000000..84422ad6ec7b --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tasks.py @@ -0,0 +1,84 @@ +""" +Tasks for course to library import. +""" + +from celery import shared_task +from celery.utils.log import get_task_logger +from django.db import transaction +from edx_django_utils.monitoring import set_code_owner_attribute + +from openedx.core.djangoapps.content_staging import api as content_staging_api + +from .constants import IMPORT_FROM_MODULESTORE_PURPOSE +from .helpers import get_items_to_import, ImportClient +from .models import Import, StagedContentForImport +from .validators import validate_composition_level + +log = get_task_logger(__name__) + + +@shared_task +@set_code_owner_attribute +def save_legacy_content_to_staged_content_task(import_uuid: str) -> None: + """ + Save courses to staged content task. + """ + import_event = Import.get_by_uuid(import_uuid) + if not import_event: + return + + import_event.clean_related_staged_content() + try: + with transaction.atomic(): + items_to_import = get_items_to_import(import_event) + for item in items_to_import: + staged_content = content_staging_api.stage_xblock_temporarily( + item, + import_event.user.id, + purpose=IMPORT_FROM_MODULESTORE_PURPOSE, + ) + StagedContentForImport.objects.create( + staged_content=staged_content, + import_event=import_event + ) + + if items_to_import: + import_event.ready() + else: + import_event.error() + except Exception as exc: # pylint: disable=broad-except + import_event.error() + raise exc + + +@shared_task +@set_code_owner_attribute +def import_course_staged_content_to_library_task( + usage_ids: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library task. + """ + validate_composition_level(composition_level) + import_event = Import.get_ready_by_uuid(import_uuid) + if not import_event or import_event.user_id != user_id: + log.info('Ready import from modulestore not found') + return + + with transaction.atomic(): + for usage_id in usage_ids: + if staged_content_item := import_event.get_staged_content_by_block_usage_id(usage_id): + import_client = ImportClient( + import_event, + usage_id, + staged_content_item, + composition_level, + override, + ) + import_client.import_from_staged_content() + + import_event.imported() diff --git a/cms/djangoapps/import_from_modulestore/templates/admin/custom_course_import_form.html b/cms/djangoapps/import_from_modulestore/templates/admin/custom_course_import_form.html new file mode 100644 index 000000000000..faf9ef3a7352 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/templates/admin/custom_course_import_form.html @@ -0,0 +1,54 @@ +{% load i18n %} +{% extends "admin/base_site.html" %} +{% block content %} +
+ {% csrf_token %} + + + {% for obj in queryset %} + + {% endfor %} + +
+ {% if form.errors %} +

+ {% blocktranslate count counter=form.errors|length|escape %} + Please correct the error below. + {% plural %} + Please correct the errors below. + {% endblocktranslate %} +

+ {% endif %} + +
+ {% for field in form %} +
+
+ {{ field.label_tag }} + {% if field.widget_type == 'textarea' %} +
+ {% endif %} + {{ field }} + + {% if field.help_text %} +

{{ field.help_text }}

+ {% endif %} + + {% if field.errors %} +
    + {% for error in field.errors %} +
  • {{ error }}
    • + {% endfor %} +
+ {% endif %} +
+ {% endfor %} +
+ +
+ + {% translate 'Cancel'|escape %} +
+
+
+{% endblock %} diff --git a/cms/djangoapps/import_from_modulestore/tests/__init__.py b/cms/djangoapps/import_from_modulestore/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/tests/factories.py b/cms/djangoapps/import_from_modulestore/tests/factories.py new file mode 100644 index 000000000000..2207a799ef33 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/factories.py @@ -0,0 +1,30 @@ +""" +Factories for Import model. +""" + +import uuid + +import factory +from factory.django import DjangoModelFactory +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories + + +class ImportFactory(DjangoModelFactory): + """ + Factory for Import model. + """ + + class Meta: + model = Import + + @factory.lazy_attribute + def source_key(self): + return CourseKey.from_string(f'course-v1:edX+DemoX+{self.uuid}') + + target = factory.SubFactory(factories.LearningPackageFactory) + uuid = factory.LazyFunction(lambda: str(uuid.uuid4())) + user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py new file mode 100644 index 000000000000..74d9c4b8e5da --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_api.py @@ -0,0 +1,77 @@ +""" +Test cases for import_from_modulestore.api module. +""" + +from unittest.mock import patch + +import pytest +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.api import create_import, import_course_staged_content_to_library +from cms.djangoapps.import_from_modulestore.data import ImportStatus +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from .factories import ImportFactory + + +@pytest.mark.django_db +class TestCourseToLibraryImportAPI(ModuleStoreTestCase): + """ + Test cases for Import API. + """ + + def setUp(self): + super().setUp() + + self.library = factories.ContentLibraryFactory() + + def test_create_import(self): + """ + Test create_import function. + """ + course_id = "course-v1:edX+DemoX+Demo_Course" + user = UserFactory() + create_import(course_id, user.id, self.library.learning_package_id) + + import_event = Import.objects.get() + assert import_event.source_key == CourseKey.from_string(course_id) + assert import_event.target == self.library.learning_package + assert import_event.user_id == user.id + assert import_event.status == ImportStatus.PENDING + + def test_import_course_staged_content_to_library(self): + """ + Test import_course_staged_content_to_library function with different override values. + """ + import_event = ImportFactory( + target=self.library.learning_package, + source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"), + ) + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@html+block@123", + "block-v1:edX+DemoX+Demo_Course+type@html+block@456", + ] + override = False + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_course_staged_content_to_library_task" + ) as import_course_staged_content_to_library_task_mock: + import_course_staged_content_to_library( + usage_ids, + import_event.uuid, + import_event.user.id, + 'xblock', + override + ) + + import_course_staged_content_to_library_task_mock.apply_async.assert_called_once_with( + kwargs={ + 'usage_ids': usage_ids, + 'import_uuid': import_event.uuid, + 'user_id': import_event.user.id, + 'composition_level': 'xblock', + 'override': override, + }, + ) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py new file mode 100644 index 000000000000..573c61945a62 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py @@ -0,0 +1,371 @@ +""" +Tests for the import_from_modulestore helper functions. +""" +import ddt +from organizations.models import Organization +from unittest import mock +from unittest.mock import patch + +from lxml import etree +from openedx_learning.api.authoring_models import LearningPackage + +from cms.djangoapps.import_from_modulestore import api +from cms.djangoapps.import_from_modulestore.helpers import ImportClient +from common.djangoapps.student.tests.factories import UserFactory + +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory + + +@ddt.ddt +class TestImportClient(ModuleStoreTestCase): + """ + Functional tests for the ImportClient class. + """ + + def setUp(self): + super().setUp() + self.library = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.user = UserFactory() + self.course = CourseFactory.create() + self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter') + self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential') + self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical') + self.problem = BlockFactory.create( + category='problem', + parent=self.vertical, + display_name='Problem', + data="""""", + ) + self.video = BlockFactory.create( + category='video', + parent=self.vertical, + display_name='Video', + data="""