diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py index ed1d7a202303..86caf6d180f2 100644 --- a/cms/djangoapps/import_from_modulestore/admin.py +++ b/cms/djangoapps/import_from_modulestore/admin.py @@ -12,23 +12,19 @@ class ImportAdmin(admin.ModelAdmin): """ list_display = ( - 'uuid', 'created', - 'status', 'source_key', + 'target_key', + 'override', + 'composition_level', 'target_change', ) - list_filter = ( - 'status', - ) search_fields = ( 'source_key', + 'target_key', 'target_change', ) - raw_id_fields = ('user',) - readonly_fields = ('status',) - admin.site.register(Import, ImportAdmin) admin.site.register(PublishableEntityImport) diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py index 7f8dc16b76cb..6d3d503602d6 100644 --- a/cms/djangoapps/import_from_modulestore/api.py +++ b/cms/djangoapps/import_from_modulestore/api.py @@ -1,9 +1,7 @@ """ API for course to library import. """ -from typing import Sequence - -from opaque_keys.edx.keys import LearningContextKey, UsageKey +from opaque_keys.edx.keys import LearningContextKey from .helpers import cancel_incomplete_old_imports from .models import Import as _Import @@ -22,7 +20,7 @@ def stage_content_for_import(source_key: LearningContextKey, user_id: int) -> _I def import_staged_content_to_library( - usage_ids: Sequence[str | UsageKey], + usage_ids: list[str], import_uuid: str, target_learning_package_id: int, user_id: int, @@ -35,7 +33,7 @@ def import_staged_content_to_library( validate_usage_keys_to_import(usage_ids) import_staged_content_to_library_task.apply_async( kwargs={ - 'usage_key_strings': usage_ids, + 'usage_keys_string': usage_ids, 'import_uuid': import_uuid, 'learning_package_id': target_learning_package_id, 'user_id': user_id, diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py index 998ea8dfc745..c2f547b68a7f 100644 --- a/cms/djangoapps/import_from_modulestore/data.py +++ b/cms/djangoapps/import_from_modulestore/data.py @@ -3,7 +3,6 @@ """ from collections import namedtuple from enum import Enum -from openedx.core.djangoapps.content_libraries import api as content_libraries_api from django.db.models import TextChoices from django.utils.translation import gettext_lazy as _ @@ -24,31 +23,4 @@ class ImportStatus(TextChoices): CANCELED = 'canceled', _('Canceled') -class CompositionLevel(Enum): - """ - Enumeration of composition levels for course content. - Defines the different levels of composition for course content, - including chapters, sequentials, verticals, and xblocks. - It also categorizes these levels into complicated and flat - levels for easier processing. - """ - - CHAPTER = content_libraries_api.ContainerType.Section - SEQUENTIAL = content_libraries_api.ContainerType.Subsection - VERTICAL = content_libraries_api.ContainerType.Unit - COMPONENT = 'component' - OLX_COMPLEX_LEVELS = [ - VERTICAL.olx_tag, - SEQUENTIAL.olx_tag, - CHAPTER.olx_tag, - ] - - @classmethod - def values(cls): - """ - Returns all levels of composition levels. - """ - return [composition_level.value for composition_level in cls] - - PublishableVersionWithMapping = namedtuple('PublishableVersionWithMapping', ['publishable_version', 'mapping']) diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py index e540e0ff3dba..cd3582d03ca7 100644 --- a/cms/djangoapps/import_from_modulestore/helpers.py +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -1,13 +1,14 @@ """ Helper functions for importing course content into a library. """ +from __future__ import annotations + from datetime import datetime, timezone -from functools import partial import logging import mimetypes import os import secrets -from typing import TYPE_CHECKING +import typing as t from django.db import transaction from django.db.utils import IntegrityError @@ -19,18 +20,13 @@ from openedx_learning.api.authoring_models import Component, Container, ContainerVersion, PublishableEntity from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_libraries.api import ContainerType from openedx.core.djangoapps.content_staging import api as content_staging_api from xmodule.modulestore.django import modulestore -from .data import CompositionLevel, ImportStatus, PublishableVersionWithMapping +from .data import PublishableVersionWithMapping from .models import Import, PublishableEntityMapping -if TYPE_CHECKING: - from openedx_learning.apps.authoring_models import LearningPackage - from xblock.core import XBlock - - from openedx.core.djangoapps.content_staging.api import _StagedContent as StagedContent - log = logging.getLogger(__name__) parser = etree.XMLParser(strip_cdata=False) @@ -45,18 +41,20 @@ class ImportClient: specified composition level. """ - # The create functions have different kwarg names for the child list, - # so we need to use partial to set the child list to empty. - CONTAINER_CREATORS_MAP: dict[str, partial] = { - api.ContainerType.Section.olx_tag: partial(authoring_api.create_section_and_version, subsections=[]), - api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_subsection_and_version, units=[]), - api.ContainerType.Unit.olx_tag: partial(authoring_api.create_unit_and_version, components=[]), + CONTAINER_CREATORS_MAP: dict[ContainerType, t.Callable] = { + # Sequential- and Chapter-level composition to be added in the future as part of + # https://github.com/openedx/frontend-app-authoring/issues/1602 + # 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version + # 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version + ContainerType.Unit: authoring_api.create_unit_and_version, } - CONTAINER_OVERRIDERS_MAP: dict[str, partial] = { - api.ContainerType.Section.olx_tag: partial(authoring_api.create_next_section_version, subsections=[]), - api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_next_subsection_version, units=[]), - api.ContainerType.Unit.olx_tag: partial(authoring_api.create_next_unit_version, components=[]), + CONTAINER_OVERRIDERS_MAP: dict[ContainerType, t.Callable] = { + # Sequential- and Chapter-level composition to be added in the future as part of + # https://github.com/openedx/frontend-app-authoring/issues/1602 + # 'chapter': authoring_api.create_next_unit_version, # TODO: replace with create_next_module_version + # 'sequential': authoring_api.create_next_unit_version, # TODO: replace with create_next_section_version + ContainerType.Unit: authoring_api.create_next_unit_version, } def __init__( @@ -65,7 +63,7 @@ def __init__( block_usage_key_to_import: str, target_learning_package: 'LearningPackage', staged_content: 'StagedContent', - composition_level: str, + composition_level: ContainerType | None, override: bool = False, ): self.import_event = import_event @@ -86,7 +84,7 @@ def import_from_staged_content(self) -> list[PublishableVersionWithMapping]: """ node = etree.fromstring(self.staged_content.olx, parser=parser) usage_key = UsageKey.from_string(self.block_usage_key_to_import) - block_to_import = get_node_for_usage_key(node, usage_key) + block_to_import = get_block_to_import(node, usage_key) if block_to_import is None: return [] @@ -103,7 +101,7 @@ def _process_import(self, usage_key_string, block_to_import) -> list[Publishable usage_key = UsageKey.from_string(usage_key_string) result = [] - if block_to_import.tag not in CompositionLevel.OLX_COMPLEX_LEVELS.value: + if block_to_import.tag not in CompositionLevel.COMPLEX_LEVELS.value: return self._import_simple_block(block_to_import, usage_key) for child in block_to_import.getchildren(): @@ -115,7 +113,7 @@ def _process_import(self, usage_key_string, block_to_import) -> list[Publishable result.extend(self._import_child_block(child, child_usage_key_string)) - if self.composition_level == CompositionLevel.COMPONENT.value: + if not self.composition_level: return [ publishable_version_with_mapping for publishable_version_with_mapping in result if not isinstance(publishable_version_with_mapping.publishable_version, ContainerVersion) @@ -140,7 +138,7 @@ def _import_child_block(self, child, child_usage_key_string): delegates the import process to the appropriate helper method. """ child_usage_key = UsageKey.from_string(child_usage_key_string) - if child.tag in CompositionLevel.OLX_COMPLEX_LEVELS.value: + if child.tag in CompositionLevel.COMPLEX_LEVELS.value: return self._import_complicated_child(child, child_usage_key_string) else: return self._import_simple_block(child, child_usage_key) @@ -170,18 +168,15 @@ def _import_complicated_child(self, child, child_usage_key_string): self._update_container_components(container_version_with_mapping.publishable_version, child_component_versions) return [container_version_with_mapping] + child_component_versions_with_mapping - def _should_create_container(self, container_type: str) -> bool: + def _should_create_container(self, olx_tag: str) -> bool: """ Determine if a new container should be created. Container type should be at a lower level than the current composition level. """ - composition_hierarchy = CompositionLevel.OLX_COMPLEX_LEVELS.value - return ( - container_type in composition_hierarchy and - self.composition_level in composition_hierarchy and - composition_hierarchy.index(container_type) <= composition_hierarchy.index(self.composition_level) - ) + if self.composition_level and (container_type := ContainerType.from_source_olx_tag(olx_tag)): + return self.composition_level >= container_type + return None def get_or_create_container( self, @@ -196,11 +191,10 @@ def get_or_create_container( Creates a container (e.g., chapter, sequential, vertical) in the content library. """ - try: - container_creator_func = self.CONTAINER_CREATORS_MAP[container_type] - container_override_func = self.CONTAINER_OVERRIDERS_MAP[container_type] - except KeyError as exc: - raise ValueError(f"Unknown container type: {container_type}") from exc + container_creator_func = self.CONTAINER_CREATORS_MAP.get(container_type) + container_override_func = self.CONTAINER_OVERRIDERS_MAP.get(container_type) + if not all((container_creator_func, container_override_func)): + raise ValueError(f"Unknown container type: {container_type}") try: container_version = self.content_library.learning_package.publishable_entities.get(key=key) @@ -211,6 +205,7 @@ def get_or_create_container( container_version = container_override_func( container_version.container, title=display_name or f"New {container_type}", + components=[], created=datetime.now(tz=timezone.utc), created_by=self.import_event.user_id, ) @@ -219,6 +214,7 @@ def get_or_create_container( self.learning_package.id, key=key or secrets.token_hex(16), title=display_name or f"New {container_type}", + components=[], created=datetime.now(tz=timezone.utc), created_by=self.import_event.user_id, ) @@ -266,7 +262,7 @@ def _create_block_in_library(self, block_to_import, usage_key) -> PublishableVer if does_component_exist: if not self.override: log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.") - return None + return else: component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import)) else: @@ -278,7 +274,7 @@ def _create_block_in_library(self, block_to_import, usage_key) -> PublishableVer ) except api.IncompatibleTypesError as e: log.error(f"Error validating block {usage_key} for library {self.library_key}: {e}") - return None + return authoring_api.create_component( self.learning_package.id, @@ -373,30 +369,6 @@ def _process_staged_content_files( pass # Content already exists -def import_from_staged_content( - import_event: Import, - usage_key_string: str, - target_learning_package: 'LearningPackage', - staged_content: 'StagedContent', - composition_level: str, - override: bool = False, -) -> list[PublishableVersionWithMapping]: - """ - Import staged content to a library from staged content. - - Returns a list of PublishableVersionWithMappings created during the import. - """ - import_client = ImportClient( - import_event, - usage_key_string, - target_learning_package, - staged_content, - composition_level, - override, - ) - return import_client.import_from_staged_content() - - def get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]: """ Creates a mapping between the source usage key and the target publishable entity. @@ -412,33 +384,32 @@ def get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[Publ ) -def get_usage_key_string_from_staged_content(staged_content: 'StagedContent', block_id: str) -> str | None: +def get_usage_key_string_from_staged_content(staged_content, block_id): """ Get the usage ID from a staged content by block ID. """ - if staged_content.tags is None: - return None return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None) -def get_node_for_usage_key(node: etree._Element, usage_key: UsageKey) -> etree._Element: +def get_block_to_import(node, usage_key): """ - Get the node in an XML tree which matches to the usage key. + Get the block to import from a node. """ - if node.tag == usage_key.block_type and node.get('url_name') == usage_key.block_id: + + if node.get('url_name') == usage_key.block_id: return node for child in node.getchildren(): - found = get_node_for_usage_key(child, usage_key) + found = get_block_to_import(child, usage_key) if found is not None: return found -def get_items_to_import(import_event: Import) -> list['XBlock']: +def get_items_to_import(import_event): """ Collect items to import from a course. """ - items_to_import: list['XBlock'] = [] + items_to_import = [] if isinstance(import_event.source_key, CourseLocator): items_to_import.extend( modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or [] @@ -450,17 +421,17 @@ def get_items_to_import(import_event: Import) -> list['XBlock']: return items_to_import -def cancel_incomplete_old_imports(import_event: Import) -> None: +def cancel_incomplete_old_imports(instance): """ Cancel any incomplete imports that have the same target as the current import. When a new import is created, we want to cancel any other incomplete user imports that have the same target. """ incomplete_user_imports_with_same_target = Import.objects.filter( - user=import_event.user, - target_change=import_event.target_change, - source_key=import_event.source_key, + user=instance.user, + target_change=instance.target_change, + source_key=instance.source_key, staged_content_for_import__isnull=False - ).exclude(uuid=import_event.uuid) + ).exclude(uuid=instance.uuid) for incomplete_import in incomplete_user_imports_with_same_target: incomplete_import.set_status(ImportStatus.CANCELED) diff --git a/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_status_remove_import_user_and_more.py b/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_status_remove_import_user_and_more.py new file mode 100644 index 000000000000..8a850b553cf8 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_status_remove_import_user_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.20 on 2025-04-23 09:52 + +from django.db import migrations, models +import django.db.models.deletion +import opaque_keys.edx.django.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_tasks', '0004_url_textfield'), + ('import_from_modulestore', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='import', + name='status', + ), + migrations.RemoveField( + model_name='import', + name='user', + ), + migrations.RemoveField( + model_name='import', + name='uuid', + ), + migrations.AddField( + model_name='import', + name='composition_level', + field=models.CharField(choices=[('component', 'Component'), ('unit', 'Unit'), ('subsection', 'Subsection'), ('section', 'Section')], default='component', max_length=64), + ), + migrations.AddField( + model_name='import', + name='override', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='import', + name='target_key', + field=opaque_keys.edx.django.models.LearningContextKeyField(blank=True, db_index=True, help_text='Target library key', max_length=255, null=True), + ), + migrations.AddField( + model_name='import', + name='user_task_status', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_event', to='user_tasks.usertaskstatus'), + ), + migrations.AlterField( + model_name='import', + name='source_key', + field=opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, help_text='Source course key', max_length=255), + ), + ] diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py index 5b6122749bba..8798822e11f9 100644 --- a/cms/djangoapps/import_from_modulestore/models.py +++ b/cms/djangoapps/import_from_modulestore/models.py @@ -1,22 +1,29 @@ """ Models for the course to library import app. """ +from typing import Optional import uuid as uuid_tools from django.contrib.auth import get_user_model from django.db import models from django.utils.translation import gettext_lazy as _ +from user_tasks.models import UserTaskStatus from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import ( LearningContextKeyField, UsageKeyField, ) -from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity +from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity, DraftChangeLog -from .data import ImportStatus +from openedx.core.djangoapps.content_libraries.api.containers import ContainerType -User = get_user_model() +COMPOSITION_LEVEL_COMPONENT = "component" + +COMPOSITION_LEVEL_CHOICES = [ + (COMPOSITION_LEVEL_COMPONENT, "Component"), + *[(ct.value, ct.name) for ct in ContainerType], +] class Import(TimeStampedModel): @@ -24,19 +31,24 @@ class Import(TimeStampedModel): Represents the action of a user importing a modulestore-based course or legacy library into a learning-core based learning package (today, that is always a content library). """ - - uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True) - status = models.CharField( - max_length=100, - choices=ImportStatus.choices, - default=ImportStatus.NOT_STARTED, - db_index=True + user_task_status = models.OneToOneField( + to=UserTaskStatus, related_name="import_event", + null=True, on_delete=models.SET_NULL, ) - user = models.ForeignKey(User, on_delete=models.CASCADE) # Note: For now, this will always be a course key. In the future, it may be a legacy library key. - source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True) - target_change = models.ForeignKey(to='oel_publishing.DraftChangeLog', on_delete=models.SET_NULL, null=True) + source_key = LearningContextKeyField(help_text=_('Source course key'), max_length=255, db_index=True) + target_key = LearningContextKeyField( + help_text=_('Target library key'), + max_length=255, db_index=True, null=True, blank=True + ) + composition_level = models.CharField( + choices=COMPOSITION_LEVEL_CHOICES, + max_length=64, + default=COMPOSITION_LEVEL_COMPONENT, + ) + override = models.BooleanField(default=False) + target_change = models.ForeignKey(to=DraftChangeLog, on_delete=models.SET_NULL, null=True) class Meta: verbose_name = _('Import from modulestore') @@ -45,15 +57,6 @@ class Meta: def __str__(self): return f'{self.source_key} → {self.target_change}' - def set_status(self, status: ImportStatus): - """ - Set import status. - """ - self.status = status - self.save() - if status in [ImportStatus.IMPORTED, ImportStatus.CANCELED]: - self.clean_related_staged_content() - def clean_related_staged_content(self) -> None: """ Clean related staged content. diff --git a/cms/djangoapps/import_from_modulestore/rest_api/__init__.py b/cms/djangoapps/import_from_modulestore/rest_api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/rest_api/urls.py b/cms/djangoapps/import_from_modulestore/rest_api/urls.py new file mode 100644 index 000000000000..597fc23565c7 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/rest_api/urls.py @@ -0,0 +1,13 @@ +""" +URLs for import_from_modulstore REST API +""" + +from django.urls import include, path + +from .v0 import urls as v0_urls + +app_name = 'import_from_modulestore' + +urlpatterns = [ + path('v0/', include(v0_urls)), +] diff --git a/cms/djangoapps/import_from_modulestore/rest_api/v0/__init__.py b/cms/djangoapps/import_from_modulestore/rest_api/v0/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/rest_api/v0/serializers.py b/cms/djangoapps/import_from_modulestore/rest_api/v0/serializers.py new file mode 100644 index 000000000000..23b89676b0a4 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/rest_api/v0/serializers.py @@ -0,0 +1,18 @@ +from rest_framework.serializers import ModelSerializer +from user_tasks.serializers import StatusSerializer + +from ...models import Import + + +class ImportSerializer(ModelSerializer): + class Meta: + model = Import + + +class StatusWithImportSerializer(StatusSerializer): + import_event = ImportSerializer() + + class Meta: + model = StatusSerializer.Meta.model + fields = [*StatusSerializer.Meta.fields, 'import_event'] + diff --git a/cms/djangoapps/import_from_modulestore/rest_api/v0/urls.py b/cms/djangoapps/import_from_modulestore/rest_api/v0/urls.py new file mode 100644 index 000000000000..fd34ef3a6bd0 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/rest_api/v0/urls.py @@ -0,0 +1,7 @@ +from rest_framework.routers import SimpleRouter +from .views import ImportViewSet + +ROUTER = SimpleRouter() +ROUTER.register(r'imports', ImportViewSet) + +urlpatterns = ROUTER.urls diff --git a/cms/djangoapps/import_from_modulestore/rest_api/v0/views.py b/cms/djangoapps/import_from_modulestore/rest_api/v0/views.py new file mode 100644 index 000000000000..533f5d885896 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/rest_api/v0/views.py @@ -0,0 +1,31 @@ + +from user_tasks.models import UserTaskStatus +from user_tasks.views import StatusViewSet +from rest_framework.response import Response + +from .serializers import StatusWithImportSerializer +from ...models import COMPOSITION_LEVEL_COMPONENT +from ...tasks import import_legacy_content + + +class ImportViewSet(StatusViewSet): + + # TODO: it would be more robust to instead filter the UserTaskStatuses by whether + # they have a related Import object, but I'm not sure how to write that query. + queryset = StatusViewSet.queryset.filter( + task_class="cms.djangoapps.import_from_modulestore.tasks.import_legacy_content" + ) + + serializer_class = StatusWithImportSerializer + + def create(self, request, *args, **kwargs): + task = import_legacy_content.delay( + user_id=request.user.id, + # TODO: Need to validate these parameters. + # I think the serializer can be used to do this. + source_key=request.data['source_key'], + target_key=request.data['target_key'], + override=request.data.get('override', False), + composition_level=request.data.get('composition_level', COMPOSITION_LEVEL_COMPONENT), + ) + return UserTaskStatus.objects.get(uuid=task.id) diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py index 4644b29e4904..3bce6b4cffc9 100644 --- a/cms/djangoapps/import_from_modulestore/tasks.py +++ b/cms/djangoapps/import_from_modulestore/tasks.py @@ -6,93 +6,86 @@ from celery.utils.log import get_task_logger from django.db import transaction from edx_django_utils.monitoring import set_code_owner_attribute +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocatorV2 +from user_tasks.tasks import UserTask from openedx_learning.api import authoring as authoring_api from openedx_learning.api.authoring_models import LearningPackage from openedx.core.djangoapps.content_staging import api as content_staging_api +from openedx.core.djangoapps.content_libraries.api import ContainerType +from xmodule.modulestore.exceptions import ItemNotFoundError from .constants import IMPORT_FROM_MODULESTORE_STAGING_PURPOSE -from .data import ImportStatus -from .helpers import get_items_to_import, import_from_staged_content -from .models import Import, PublishableEntityImport, StagedContentForImport -from .validators import validate_composition_level +from .helpers import get_items_to_import, ImportClient +from .models import ( + Import, PublishableEntityImport, StagedContentForImport, + COMPOSITION_LEVEL_COMPONENT, +) log = get_task_logger(__name__) -@shared_task -@set_code_owner_attribute -def save_legacy_content_to_staged_content_task(import_uuid: str) -> None: - """ - Save courses to staged content task by sections/chapters. - """ - import_event = Import.objects.get(uuid=import_uuid) - - import_event.clean_related_staged_content() - import_event.set_status(ImportStatus.STAGING) - try: - with transaction.atomic(): - items_to_import = get_items_to_import(import_event) - for item in items_to_import: - staged_content = content_staging_api.stage_xblock_temporarily( - item, - import_event.user.id, - purpose=IMPORT_FROM_MODULESTORE_STAGING_PURPOSE, - ) - StagedContentForImport.objects.create( - staged_content=staged_content, - import_event=import_event, - source_usage_key=item.location - ) - - if items_to_import: - import_event.set_status(ImportStatus.STAGED) - else: - import_event.set_status(ImportStatus.STAGING_FAILED) - except Exception as exc: # pylint: disable=broad-except - import_event.set_status(ImportStatus.STAGING_FAILED) - raise exc +class ImportTask(UserTask): + pass -@shared_task -@set_code_owner_attribute -def import_staged_content_to_library_task( - usage_key_strings: list[str], - import_uuid: str, - learning_package_id: int, +@shared_task(base=ImportTask, bind=True) +def import_legacy_content( + self: ImportTask, user_id: int, - composition_level: str, + *, + source_key: str, + target_key: str, override: bool, + composition_level: str, ) -> None: - """ - Import staged content to a library task. - """ - validate_composition_level(composition_level) - - import_event = Import.objects.get(uuid=import_uuid, status=ImportStatus.STAGED, user_id=user_id) - target_learning_package = LearningPackage.objects.get(id=learning_package_id) - - imported_publishable_versions = [] + import_event = Import.objects.create( + user_task_status=self.status, + source_key=CourseKey.from_string(source_key), + target_key=LibraryLocatorV2.from_string(target_key), + override=override, + composition_level=composition_level, + ) + import_event.set_state(f"Preparing to stage content") + import_event.clean_related_staged_content() + target_package = authoring_api.get_learning_package_by_key(target_key) + items_to_import: list['XBlock'] = get_items_to_import(import_event) + if not items_to_import: + import_event.fail("Nothing to stage for import") + return + for item in items_to_import: + import_event.set_state(f"Staging content: {item.usage_key}") + staged_content = content_staging_api.stage_xblock_temporarily( + item, + import_event.user.id, + purpose=IMPORT_FROM_MODULESTORE_STAGING_PURPOSE, + ) + StagedContentForImport.objects.create( + staged_content=staged_content, + import_event=import_event, + source_usage_key=item.location + ) + import_event.set_state(f"Content is staged, preparing to import") with authoring_api.bulk_draft_changes_for(learning_package_id=learning_package_id) as change_log: - try: - for usage_key_string in usage_key_strings: - staged_content_for_import = import_event.staged_content_for_import.get( - source_usage_key=usage_key_string - ) - publishable_versions = import_from_staged_content( - import_event, - usage_key_string, - target_learning_package, - staged_content_for_import.staged_content, - composition_level, - override, - ) - imported_publishable_versions.extend(publishable_versions) - except: # pylint: disable=bare-except - import_event.set_status(ImportStatus.IMPORTING_FAILED) - raise - - import_event.set_status(ImportStatus.IMPORTED) + for staged_content_item in import_event.staged_content_for_import: + source_key = staged_content_item.source_usage_key + import_event.set_state(f"Importing staged content: {source_key}") + import_client = ImportClient( + import_event=import_event, + usage_key_string=source_key, + target_learning_package=target_package, + staged_content=staged_content_item.staged_content, + composition_level=( + None + if composition_level == COMPOSITION_LEVEL_COMPONENT + else ContainerType(composition_level) + ), + override=override, + ) + imported_publishable_versions.extend(import_client.import_from_staged_content()) + import_event.set_state(f"Finalizing import") for imported_component_version in imported_publishable_versions: PublishableEntityImport.objects.create( import_event=import_event, diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py index 62fe2e4159a9..02cefcb19a36 100644 --- a/cms/djangoapps/import_from_modulestore/tests/test_api.py +++ b/cms/djangoapps/import_from_modulestore/tests/test_api.py @@ -73,7 +73,7 @@ def test_import_staged_content_to_library(self): import_staged_content_to_library_task_mock.apply_async.assert_called_once_with( kwargs={ - "usage_key_strings": usage_ids, + "usage_keys_string": usage_ids, "import_uuid": import_event.uuid, "learning_package_id": self.library.learning_package.id, "user_id": import_event.user.id, diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py index 1d2ce26867aa..a847c05f9df5 100644 --- a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py +++ b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py @@ -56,10 +56,7 @@ def setUp(self): def test_import_from_staged_content(self): expected_imported_xblocks = [self.video, self.problem] - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, staged_content=staged_content, @@ -75,10 +72,7 @@ def test_import_from_staged_content(self): @patch('cms.djangoapps.import_from_modulestore.helpers.ImportClient._process_import') def test_import_from_staged_content_block_not_found(self, mocked_process_import): - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, staged_content=staged_content, @@ -94,19 +88,19 @@ def test_import_from_staged_content_block_not_found(self, mocked_process_import) mocked_process_import.assert_not_called() @ddt.data( - 'chapter', - 'sequential', + # TODO: Chapter and sequential will be uncommented in the future + # after adding it to the Learning Core. + # 'chapter', + # 'sequential', 'vertical' ) def test_create_container(self, block_lvl): container_to_import = getattr(self, block_lvl) block_usage_key_to_import = str(container_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) + staged_content = self.import_event.get_staged_content_by_source_usage_key(block_usage_key_to_import) import_client = ImportClient( import_event=self.import_event, - staged_content=staged_content_for_import.staged_content, + staged_content=staged_content, target_learning_package=self.learning_package, block_usage_key_to_import=block_usage_key_to_import, composition_level='xblock', @@ -123,12 +117,10 @@ def test_create_container(self, block_lvl): def test_create_container_with_xblock(self): block_usage_key_to_import = str(self.problem.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) + staged_content = self.import_event.get_staged_content_by_source_usage_key(block_usage_key_to_import) import_client = ImportClient( import_event=self.import_event, - staged_content=staged_content_for_import.staged_content, + staged_content=staged_content, target_learning_package=self.learning_package, block_usage_key_to_import=block_usage_key_to_import, composition_level='xblock', @@ -146,10 +138,7 @@ def test_create_container_with_xblock(self): def test_process_import_with_complicated_blocks(self, block_lvl): container_to_import = getattr(self, block_lvl) block_usage_key_to_import = str(container_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) expected_imported_xblocks = [self.problem, self.video] import_client = ImportClient( @@ -171,13 +160,11 @@ def test_process_import_with_complicated_blocks(self, block_lvl): def test_process_import_with_simple_blocks(self, block_type_to_import): block_to_import = getattr(self, block_type_to_import) block_usage_key_to_import = str(block_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) expected_imported_xblocks = [block_to_import] import_client = ImportClient( import_event=self.import_event, - staged_content=staged_content_for_import.staged_content, + staged_content=staged_content, target_learning_package=self.learning_package, block_usage_key_to_import=block_usage_key_to_import, composition_level='xblock', @@ -195,13 +182,11 @@ def test_process_import_with_simple_blocks(self, block_type_to_import): def test_process_import_with_override(self, override): block_to_import = self.problem block_usage_key_to_import = str(block_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, - staged_content=staged_content_for_import.staged_content, + staged_content=staged_content, target_learning_package=self.learning_package, block_usage_key_to_import=block_usage_key_to_import, composition_level='xblock', @@ -215,11 +200,7 @@ def test_process_import_with_override(self, override): with self.captureOnCommitCallbacks(execute=True): new_import_event = api.stage_content_for_import(source_key=self.course.id, user_id=self.user.id) - - staged_content_for_import = new_import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - new_staged_content = staged_content_for_import.staged_content + new_staged_content = new_import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=new_import_event, staged_content=new_staged_content, @@ -253,10 +234,7 @@ def test_process_import_with_override(self, override): def test_container_override(self, mock_authoring_api): container_to_import = self.vertical block_usage_key_to_import = str(container_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, @@ -305,10 +283,7 @@ def test_composition_levels(self, composition_level): container_to_import = self.vertical block_usage_key_to_import = str(container_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, @@ -329,10 +304,7 @@ def test_composition_levels(self, composition_level): def test_process_staged_content_files(self, mock_content_staging_api): block_to_import = self.problem block_usage_key_to_import = str(block_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) - staged_content = staged_content_for_import.staged_content + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, @@ -367,13 +339,11 @@ def test_process_staged_content_files(self, mock_content_staging_api): def test_update_container_components(self): container_to_import = self.vertical block_usage_key_to_import = str(container_to_import.location) - staged_content_for_import = self.import_event.staged_content_for_import.get( - source_usage_key=self.chapter.location - ) + staged_content = self.import_event.get_staged_content_by_source_usage_key(self.chapter.location) import_client = ImportClient( import_event=self.import_event, - staged_content=staged_content_for_import.staged_content, + staged_content=staged_content, target_learning_package=self.learning_package, block_usage_key_to_import=block_usage_key_to_import, composition_level='container', diff --git a/cms/djangoapps/import_from_modulestore/tests/test_tasks.py b/cms/djangoapps/import_from_modulestore/tests/test_tasks.py index 8c24fa95c1cc..cf6ae7471f41 100644 --- a/cms/djangoapps/import_from_modulestore/tests/test_tasks.py +++ b/cms/djangoapps/import_from_modulestore/tests/test_tasks.py @@ -1,7 +1,6 @@ """ Tests for tasks in import_from_modulestore app. """ -from django.core.exceptions import ObjectDoesNotExist from organizations.models import Organization from openedx_learning.api.authoring_models import LearningPackage from unittest.mock import patch @@ -104,7 +103,7 @@ def test_import_staged_content_to_library_task(self): self.import_event.uuid, self.content_library.learning_package.id, self.user.id, - 'component', + 'xblock', override=True ) @@ -118,22 +117,21 @@ def test_import_staged_content_to_library_task(self): self.assertEqual(library_learning_package.content_set.count(), len(expected_imported_xblocks)) self.assertEqual(self.import_event.publishableentityimport_set.count(), len(expected_imported_xblocks)) - @patch('cms.djangoapps.import_from_modulestore.tasks.import_from_staged_content') - def test_import_library_block_not_found(self, mock_import_from_staged_content): + @patch('cms.djangoapps.import_from_modulestore.tasks.ImportClient') + def test_import_library_block_not_found(self, mock_import_client): """ Test that if a block is not found in the staged content, it is not imported. """ non_existent_usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] save_legacy_content_to_staged_content_task(self.import_event.uuid) - with self.allow_transaction_exception(): - with self.assertRaises(ObjectDoesNotExist): - import_staged_content_to_library_task( - non_existent_usage_ids, - str(self.import_event.uuid), - self.content_library.learning_package.id, - self.user.id, - 'component', - override=True, - ) - mock_import_from_staged_content.assert_not_called() + + import_staged_content_to_library_task( + non_existent_usage_ids, + str(self.import_event.uuid), + self.content_library.learning_package.id, + self.user.id, + 'xblock', + override=True, + ) + mock_import_client.assert_not_called() def test_cannot_import_staged_content_twice(self): """ @@ -153,7 +151,7 @@ def test_cannot_import_staged_content_twice(self): str(self.import_event.uuid), self.content_library.learning_package.id, self.user.id, - 'component', + 'xblock', override=True, ) diff --git a/cms/djangoapps/import_from_modulestore/validators.py b/cms/djangoapps/import_from_modulestore/validators.py index ecf0c72e08e8..03a8eff7674a 100644 --- a/cms/djangoapps/import_from_modulestore/validators.py +++ b/cms/djangoapps/import_from_modulestore/validators.py @@ -1,14 +1,10 @@ """ Validators for the import_from_modulestore app. """ -from typing import Sequence - from opaque_keys.edx.keys import UsageKey -from .data import CompositionLevel - -def validate_usage_keys_to_import(usage_keys: Sequence[str | UsageKey]): +def validate_usage_keys_to_import(usage_keys: list[str | UsageKey]): """ Validate the usage keys to import. @@ -19,8 +15,3 @@ def validate_usage_keys_to_import(usage_keys: Sequence[str | UsageKey]): usage_key = UsageKey.from_string(usage_key) if usage_key.block_type != 'chapter': raise ValueError(f'Importing from modulestore only supports chapters, not {usage_key.block_type}') - - -def validate_composition_level(composition_level): - if composition_level not in CompositionLevel.values(): - raise ValueError(f'Invalid composition level: {composition_level}') diff --git a/cms/urls.py b/cms/urls.py index d01e89d9d276..b357bb98cf6e 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -141,6 +141,9 @@ # rest api for course import/export path('api/courses/', include('cms.djangoapps.contentstore.api.urls', namespace='courses_api') ), + path('api/import_from_modulestore/', + include('cms.djangoapps.import_from_modulestore.rest_api.urls', namespace='import_from_modulestore_api'), + ), re_path(fr'^export/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_handler, name='export_handler'), re_path(fr'^export_output/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_output_handler,