diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json
index 7184bf917ecb..827366365fa8 100644
--- a/.github/workflows/unit-test-shards.json
+++ b/.github/workflows/unit-test-shards.json
@@ -238,7 +238,7 @@
"cms/djangoapps/cms_user_tasks/",
"cms/djangoapps/course_creators/",
"cms/djangoapps/export_course_metadata/",
- "cms/djangoapps/import_from_modulestore/",
+ "cms/djangoapps/modulestore_migrator/",
"cms/djangoapps/maintenance/",
"cms/djangoapps/models/",
"cms/djangoapps/pipeline_js/",
diff --git a/cms/djangoapps/import_from_modulestore/README.rst b/cms/djangoapps/import_from_modulestore/README.rst
deleted file mode 100644
index f2725ef4226d..000000000000
--- a/cms/djangoapps/import_from_modulestore/README.rst
+++ /dev/null
@@ -1,31 +0,0 @@
-========================
-Import from Modulestore
-========================
-
-The new Django application `import_from_modulestore` is designed to
-automate the process of importing course legacy OLX content from Modulestore
-to Content Libraries. The application allows users to easily and quickly
-migrate existing course content, minimizing the manual work and potential
-errors associated with manual migration.
-The new app makes the import process automated and easy to manage.
-
-The main problems solved by the application:
-
-* Reducing the time to import course content.
-* Ensuring data integrity during the transfer.
-* Ability to choose which content to import before the final import.
-
-------------------------------
-Import from Modulestore Usage
-------------------------------
-
-* Import course elements at the level of sections, subsections, units,
- and xblocks into the Content Libraries.
-* Choose the structure of this import, whether it will be only xblocks
- from a particular course or full sections/subsections/units.
-* Store the history of imports.
-* Synchronize the course content with the library content (when re-importing,
- the blocks can be updated according to changes in the original course).
-* The new import mechanism ensures data integrity at the time of importing
- by saving the course in StagedContent.
-* Importing the legacy library content into the new Content Libraries.
diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py
deleted file mode 100644
index ed1d7a202303..000000000000
--- a/cms/djangoapps/import_from_modulestore/admin.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""
-This module contains the admin configuration for the Import model.
-"""
-from django.contrib import admin
-
-from .models import Import, PublishableEntityImport, PublishableEntityMapping
-
-
-class ImportAdmin(admin.ModelAdmin):
- """
- Admin configuration for the Import model.
- """
-
- list_display = (
- 'uuid',
- 'created',
- 'status',
- 'source_key',
- 'target_change',
- )
- list_filter = (
- 'status',
- )
- search_fields = (
- 'source_key',
- 'target_change',
- )
-
- raw_id_fields = ('user',)
- readonly_fields = ('status',)
-
-
-admin.site.register(Import, ImportAdmin)
-admin.site.register(PublishableEntityImport)
-admin.site.register(PublishableEntityMapping)
diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py
deleted file mode 100644
index 7f8dc16b76cb..000000000000
--- a/cms/djangoapps/import_from_modulestore/api.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""
-API for course to library import.
-"""
-from typing import Sequence
-
-from opaque_keys.edx.keys import LearningContextKey, UsageKey
-
-from .helpers import cancel_incomplete_old_imports
-from .models import Import as _Import
-from .tasks import import_staged_content_to_library_task, save_legacy_content_to_staged_content_task
-from .validators import validate_usage_keys_to_import
-
-
-def stage_content_for_import(source_key: LearningContextKey, user_id: int) -> _Import:
- """
- Create a new import event to import a course to a library and save course to staged content.
- """
- import_from_modulestore = _Import.objects.create(source_key=source_key, user_id=user_id)
- cancel_incomplete_old_imports(import_from_modulestore)
- save_legacy_content_to_staged_content_task.delay_on_commit(import_from_modulestore.uuid)
- return import_from_modulestore
-
-
-def import_staged_content_to_library(
- usage_ids: Sequence[str | UsageKey],
- import_uuid: str,
- target_learning_package_id: int,
- user_id: int,
- composition_level: str,
- override: bool,
-) -> None:
- """
- Import staged content to a library from staged content.
- """
- validate_usage_keys_to_import(usage_ids)
- import_staged_content_to_library_task.apply_async(
- kwargs={
- 'usage_key_strings': usage_ids,
- 'import_uuid': import_uuid,
- 'learning_package_id': target_learning_package_id,
- 'user_id': user_id,
- 'composition_level': composition_level,
- 'override': override,
- },
- )
diff --git a/cms/djangoapps/import_from_modulestore/apps.py b/cms/djangoapps/import_from_modulestore/apps.py
deleted file mode 100644
index 81b4471daceb..000000000000
--- a/cms/djangoapps/import_from_modulestore/apps.py
+++ /dev/null
@@ -1,13 +0,0 @@
-"""
-App for importing from the modulestore tools.
-"""
-
-from django.apps import AppConfig
-
-
-class ImportFromModulestoreConfig(AppConfig):
- """
- App for importing legacy content from the modulestore.
- """
-
- name = 'cms.djangoapps.import_from_modulestore'
diff --git a/cms/djangoapps/import_from_modulestore/constants.py b/cms/djangoapps/import_from_modulestore/constants.py
deleted file mode 100644
index 09e0d4e30f1a..000000000000
--- a/cms/djangoapps/import_from_modulestore/constants.py
+++ /dev/null
@@ -1,5 +0,0 @@
-"""
-Constants for import_from_modulestore app
-"""
-
-IMPORT_FROM_MODULESTORE_STAGING_PURPOSE = "import_from_modulestore"
diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py
deleted file mode 100644
index 998ea8dfc745..000000000000
--- a/cms/djangoapps/import_from_modulestore/data.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""
-This module contains the data models for the import_from_modulestore app.
-"""
-from collections import namedtuple
-from enum import Enum
-from openedx.core.djangoapps.content_libraries import api as content_libraries_api
-
-from django.db.models import TextChoices
-from django.utils.translation import gettext_lazy as _
-
-
-class ImportStatus(TextChoices):
- """
- The status of this modulestore-to-learning-core import.
- """
-
- NOT_STARTED = 'not_started', _('Waiting to stage content')
- STAGING = 'staging', _('Staging content for import')
- STAGING_FAILED = _('Failed to stage content')
- STAGED = 'staged', _('Content is staged and ready for import')
- IMPORTING = 'importing', _('Importing staged content')
- IMPORTING_FAILED = 'importing_failed', _('Failed to import staged content')
- IMPORTED = 'imported', _('Successfully imported content')
- CANCELED = 'canceled', _('Canceled')
-
-
-class CompositionLevel(Enum):
- """
- Enumeration of composition levels for course content.
- Defines the different levels of composition for course content,
- including chapters, sequentials, verticals, and xblocks.
- It also categorizes these levels into complicated and flat
- levels for easier processing.
- """
-
- CHAPTER = content_libraries_api.ContainerType.Section
- SEQUENTIAL = content_libraries_api.ContainerType.Subsection
- VERTICAL = content_libraries_api.ContainerType.Unit
- COMPONENT = 'component'
- OLX_COMPLEX_LEVELS = [
- VERTICAL.olx_tag,
- SEQUENTIAL.olx_tag,
- CHAPTER.olx_tag,
- ]
-
- @classmethod
- def values(cls):
- """
- Returns all levels of composition levels.
- """
- return [composition_level.value for composition_level in cls]
-
-
-PublishableVersionWithMapping = namedtuple('PublishableVersionWithMapping', ['publishable_version', 'mapping'])
diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py
deleted file mode 100644
index e540e0ff3dba..000000000000
--- a/cms/djangoapps/import_from_modulestore/helpers.py
+++ /dev/null
@@ -1,466 +0,0 @@
-"""
-Helper functions for importing course content into a library.
-"""
-from datetime import datetime, timezone
-from functools import partial
-import logging
-import mimetypes
-import os
-import secrets
-from typing import TYPE_CHECKING
-
-from django.db import transaction
-from django.db.utils import IntegrityError
-from lxml import etree
-
-from opaque_keys.edx.keys import UsageKey
-from opaque_keys.edx.locator import CourseLocator
-from openedx_learning.api import authoring as authoring_api
-from openedx_learning.api.authoring_models import Component, Container, ContainerVersion, PublishableEntity
-
-from openedx.core.djangoapps.content_libraries import api
-from openedx.core.djangoapps.content_staging import api as content_staging_api
-from xmodule.modulestore.django import modulestore
-
-from .data import CompositionLevel, ImportStatus, PublishableVersionWithMapping
-from .models import Import, PublishableEntityMapping
-
-if TYPE_CHECKING:
- from openedx_learning.apps.authoring_models import LearningPackage
- from xblock.core import XBlock
-
- from openedx.core.djangoapps.content_staging.api import _StagedContent as StagedContent
-
-
-log = logging.getLogger(__name__)
-parser = etree.XMLParser(strip_cdata=False)
-
-
-class ImportClient:
- """
- Client for importing course content into a library.
-
- This class handles the import of course content from staged content into a
- content library, creating the appropriate container hierarchy based on the
- specified composition level.
- """
-
- # The create functions have different kwarg names for the child list,
- # so we need to use partial to set the child list to empty.
- CONTAINER_CREATORS_MAP: dict[str, partial] = {
- api.ContainerType.Section.olx_tag: partial(authoring_api.create_section_and_version, subsections=[]),
- api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_subsection_and_version, units=[]),
- api.ContainerType.Unit.olx_tag: partial(authoring_api.create_unit_and_version, components=[]),
- }
-
- CONTAINER_OVERRIDERS_MAP: dict[str, partial] = {
- api.ContainerType.Section.olx_tag: partial(authoring_api.create_next_section_version, subsections=[]),
- api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_next_subsection_version, units=[]),
- api.ContainerType.Unit.olx_tag: partial(authoring_api.create_next_unit_version, components=[]),
- }
-
- def __init__(
- self,
- import_event: Import,
- block_usage_key_to_import: str,
- target_learning_package: 'LearningPackage',
- staged_content: 'StagedContent',
- composition_level: str,
- override: bool = False,
- ):
- self.import_event = import_event
- self.block_usage_key_to_import = block_usage_key_to_import
- self.learning_package = target_learning_package
- self.staged_content = staged_content
- self.composition_level = composition_level
- self.override = override
-
- self.user_id = import_event.user_id
- self.content_library = target_learning_package.contentlibrary
- self.library_key = self.content_library.library_key
- self.parser = etree.XMLParser(strip_cdata=False)
-
- def import_from_staged_content(self) -> list[PublishableVersionWithMapping]:
- """
- Import staged content into a library.
- """
- node = etree.fromstring(self.staged_content.olx, parser=parser)
- usage_key = UsageKey.from_string(self.block_usage_key_to_import)
- block_to_import = get_node_for_usage_key(node, usage_key)
- if block_to_import is None:
- return []
-
- return self._process_import(self.block_usage_key_to_import, block_to_import)
-
- def _process_import(self, usage_key_string, block_to_import) -> list[PublishableVersionWithMapping]:
- """
- Process import of a block from staged content into a library.
-
- Imports a block and its children into the library based on the
- composition level. It handles both simple and complicated blocks, creating
- the necessary container hierarchy.
- """
- usage_key = UsageKey.from_string(usage_key_string)
- result = []
-
- if block_to_import.tag not in CompositionLevel.OLX_COMPLEX_LEVELS.value:
- return self._import_simple_block(block_to_import, usage_key)
-
- for child in block_to_import.getchildren():
- child_usage_key_string = get_usage_key_string_from_staged_content(
- self.staged_content, child.get('url_name')
- )
- if not child_usage_key_string:
- continue
-
- result.extend(self._import_child_block(child, child_usage_key_string))
-
- if self.composition_level == CompositionLevel.COMPONENT.value:
- return [
- publishable_version_with_mapping for publishable_version_with_mapping in result
- if not isinstance(publishable_version_with_mapping.publishable_version, ContainerVersion)
- ]
- return result
-
- def _import_simple_block(self, block_to_import, usage_key) -> list[PublishableVersionWithMapping]:
- """
- Import a simple block into the library.
-
- Creates a block in the library from the staged content block.
- It returns a list containing the created component version.
- """
- publishable_version_with_mapping = self._create_block_in_library(block_to_import, usage_key)
- return [publishable_version_with_mapping] if publishable_version_with_mapping else []
-
- def _import_child_block(self, child, child_usage_key_string):
- """
- Import a child block into the library.
-
- Determines whether the child block is simple or complicated and
- delegates the import process to the appropriate helper method.
- """
- child_usage_key = UsageKey.from_string(child_usage_key_string)
- if child.tag in CompositionLevel.OLX_COMPLEX_LEVELS.value:
- return self._import_complicated_child(child, child_usage_key_string)
- else:
- return self._import_simple_block(child, child_usage_key)
-
- def _import_complicated_child(self, child, child_usage_key_string):
- """
- Import a complicated child block into the library.
-
- Handles the import of complicated child blocks, including creating
- containers and updating components.
- Returns a list containing the created container version.
- """
- if not self._should_create_container(child.tag):
- return self._process_import(child_usage_key_string, child)
-
- container_version_with_mapping = self.get_or_create_container(
- child.tag,
- child.get('url_name'),
- child.get('display_name', child.tag),
- child_usage_key_string,
- )
- child_component_versions_with_mapping = self._process_import(child_usage_key_string, child)
- child_component_versions = [
- child_component_version.publishable_version for child_component_version
- in child_component_versions_with_mapping
- ]
- self._update_container_components(container_version_with_mapping.publishable_version, child_component_versions)
- return [container_version_with_mapping] + child_component_versions_with_mapping
-
- def _should_create_container(self, container_type: str) -> bool:
- """
- Determine if a new container should be created.
-
- Container type should be at a lower level than the current composition level.
- """
- composition_hierarchy = CompositionLevel.OLX_COMPLEX_LEVELS.value
- return (
- container_type in composition_hierarchy and
- self.composition_level in composition_hierarchy and
- composition_hierarchy.index(container_type) <= composition_hierarchy.index(self.composition_level)
- )
-
- def get_or_create_container(
- self,
- container_type: str,
- key: str,
- display_name: str,
- block_usage_key_string: str
- ) -> PublishableVersionWithMapping:
- """
- Create a container of the specified type.
-
- Creates a container (e.g., chapter, sequential, vertical) in the
- content library.
- """
- try:
- container_creator_func = self.CONTAINER_CREATORS_MAP[container_type]
- container_override_func = self.CONTAINER_OVERRIDERS_MAP[container_type]
- except KeyError as exc:
- raise ValueError(f"Unknown container type: {container_type}") from exc
-
- try:
- container_version = self.content_library.learning_package.publishable_entities.get(key=key)
- except PublishableEntity.DoesNotExist:
- container_version = None
-
- if container_version and self.override:
- container_version = container_override_func(
- container_version.container,
- title=display_name or f"New {container_type}",
- created=datetime.now(tz=timezone.utc),
- created_by=self.import_event.user_id,
- )
- elif not container_version:
- _, container_version = container_creator_func(
- self.learning_package.id,
- key=key or secrets.token_hex(16),
- title=display_name or f"New {container_type}",
- created=datetime.now(tz=timezone.utc),
- created_by=self.import_event.user_id,
- )
-
- publishable_entity_mapping, _ = get_or_create_publishable_entity_mapping(
- block_usage_key_string,
- container_version.container
- )
-
- return PublishableVersionWithMapping(container_version, publishable_entity_mapping)
-
- def _update_container_components(self, container_version, component_versions):
- """
- Update components of a container.
- """
- entity_rows = [
- authoring_api.ContainerEntityRow(
- entity_pk=cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk,
- version_pk=cv.pk,
- )
- for cv in component_versions
- ]
- return authoring_api.create_next_container_version(
- container_pk=container_version.container.pk,
- title=container_version.title,
- entity_rows=entity_rows,
- created=datetime.now(tz=timezone.utc),
- created_by=self.import_event.user_id,
- container_version_cls=container_version.__class__,
- )
-
- def _create_block_in_library(self, block_to_import, usage_key) -> PublishableVersionWithMapping | None:
- """
- Create a block in a library from a staged content block.
- """
- now = datetime.now(tz=timezone.utc)
- staged_content_files = content_staging_api.get_staged_content_static_files(self.staged_content.id)
-
- with transaction.atomic():
- component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type)
- does_component_exist = authoring_api.get_components(
- self.learning_package.id
- ).filter(local_key=usage_key.block_id).exists()
-
- if does_component_exist:
- if not self.override:
- log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.")
- return None
- else:
- component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import))
- else:
- try:
- _, library_usage_key = api.validate_can_add_block_to_library(
- self.library_key,
- block_to_import.tag,
- usage_key.block_id,
- )
- except api.IncompatibleTypesError as e:
- log.error(f"Error validating block {usage_key} for library {self.library_key}: {e}")
- return None
-
- authoring_api.create_component(
- self.learning_package.id,
- component_type=component_type,
- local_key=usage_key.block_id,
- created=now,
- created_by=self.import_event.user_id,
- )
- component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import))
-
- self._process_staged_content_files(
- component_version,
- staged_content_files,
- usage_key,
- block_to_import,
- now,
- )
- publishable_entity_mapping, _ = get_or_create_publishable_entity_mapping(
- usage_key,
- component_version.component
- )
- return PublishableVersionWithMapping(component_version, publishable_entity_mapping)
-
- def _handle_component_override(self, usage_key, new_content):
- """
- Create new ComponentVersion for overridden component.
- """
- component_version = None
- try:
- component = authoring_api.get_components(self.learning_package.id).get(local_key=usage_key.block_id)
- except Component.DoesNotExist:
- return component_version
- library_usage_key = api.library_component_usage_key(self.library_key, component)
-
- component_version = api.set_library_block_olx(library_usage_key, new_content)
-
- return component_version
-
- def _process_staged_content_files(
- self,
- component_version,
- staged_content_files,
- usage_key,
- block_to_import,
- created_at,
- ):
- """
- Process staged content files for a component.
-
- Processes the staged content files for a component, creating the
- necessary file content and associating it with the component version.
- """
- block_olx = etree.tostring(block_to_import).decode('utf-8')
-
- for staged_content_file_data in staged_content_files:
- original_filename = staged_content_file_data.filename
- file_basename = os.path.basename(original_filename)
- file_basename_no_ext, _ = os.path.splitext(file_basename)
-
- # Skip files not referenced in the block
- if file_basename not in block_olx and file_basename_no_ext not in block_olx:
- log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}")
- continue
-
- file_data = content_staging_api.get_staged_content_static_file_data(
- self.staged_content.id,
- original_filename,
- )
- if not file_data:
- log.error(
- f"Staged content {self.staged_content.id} included referenced "
- f"file {original_filename}, but no file data was found."
- )
- continue
-
- filename = f"static/{file_basename}"
- media_type_str, _ = mimetypes.guess_type(filename)
- if not media_type_str:
- media_type_str = "application/octet-stream"
-
- media_type = authoring_api.get_or_create_media_type(media_type_str)
- content = authoring_api.get_or_create_file_content(
- self.learning_package.id,
- media_type.id,
- data=file_data,
- created=created_at,
- )
-
- try:
- authoring_api.create_component_version_content(component_version.pk, content.id, key=filename)
- except IntegrityError:
- pass # Content already exists
-
-
-def import_from_staged_content(
- import_event: Import,
- usage_key_string: str,
- target_learning_package: 'LearningPackage',
- staged_content: 'StagedContent',
- composition_level: str,
- override: bool = False,
-) -> list[PublishableVersionWithMapping]:
- """
- Import staged content to a library from staged content.
-
- Returns a list of PublishableVersionWithMappings created during the import.
- """
- import_client = ImportClient(
- import_event,
- usage_key_string,
- target_learning_package,
- staged_content,
- composition_level,
- override,
- )
- return import_client.import_from_staged_content()
-
-
-def get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]:
- """
- Creates a mapping between the source usage key and the target publishable entity.
- """
- if isinstance(component, Container):
- target_package = component.publishable_entity.learning_package
- else:
- target_package = component.learning_package
- return PublishableEntityMapping.objects.get_or_create(
- source_usage_key=usage_key,
- target_entity=component.publishable_entity,
- target_package=target_package,
- )
-
-
-def get_usage_key_string_from_staged_content(staged_content: 'StagedContent', block_id: str) -> str | None:
- """
- Get the usage ID from a staged content by block ID.
- """
- if staged_content.tags is None:
- return None
- return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None)
-
-
-def get_node_for_usage_key(node: etree._Element, usage_key: UsageKey) -> etree._Element:
- """
- Get the node in an XML tree which matches to the usage key.
- """
- if node.tag == usage_key.block_type and node.get('url_name') == usage_key.block_id:
- return node
-
- for child in node.getchildren():
- found = get_node_for_usage_key(child, usage_key)
- if found is not None:
- return found
-
-
-def get_items_to_import(import_event: Import) -> list['XBlock']:
- """
- Collect items to import from a course.
- """
- items_to_import: list['XBlock'] = []
- if isinstance(import_event.source_key, CourseLocator):
- items_to_import.extend(
- modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or []
- )
- items_to_import.extend(
- modulestore().get_items(import_event.source_key, qualifiers={"category": "static_tab"}) or []
- )
-
- return items_to_import
-
-
-def cancel_incomplete_old_imports(import_event: Import) -> None:
- """
- Cancel any incomplete imports that have the same target as the current import.
-
- When a new import is created, we want to cancel any other incomplete user imports that have the same target.
- """
- incomplete_user_imports_with_same_target = Import.objects.filter(
- user=import_event.user,
- target_change=import_event.target_change,
- source_key=import_event.source_key,
- staged_content_for_import__isnull=False
- ).exclude(uuid=import_event.uuid)
- for incomplete_import in incomplete_user_imports_with_same_target:
- incomplete_import.set_status(ImportStatus.CANCELED)
diff --git a/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py b/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py
deleted file mode 100644
index a61040b9f19e..000000000000
--- a/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Generated by Django 4.2.20 on 2025-04-21 16:23
-
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-import django.utils.timezone
-import model_utils.fields
-import opaque_keys.edx.django.models
-import uuid
-
-
-class Migration(migrations.Migration):
-
- initial = True
-
- dependencies = [
- ('content_staging', '0005_stagedcontent_version_num'),
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
- ]
-
- operations = [
- migrations.CreateModel(
- name='Import',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
- ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
- ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)),
- ('status', models.CharField(choices=[('not_started', 'Waiting to stage content'), ('staging', 'Staging content for import'), ('Failed to stage content', 'Staging Failed'), ('staged', 'Content is staged and ready for import'), ('importing', 'Importing staged content'), ('importing_failed', 'Failed to import staged content'), ('imported', 'Successfully imported content'), ('canceled', 'Canceled')], db_index=True, default='not_started', max_length=100)),
- ('source_key', opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, help_text='The modulestore course', max_length=255)),
- ('target_change', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelog')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
- ],
- options={
- 'verbose_name': 'Import from modulestore',
- 'verbose_name_plural': 'Imports from modulestore',
- },
- ),
- migrations.CreateModel(
- name='PublishableEntityMapping',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
- ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
- ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key/ID of the thing that has been imported.', max_length=255)),
- ('target_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishableentity')),
- ('target_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')),
- ],
- options={
- 'unique_together': {('source_usage_key', 'target_package')},
- },
- ),
- migrations.CreateModel(
- name='StagedContentForImport',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
- ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
- ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(help_text='The original Usage key of the highest-level component that was saved in StagedContent.', max_length=255)),
- ('import_event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='import_from_modulestore.import')),
- ('staged_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='content_staging.stagedcontent')),
- ],
- options={
- 'unique_together': {('import_event', 'staged_content')},
- },
- ),
- migrations.CreateModel(
- name='PublishableEntityImport',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
- ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
- ('import_event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_from_modulestore.import')),
- ('resulting_change', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelogrecord')),
- ('resulting_mapping', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='import_from_modulestore.publishableentitymapping')),
- ],
- options={
- 'unique_together': {('import_event', 'resulting_mapping')},
- },
- ),
- ]
diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py
deleted file mode 100644
index 5b6122749bba..000000000000
--- a/cms/djangoapps/import_from_modulestore/models.py
+++ /dev/null
@@ -1,139 +0,0 @@
-"""
-Models for the course to library import app.
-"""
-import uuid as uuid_tools
-
-from django.contrib.auth import get_user_model
-from django.db import models
-from django.utils.translation import gettext_lazy as _
-
-from model_utils.models import TimeStampedModel
-from opaque_keys.edx.django.models import (
- LearningContextKeyField,
- UsageKeyField,
-)
-from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity
-
-from .data import ImportStatus
-
-User = get_user_model()
-
-
-class Import(TimeStampedModel):
- """
- Represents the action of a user importing a modulestore-based course or legacy
- library into a learning-core based learning package (today, that is always a content library).
- """
-
- uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True)
- status = models.CharField(
- max_length=100,
- choices=ImportStatus.choices,
- default=ImportStatus.NOT_STARTED,
- db_index=True
- )
- user = models.ForeignKey(User, on_delete=models.CASCADE)
-
- # Note: For now, this will always be a course key. In the future, it may be a legacy library key.
- source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True)
- target_change = models.ForeignKey(to='oel_publishing.DraftChangeLog', on_delete=models.SET_NULL, null=True)
-
- class Meta:
- verbose_name = _('Import from modulestore')
- verbose_name_plural = _('Imports from modulestore')
-
- def __str__(self):
- return f'{self.source_key} → {self.target_change}'
-
- def set_status(self, status: ImportStatus):
- """
- Set import status.
- """
- self.status = status
- self.save()
- if status in [ImportStatus.IMPORTED, ImportStatus.CANCELED]:
- self.clean_related_staged_content()
-
- def clean_related_staged_content(self) -> None:
- """
- Clean related staged content.
- """
- for staged_content_for_import in self.staged_content_for_import.all():
- staged_content_for_import.staged_content.delete()
-
-
-class PublishableEntityMapping(TimeStampedModel):
- """
- Represents a mapping between a source usage key and a target publishable entity.
- """
-
- source_usage_key = UsageKeyField(
- max_length=255,
- help_text=_('Original usage key/ID of the thing that has been imported.'),
- )
- target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE)
- target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE)
-
- class Meta:
- unique_together = ('source_usage_key', 'target_package')
-
- def __str__(self):
- return f'{self.source_usage_key} → {self.target_entity}'
-
-
-class PublishableEntityImport(TimeStampedModel):
- """
- Represents a publishableentity version that has been imported into a learning package (e.g. content library)
-
- This is a many-to-many relationship between a container version and a course to library import.
- """
-
- import_event = models.ForeignKey(Import, on_delete=models.CASCADE)
- resulting_mapping = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True)
- resulting_change = models.OneToOneField(
- to='oel_publishing.DraftChangeLogRecord',
- # a changelog record can be pruned, which would set this to NULL, but not delete the
- # entire import record
- null=True,
- on_delete=models.SET_NULL,
- )
-
- class Meta:
- unique_together = (
- ('import_event', 'resulting_mapping'),
- )
-
- def __str__(self):
- return f'{self.import_event} → {self.resulting_mapping}'
-
-
-class StagedContentForImport(TimeStampedModel):
- """
- Represents m2m relationship between an import and staged content created for that import.
- """
-
- import_event = models.ForeignKey(
- Import,
- on_delete=models.CASCADE,
- related_name='staged_content_for_import',
- )
- staged_content = models.OneToOneField(
- to='content_staging.StagedContent',
- on_delete=models.CASCADE,
- related_name='staged_content_for_import',
- )
- # Since StagedContent stores all the keys of the saved blocks, this field was added to optimize search.
- source_usage_key = UsageKeyField(
- max_length=255,
- help_text=_(
- 'The original Usage key of the highest-level component that was saved in StagedContent.'
- ),
- )
-
- class Meta:
- unique_together = (
- ('import_event', 'staged_content'),
- )
-
- def __str__(self):
- return f'{self.import_event} → {self.staged_content}'
diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py
deleted file mode 100644
index 4644b29e4904..000000000000
--- a/cms/djangoapps/import_from_modulestore/tasks.py
+++ /dev/null
@@ -1,101 +0,0 @@
-"""
-Tasks for course to library import.
-"""
-
-from celery import shared_task
-from celery.utils.log import get_task_logger
-from django.db import transaction
-from edx_django_utils.monitoring import set_code_owner_attribute
-
-from openedx_learning.api import authoring as authoring_api
-from openedx_learning.api.authoring_models import LearningPackage
-from openedx.core.djangoapps.content_staging import api as content_staging_api
-
-from .constants import IMPORT_FROM_MODULESTORE_STAGING_PURPOSE
-from .data import ImportStatus
-from .helpers import get_items_to_import, import_from_staged_content
-from .models import Import, PublishableEntityImport, StagedContentForImport
-from .validators import validate_composition_level
-
-log = get_task_logger(__name__)
-
-
-@shared_task
-@set_code_owner_attribute
-def save_legacy_content_to_staged_content_task(import_uuid: str) -> None:
- """
- Save courses to staged content task by sections/chapters.
- """
- import_event = Import.objects.get(uuid=import_uuid)
-
- import_event.clean_related_staged_content()
- import_event.set_status(ImportStatus.STAGING)
- try:
- with transaction.atomic():
- items_to_import = get_items_to_import(import_event)
- for item in items_to_import:
- staged_content = content_staging_api.stage_xblock_temporarily(
- item,
- import_event.user.id,
- purpose=IMPORT_FROM_MODULESTORE_STAGING_PURPOSE,
- )
- StagedContentForImport.objects.create(
- staged_content=staged_content,
- import_event=import_event,
- source_usage_key=item.location
- )
-
- if items_to_import:
- import_event.set_status(ImportStatus.STAGED)
- else:
- import_event.set_status(ImportStatus.STAGING_FAILED)
- except Exception as exc: # pylint: disable=broad-except
- import_event.set_status(ImportStatus.STAGING_FAILED)
- raise exc
-
-
-@shared_task
-@set_code_owner_attribute
-def import_staged_content_to_library_task(
- usage_key_strings: list[str],
- import_uuid: str,
- learning_package_id: int,
- user_id: int,
- composition_level: str,
- override: bool,
-) -> None:
- """
- Import staged content to a library task.
- """
- validate_composition_level(composition_level)
-
- import_event = Import.objects.get(uuid=import_uuid, status=ImportStatus.STAGED, user_id=user_id)
- target_learning_package = LearningPackage.objects.get(id=learning_package_id)
-
- imported_publishable_versions = []
- with authoring_api.bulk_draft_changes_for(learning_package_id=learning_package_id) as change_log:
- try:
- for usage_key_string in usage_key_strings:
- staged_content_for_import = import_event.staged_content_for_import.get(
- source_usage_key=usage_key_string
- )
- publishable_versions = import_from_staged_content(
- import_event,
- usage_key_string,
- target_learning_package,
- staged_content_for_import.staged_content,
- composition_level,
- override,
- )
- imported_publishable_versions.extend(publishable_versions)
- except: # pylint: disable=bare-except
- import_event.set_status(ImportStatus.IMPORTING_FAILED)
- raise
-
- import_event.set_status(ImportStatus.IMPORTED)
- for imported_component_version in imported_publishable_versions:
- PublishableEntityImport.objects.create(
- import_event=import_event,
- resulting_mapping=imported_component_version.mapping,
- resulting_change=change_log.records.get(entity=imported_component_version.mapping.target_entity),
- )
diff --git a/cms/djangoapps/import_from_modulestore/tests/factories.py b/cms/djangoapps/import_from_modulestore/tests/factories.py
deleted file mode 100644
index 368cc0ed94ff..000000000000
--- a/cms/djangoapps/import_from_modulestore/tests/factories.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Factories for Import model.
-"""
-
-import uuid
-
-import factory
-from factory.django import DjangoModelFactory
-from opaque_keys.edx.keys import CourseKey
-
-from common.djangoapps.student.tests.factories import UserFactory
-from cms.djangoapps.import_from_modulestore.models import Import
-
-
-class ImportFactory(DjangoModelFactory):
- """
- Factory for Import model.
- """
-
- class Meta:
- model = Import
-
- @factory.lazy_attribute
- def source_key(self):
- return CourseKey.from_string(f'course-v1:edX+DemoX+{self.uuid}')
-
- uuid = factory.LazyFunction(lambda: str(uuid.uuid4()))
- user = factory.SubFactory(UserFactory)
diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py
deleted file mode 100644
index 62fe2e4159a9..000000000000
--- a/cms/djangoapps/import_from_modulestore/tests/test_api.py
+++ /dev/null
@@ -1,109 +0,0 @@
-"""
-Test cases for import_from_modulestore.api module.
-"""
-from unittest.mock import patch
-
-import pytest
-from opaque_keys.edx.keys import CourseKey
-from organizations.models import Organization
-
-from common.djangoapps.student.tests.factories import UserFactory
-from cms.djangoapps.import_from_modulestore.api import import_staged_content_to_library, stage_content_for_import
-from cms.djangoapps.import_from_modulestore.data import ImportStatus
-from cms.djangoapps.import_from_modulestore.models import Import
-from openedx.core.djangoapps.content_libraries import api as content_libraries_api
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from .factories import ImportFactory
-
-
-@pytest.mark.django_db
-class TestCourseToLibraryImportAPI(ModuleStoreTestCase):
- """
- Test cases for Import API.
- """
-
- def setUp(self):
- super().setUp()
-
- _library_metadata = content_libraries_api.create_library(
- org=Organization.objects.create(name='Organization 1', short_name='org1'),
- slug='lib_1',
- title='Library Org 1',
- description='This is a library from Org 1',
- )
- self.library = content_libraries_api.ContentLibrary.objects.get_by_key(_library_metadata.key)
-
- def test_stage_content_for_import(self):
- """
- Test stage_content_for_import function.
- """
- course_id = "course-v1:edX+DemoX+Demo_Course"
- user = UserFactory()
- stage_content_for_import(course_id, user.id)
-
- import_event = Import.objects.get()
- assert import_event.source_key == CourseKey.from_string(course_id)
- assert import_event.user_id == user.id
- assert import_event.status == ImportStatus.NOT_STARTED
-
- def test_import_staged_content_to_library(self):
- """
- Test import_staged_content_to_library function with different override values.
- """
- import_event = ImportFactory(
- source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"),
- )
- usage_ids = [
- "block-v1:edX+DemoX+Demo_Course+type@chapter+block@123",
- "block-v1:edX+DemoX+Demo_Course+type@chapter+block@456",
- ]
- override = False
-
- with patch(
- "cms.djangoapps.import_from_modulestore.api.import_staged_content_to_library_task"
- ) as import_staged_content_to_library_task_mock:
- import_staged_content_to_library(
- usage_ids,
- import_event.uuid,
- self.library.learning_package.id,
- import_event.user.id,
- "xblock",
- override
- )
-
- import_staged_content_to_library_task_mock.apply_async.assert_called_once_with(
- kwargs={
- "usage_key_strings": usage_ids,
- "import_uuid": import_event.uuid,
- "learning_package_id": self.library.learning_package.id,
- "user_id": import_event.user.id,
- "composition_level": "xblock",
- "override": override,
- },
- )
-
- def test_import_staged_content_to_library_invalid_usage_key(self):
- """
- Test import_staged_content_to_library function with not chapter usage keys.
- """
- import_event = ImportFactory(
- source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"),
- )
- usage_ids = [
- "block-v1:edX+DemoX+Demo_Course+type@problem+block@123",
- "block-v1:edX+DemoX+Demo_Course+type@vertical+block@456",
- ]
-
- with patch(
- "cms.djangoapps.import_from_modulestore.api.import_staged_content_to_library_task"
- ) as import_staged_content_to_library_task_mock:
- with self.assertRaises(ValueError):
- import_staged_content_to_library(
- usage_ids,
- import_event.uuid,
- self.library.learning_package.id,
- import_event.user.id,
- "xblock",
- False
- )
- import_staged_content_to_library_task_mock.apply_async.assert_not_called()
diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py
deleted file mode 100644
index 1d2ce26867aa..000000000000
--- a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py
+++ /dev/null
@@ -1,396 +0,0 @@
-"""
-Tests for the import_from_modulestore helper functions.
-"""
-import ddt
-from organizations.models import Organization
-from unittest import mock
-from unittest.mock import patch
-
-from lxml import etree
-from openedx_learning.api.authoring_models import LearningPackage
-
-from cms.djangoapps.import_from_modulestore import api
-from cms.djangoapps.import_from_modulestore.helpers import ImportClient
-from common.djangoapps.student.tests.factories import UserFactory
-
-from openedx.core.djangoapps.content_libraries import api as content_libraries_api
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory
-
-
-@ddt.ddt
-class TestImportClient(ModuleStoreTestCase):
- """
- Functional tests for the ImportClient class.
- """
-
- def setUp(self):
- super().setUp()
- self.library = content_libraries_api.create_library(
- org=Organization.objects.create(name='Organization 1', short_name='org1'),
- slug='lib_1',
- title='Library Org 1',
- description='This is a library from Org 1',
- )
- self.learning_package = LearningPackage.objects.get(id=self.library.learning_package_id)
- self.user = UserFactory()
- self.course = CourseFactory.create()
- self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter')
- self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential')
- self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical')
- self.problem = BlockFactory.create(
- category='problem',
- parent=self.vertical,
- display_name='Problem',
- data="""""",
- )
- self.video = BlockFactory.create(
- category='video',
- parent=self.vertical,
- display_name='Video',
- data="""""",
- )
- with self.captureOnCommitCallbacks(execute=True):
- self.import_event = api.stage_content_for_import(source_key=self.course.id, user_id=self.user.id)
- self.parser = etree.XMLParser(strip_cdata=False)
-
- def test_import_from_staged_content(self):
- expected_imported_xblocks = [self.video, self.problem]
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=str(self.chapter.location),
- composition_level='xblock',
- override=False
- )
-
- import_client.import_from_staged_content()
-
- self.assertEqual(self.learning_package.content_set.count(), len(expected_imported_xblocks))
-
- @patch('cms.djangoapps.import_from_modulestore.helpers.ImportClient._process_import')
- def test_import_from_staged_content_block_not_found(self, mocked_process_import):
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import='block-v1:edX+Demo+2025+type@chapter+block@12345',
- composition_level='xblock',
- override=False
- )
-
- import_client.import_from_staged_content()
-
- self.assertTrue(not self.learning_package.content_set.count())
- mocked_process_import.assert_not_called()
-
- @ddt.data(
- 'chapter',
- 'sequential',
- 'vertical'
- )
- def test_create_container(self, block_lvl):
- container_to_import = getattr(self, block_lvl)
- block_usage_key_to_import = str(container_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content_for_import.staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=False
- )
- import_client.get_or_create_container(
- container_to_import.category,
- container_to_import.location.block_id,
- container_to_import.display_name,
- str(container_to_import.location)
- )
-
- self.assertEqual(self.learning_package.publishable_entities.count(), 1)
-
- def test_create_container_with_xblock(self):
- block_usage_key_to_import = str(self.problem.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content_for_import.staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=False
- )
- with self.assertRaises(ValueError):
- import_client.get_or_create_container(
- self.problem.category,
- self.problem.location.block_id,
- self.problem.display_name,
- str(self.problem.location),
- )
-
- @ddt.data('chapter', 'sequential', 'vertical')
- def test_process_import_with_complicated_blocks(self, block_lvl):
- container_to_import = getattr(self, block_lvl)
- block_usage_key_to_import = str(container_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
- expected_imported_xblocks = [self.problem, self.video]
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- block_usage_key_to_import=block_usage_key_to_import,
- target_learning_package=self.learning_package,
- composition_level='xblock',
- override=False
- )
- block_to_import = etree.fromstring(staged_content.olx, parser=self.parser)
- # pylint: disable=protected-access
- result = import_client.import_from_staged_content()
-
- self.assertEqual(self.learning_package.content_set.count(), len(expected_imported_xblocks))
- self.assertEqual(len(result), len(expected_imported_xblocks))
-
- @ddt.data('problem', 'video')
- def test_process_import_with_simple_blocks(self, block_type_to_import):
- block_to_import = getattr(self, block_type_to_import)
- block_usage_key_to_import = str(block_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- expected_imported_xblocks = [block_to_import]
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content_for_import.staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=False
- )
-
- block_to_import = etree.fromstring(block_to_import.data, parser=self.parser)
- # pylint: disable=protected-access
- result = import_client._process_import(block_usage_key_to_import, block_to_import)
-
- self.assertEqual(self.learning_package.content_set.count(), len(expected_imported_xblocks))
- self.assertEqual(len(result), len(expected_imported_xblocks))
-
- @ddt.data(True, False)
- def test_process_import_with_override(self, override):
- block_to_import = self.problem
- block_usage_key_to_import = str(block_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content_for_import.staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=False
- )
-
- block_xml = etree.fromstring(block_to_import.data, parser=self.parser)
- # pylint: disable=protected-access
- result1 = import_client._process_import(block_usage_key_to_import, block_xml)
- self.assertEqual(len(result1), 1)
-
- with self.captureOnCommitCallbacks(execute=True):
- new_import_event = api.stage_content_for_import(source_key=self.course.id, user_id=self.user.id)
-
- staged_content_for_import = new_import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- new_staged_content = staged_content_for_import.staged_content
- import_client = ImportClient(
- import_event=new_import_event,
- staged_content=new_staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=override
- )
-
- if override:
- modified_data = block_to_import.data.replace('DisplayName', 'ModifiedName')
- modified_block = BlockFactory.create(
- category='problem',
- parent=self.vertical,
- display_name='Modified Problem',
- data=modified_data,
- )
- block_xml = etree.fromstring(modified_block.data, parser=self.parser)
-
- # pylint: disable=protected-access
- result2 = import_client._process_import(block_usage_key_to_import, block_xml)
- self.assertEqual(len(result2), 1)
-
- assert result2[0].publishable_version.title == 'ModifiedName'
- else:
- # pylint: disable=protected-access
- result2 = import_client._process_import(block_usage_key_to_import, block_xml)
- self.assertEqual(result2, [])
-
- @patch('cms.djangoapps.import_from_modulestore.helpers.authoring_api')
- def test_container_override(self, mock_authoring_api):
- container_to_import = self.vertical
- block_usage_key_to_import = str(container_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='vertical',
- override=False
- )
-
- container_version_with_mapping = import_client.get_or_create_container(
- 'vertical',
- container_to_import.location.block_id,
- container_to_import.display_name,
- str(container_to_import.location),
- )
- assert container_version_with_mapping is not None
- assert container_version_with_mapping.publishable_version.title == container_to_import.display_name
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='vertical',
- override=True
- )
- container_version_with_mapping = import_client.get_or_create_container(
- 'vertical',
- container_to_import.location.block_id,
- 'New Display Name',
- str(container_to_import.location),
- )
- overrided_container_version = container_version_with_mapping.publishable_version
- assert overrided_container_version is not None
- assert overrided_container_version.title == 'New Display Name'
-
- @ddt.data('xblock', 'vertical')
- def test_composition_levels(self, composition_level):
- if composition_level == 'xblock':
- expected_imported_blocks = [self.problem, self.video]
- else:
- # The vertical block is expected to be imported as a container
- # with the same location as the original vertical block.
- expected_imported_blocks = [self.vertical, self.problem, self.video]
-
- container_to_import = self.vertical
- block_usage_key_to_import = str(container_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level=composition_level,
- override=False
- )
-
- block_xml = etree.fromstring(staged_content.olx, parser=self.parser)
- # pylint: disable=protected-access
- result = import_client._process_import(block_usage_key_to_import, block_xml)
-
- self.assertEqual(len(result), len(expected_imported_blocks))
-
- @patch('cms.djangoapps.import_from_modulestore.helpers.content_staging_api')
- def test_process_staged_content_files(self, mock_content_staging_api):
- block_to_import = self.problem
- block_usage_key_to_import = str(block_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
- staged_content = staged_content_for_import.staged_content
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='xblock',
- override=False
- )
-
- mock_file_data = b'file content'
- mock_file = mock.MagicMock()
- mock_file.filename = 'test.png'
- mock_content_staging_api.get_staged_content_static_files.return_value = [mock_file]
- mock_content_staging_api.get_staged_content_static_file_data.return_value = mock_file_data
-
- modified_data = '
'
- modified_block = BlockFactory.create(
- category='problem',
- parent=self.vertical,
- display_name='Problem With Image',
- data=modified_data,
- )
- block_xml = etree.fromstring(modified_data, parser=self.parser)
-
- # pylint: disable=protected-access
- import_client._create_block_in_library(block_xml, modified_block.location)
- mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with(
- staged_content.id, 'test.png'
- )
-
- def test_update_container_components(self):
- container_to_import = self.vertical
- block_usage_key_to_import = str(container_to_import.location)
- staged_content_for_import = self.import_event.staged_content_for_import.get(
- source_usage_key=self.chapter.location
- )
-
- import_client = ImportClient(
- import_event=self.import_event,
- staged_content=staged_content_for_import.staged_content,
- target_learning_package=self.learning_package,
- block_usage_key_to_import=block_usage_key_to_import,
- composition_level='container',
- override=False
- )
-
- with patch('cms.djangoapps.import_from_modulestore.helpers.authoring_api') as mock_authoring_api:
- mock_container_version = mock.MagicMock()
- mock_component_version1 = mock.MagicMock()
- mock_component_version2 = mock.MagicMock()
- mock_component_versions = [mock_component_version1, mock_component_version2]
-
- # pylint: disable=protected-access
- import_client._update_container_components(mock_container_version, mock_component_versions)
-
- mock_authoring_api.create_next_container_version.assert_called_once()
- call_args = mock_authoring_api.create_next_container_version.call_args[1]
- self.assertEqual(call_args['container_pk'], mock_container_version.container.pk)
- self.assertEqual(call_args['title'], mock_container_version.title)
- self.assertEqual(call_args['created_by'], self.user.id)
diff --git a/cms/djangoapps/import_from_modulestore/tests/test_tasks.py b/cms/djangoapps/import_from_modulestore/tests/test_tasks.py
deleted file mode 100644
index 8c24fa95c1cc..000000000000
--- a/cms/djangoapps/import_from_modulestore/tests/test_tasks.py
+++ /dev/null
@@ -1,169 +0,0 @@
-"""
-Tests for tasks in import_from_modulestore app.
-"""
-from django.core.exceptions import ObjectDoesNotExist
-from organizations.models import Organization
-from openedx_learning.api.authoring_models import LearningPackage
-from unittest.mock import patch
-
-from cms.djangoapps.import_from_modulestore.data import ImportStatus
-from cms.djangoapps.import_from_modulestore.tasks import (
- import_staged_content_to_library_task,
- save_legacy_content_to_staged_content_task,
-)
-from openedx.core.djangoapps.content_libraries import api as content_libraries_api
-from openedx.core.djangoapps.content_libraries.api import ContentLibrary
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory
-
-from .factories import ImportFactory
-
-
-class ImportCourseToLibraryMixin(ModuleStoreTestCase):
- """
- Mixin for setting up data for tests.
- """
-
- def setUp(self):
- super().setUp()
-
- self.library = content_libraries_api.create_library(
- org=Organization.objects.create(name='Organization 1', short_name='org1'),
- slug='lib_1',
- title='Library Org 1',
- description='This is a library from Org 1',
- )
- self.content_library = ContentLibrary.objects.get_by_key(self.library.key)
-
- self.course = CourseFactory.create()
- self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter 1')
- self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential 1')
- self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical 1')
- self.video = BlockFactory.create(category='video', parent=self.vertical, display_name='Video 1')
- self.problem = BlockFactory.create(category='problem', parent=self.vertical, display_name='Problem 1')
-
- # self.course2 = CourseFactory.create()
- # self.chapter2 = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter 2')
- self.chapter2 = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter 2')
- self.sequential2 = BlockFactory.create(category='sequential', parent=self.chapter2, display_name='Sequential 2')
- self.vertical2 = BlockFactory.create(category='vertical', parent=self.sequential2, display_name='Vertical 2')
- self.video2 = BlockFactory.create(category='video', parent=self.vertical2, display_name='Video 2')
- self.problem2 = BlockFactory.create(category='problem', parent=self.vertical2, display_name='Problem 2')
-
- self.import_event = ImportFactory(source_key=self.course.id)
- self.user = self.import_event.user
-
-
-class TestSaveCourseSectionsToStagedContentTask(ImportCourseToLibraryMixin):
- """
- Test cases for save_course_sections_to_staged_content_task.
- """
-
- def test_save_legacy_content_to_staged_content_task(self):
- """
- End-to-end test for save_legacy_content_to_staged_content_task.
- """
- course_chapters_to_import = [self.chapter, self.chapter2]
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
-
- self.import_event.refresh_from_db()
- self.assertEqual(self.import_event.staged_content_for_import.count(), len(course_chapters_to_import))
- self.assertEqual(self.import_event.status, ImportStatus.STAGED)
-
- def test_old_staged_content_deletion_before_save_new(self):
- """ Checking that repeated saving of the same content does not create duplicates. """
- course_chapters_to_import = [self.chapter, self.chapter2]
-
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
-
- self.assertEqual(self.import_event.staged_content_for_import.count(), len(course_chapters_to_import))
-
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
-
- self.assertEqual(self.import_event.staged_content_for_import.count(), len(course_chapters_to_import))
-
-
-class TestImportLibraryFromStagedContentTask(ImportCourseToLibraryMixin):
- """
- Test cases for import_staged_content_to_library_task.
- """
-
- def _is_imported(self, library, xblock):
- library_learning_package = LearningPackage.objects.get(id=library.learning_package_id)
- self.assertTrue(library_learning_package.content_set.filter(text__icontains=xblock.display_name).exists())
-
- def test_import_staged_content_to_library_task(self):
- """ End-to-end test for import_staged_content_to_library_task. """
- library_learning_package = LearningPackage.objects.get(id=self.library.learning_package_id)
- self.assertEqual(library_learning_package.content_set.count(), 0)
- expected_imported_xblocks = [self.problem, self.problem2, self.video, self.video2]
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
-
- import_staged_content_to_library_task(
- [str(self.chapter.location), str(self.chapter2.location)],
- self.import_event.uuid,
- self.content_library.learning_package.id,
- self.user.id,
- 'component',
- override=True
- )
-
- self.import_event.refresh_from_db()
- self.assertEqual(self.import_event.status, ImportStatus.IMPORTED)
-
- for xblock in expected_imported_xblocks:
- self._is_imported(self.library, xblock)
-
- library_learning_package.refresh_from_db()
- self.assertEqual(library_learning_package.content_set.count(), len(expected_imported_xblocks))
- self.assertEqual(self.import_event.publishableentityimport_set.count(), len(expected_imported_xblocks))
-
- @patch('cms.djangoapps.import_from_modulestore.tasks.import_from_staged_content')
- def test_import_library_block_not_found(self, mock_import_from_staged_content):
- """ Test that if a block is not found in the staged content, it is not imported. """
- non_existent_usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345']
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
- with self.allow_transaction_exception():
- with self.assertRaises(ObjectDoesNotExist):
- import_staged_content_to_library_task(
- non_existent_usage_ids,
- str(self.import_event.uuid),
- self.content_library.learning_package.id,
- self.user.id,
- 'component',
- override=True,
- )
- mock_import_from_staged_content.assert_not_called()
-
- def test_cannot_import_staged_content_twice(self):
- """
- Tests if after importing staged content into the library,
- the staged content is deleted and cannot be imported again.
- """
- chapters_to_import = [self.chapter, self.chapter2]
- expected_imported_xblocks = [self.problem, self.video]
- save_legacy_content_to_staged_content_task(self.import_event.uuid)
-
- self.import_event.refresh_from_db()
- self.assertEqual(self.import_event.staged_content_for_import.count(), len(chapters_to_import))
- self.assertEqual(self.import_event.status, ImportStatus.STAGED)
-
- import_staged_content_to_library_task(
- [str(self.chapter.location)],
- str(self.import_event.uuid),
- self.content_library.learning_package.id,
- self.user.id,
- 'component',
- override=True,
- )
-
- for xblock in expected_imported_xblocks:
- self._is_imported(self.library, xblock)
-
- library_learning_package = LearningPackage.objects.get(id=self.library.learning_package_id)
- self.assertEqual(library_learning_package.content_set.count(), len(expected_imported_xblocks))
-
- self.import_event.refresh_from_db()
- self.assertEqual(self.import_event.status, ImportStatus.IMPORTED)
- self.assertTrue(not self.import_event.staged_content_for_import.exists())
- self.assertEqual(self.import_event.publishableentityimport_set.count(), len(expected_imported_xblocks))
diff --git a/cms/djangoapps/import_from_modulestore/tests/test_validators.py b/cms/djangoapps/import_from_modulestore/tests/test_validators.py
deleted file mode 100644
index 961d1cc81ba7..000000000000
--- a/cms/djangoapps/import_from_modulestore/tests/test_validators.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Tests for import_from_modulestore validators
-"""
-
-from typing import get_args
-
-from django.test import TestCase
-import pytest
-
-from cms.djangoapps.import_from_modulestore.validators import (
- validate_composition_level,
-)
-from cms.djangoapps.import_from_modulestore.data import CompositionLevel
-
-
-class TestValidateCompositionLevel(TestCase):
- """
- Test cases for validate_composition_level function.
- Case 1: Valid composition level
- Case 2: Invalid composition level
- """
-
- def test_valid_composition_level(self):
- for level in get_args(CompositionLevel):
- # Should not raise an exception for valid levels
- validate_composition_level(level)
-
- def test_invalid_composition_level(self):
- with pytest.raises(ValueError) as exc:
- validate_composition_level('invalid_composition_level')
- assert 'Invalid composition level: invalid_composition_level' in str(exc.value)
diff --git a/cms/djangoapps/import_from_modulestore/validators.py b/cms/djangoapps/import_from_modulestore/validators.py
deleted file mode 100644
index ecf0c72e08e8..000000000000
--- a/cms/djangoapps/import_from_modulestore/validators.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""
-Validators for the import_from_modulestore app.
-"""
-from typing import Sequence
-
-from opaque_keys.edx.keys import UsageKey
-
-from .data import CompositionLevel
-
-
-def validate_usage_keys_to_import(usage_keys: Sequence[str | UsageKey]):
- """
- Validate the usage keys to import.
-
- Currently, supports importing from the modulestore only by chapters.
- """
- for usage_key in usage_keys:
- if isinstance(usage_key, str):
- usage_key = UsageKey.from_string(usage_key)
- if usage_key.block_type != 'chapter':
- raise ValueError(f'Importing from modulestore only supports chapters, not {usage_key.block_type}')
-
-
-def validate_composition_level(composition_level):
- if composition_level not in CompositionLevel.values():
- raise ValueError(f'Invalid composition level: {composition_level}')
diff --git a/cms/djangoapps/import_from_modulestore/__init__.py b/cms/djangoapps/modulestore_migrator/__init__.py
similarity index 100%
rename from cms/djangoapps/import_from_modulestore/__init__.py
rename to cms/djangoapps/modulestore_migrator/__init__.py
diff --git a/cms/djangoapps/modulestore_migrator/admin.py b/cms/djangoapps/modulestore_migrator/admin.py
new file mode 100644
index 000000000000..36e2acca4e83
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/admin.py
@@ -0,0 +1,185 @@
+"""
+A nice little admin interface for migrating courses and libraries from modulstore to Learning Core.
+"""
+import logging
+
+from django import forms
+from django.contrib import admin, messages
+from django.contrib.admin.helpers import ActionForm
+from django.db import models
+
+
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.locator import LibraryCollectionLocator, LibraryLocatorV2
+from user_tasks.models import UserTaskStatus
+
+from openedx.core.types.http import AuthenticatedHttpRequest
+
+from . import api
+from .data import CompositionLevel
+from .models import ModulestoreSource, ModulestoreMigration, ModulestoreBlockSource, ModulestoreBlockMigration
+
+
+log = logging.getLogger(__name__)
+
+
+class StartMigrationTaskForm(ActionForm):
+ """
+ Params for start_migration_task admin adtion, displayed next the "Go" button.
+ """
+ target_key = forms.CharField(label="Target library or collection key →", required=False)
+ replace_existing = forms.BooleanField(label="Replace existing content? →", required=False)
+ forward_to_target = forms.BooleanField(label="Forward references? →", required=False)
+ composition_level = forms.ChoiceField(
+ label="Aggregate up to →", choices=CompositionLevel.supported_choices, required=False
+ )
+
+
+def task_status_details(obj: ModulestoreMigration) -> str:
+ """
+ Return the state and, if available, details of the status of the migration.
+ """
+ details: str | None = None
+ if obj.task_status.state == UserTaskStatus.FAILED:
+ # Calling fail(msg) from a task should automatically generates an "Error" artifact with that msg.
+ # https://django-user-tasks.readthedocs.io/en/latest/user_tasks.html#user_tasks.models.UserTaskStatus.fail
+ if error_artifacts := obj.task_status.artifacts.filter(name="Error"):
+ if error_text := error_artifacts.order_by("-created").first().text:
+ details = error_text
+ elif obj.task_status.state == UserTaskStatus.SUCCEEDED:
+ details = f"Migrated {obj.block_migrations.count()} blocks"
+ return f"{obj.task_status.state}: {details}" if details else obj.task_status.state
+
+
+migration_admin_fields = (
+ "target",
+ "target_collection",
+ "task_status",
+ # The next line works, but django-stubs incorrectly thinks that these should all be strings,
+ # so we will need to use type:ignore below.
+ task_status_details,
+ "composition_level",
+ "replace_existing",
+ "change_log",
+ "staged_content",
+)
+
+
+class ModulestoreMigrationInline(admin.TabularInline):
+ """
+ Readonly table within the ModulestoreSource page; each row is a Migration from this Source.
+ """
+ model = ModulestoreMigration
+ fk_name = "source"
+ show_change_link = True
+ readonly_fields = migration_admin_fields # type: ignore[assignment]
+ ordering = ("-task_status__created",)
+
+ def has_add_permission(self, _request, _obj):
+ return False
+
+
+class ModulestoreBlockSourceInline(admin.TabularInline):
+ """
+ Readonly table within the ModulestoreSource page; each row is a BlockSource.
+ """
+ model = ModulestoreBlockSource
+ fk_name = "overall_source"
+ readonly_fields = (
+ "key",
+ "forwarded"
+ )
+
+ def has_add_permission(self, _request, _obj):
+ return False
+
+
+@admin.register(ModulestoreSource)
+class ModulestoreSourceAdmin(admin.ModelAdmin):
+ """
+ Admin interface for source legacy libraries and courses.
+ """
+ readonly_fields = ("forwarded",)
+ list_display = ("id", "key", "forwarded")
+ actions = ["start_migration_task"]
+ action_form = StartMigrationTaskForm
+ inlines = [ModulestoreMigrationInline, ModulestoreBlockSourceInline]
+
+ @admin.action(description="Start migration for selected sources")
+ def start_migration_task(
+ self,
+ request: AuthenticatedHttpRequest,
+ queryset: models.QuerySet[ModulestoreSource],
+ ) -> None:
+ """
+ Start a migration for each selected source
+ """
+ form = StartMigrationTaskForm(request.POST)
+ form.is_valid()
+ target_key_string = form.cleaned_data['target_key']
+ if not target_key_string:
+ messages.add_message(request, messages.ERROR, "Target key is required")
+ return
+ try:
+ target_library_key = LibraryLocatorV2.from_string(target_key_string)
+ target_collection_slug = None
+ except InvalidKeyError:
+ try:
+ target_collection_key = LibraryCollectionLocator.from_string(target_key_string)
+ target_library_key = target_collection_key.lib_key
+ target_collection_slug = target_collection_key.collection_id
+ except InvalidKeyError:
+ messages.add_message(request, messages.ERROR, f"Invalid target key: {target_key_string}")
+ return
+ started = 0
+ total = 0
+ for source in queryset:
+ total += 1
+ try:
+ api.start_migration_to_library(
+ user=request.user,
+ source_key=source.key,
+ target_library_key=target_library_key,
+ target_collection_slug=target_collection_slug,
+ composition_level=form.cleaned_data['composition_level'],
+ replace_existing=form.cleaned_data['replace_existing'],
+ forward_source_to_target=form.cleaned_data['forward_to_target'],
+ )
+ except Exception as exc: # pylint: disable=broad-except
+ message = f"Failed to start migration {source.key} -> {target_key_string}"
+ messages.add_message(request, messages.ERROR, f"{message}: {exc}")
+ log.exception(message)
+ continue
+ started += 1
+ click_in = "Click into the source objects to see migration details."
+
+ if not started:
+ messages.add_message(request, messages.WARNING, f"Failed to start {total} migration(s).")
+ if started < total:
+ messages.add_message(request, messages.WARNING, f"Started {started} of {total} migration(s). {click_in}")
+ else:
+ messages.add_message(request, messages.INFO, f"Started {started} migration(s). {click_in}")
+
+
+class ModulestoreBlockMigrationInline(admin.TabularInline):
+ """
+ Readonly table witin the Migration admin; each row is a block
+ """
+ model = ModulestoreBlockMigration
+ fk_name = "overall_migration"
+ readonly_fields = (
+ "source",
+ "target",
+ "change_log_record",
+ )
+ list_display = ("id", *readonly_fields)
+
+
+@admin.register(ModulestoreMigration)
+class ModulestoreMigrationAdmin(admin.ModelAdmin):
+ """
+ Readonly admin page for viewing Migrations
+ """
+ readonly_fields = ("source", *migration_admin_fields) # type: ignore[assignment]
+ list_display = ("id", "source", *migration_admin_fields) # type: ignore[assignment]
+ inlines = [ModulestoreBlockMigrationInline]
diff --git a/cms/djangoapps/modulestore_migrator/api.py b/cms/djangoapps/modulestore_migrator/api.py
new file mode 100644
index 000000000000..351fc79939e4
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/api.py
@@ -0,0 +1,52 @@
+"""
+API for migration from modulestore to learning core
+"""
+from opaque_keys.edx.locator import LibraryLocatorV2
+from opaque_keys.edx.keys import LearningContextKey
+from openedx_learning.api.authoring import get_collection
+
+from openedx.core.djangoapps.content_libraries.api import get_library
+from openedx.core.types.user import AuthUser
+
+from . import tasks
+from .data import CompositionLevel
+from .models import ModulestoreSource
+
+
+__all__ = (
+ "start_migration_to_library",
+)
+
+
+def start_migration_to_library(
+ *,
+ user: AuthUser,
+ source_key: LearningContextKey,
+ target_library_key: LibraryLocatorV2,
+ target_collection_slug: str | None = None,
+ composition_level: CompositionLevel,
+ replace_existing: bool,
+ forward_source_to_target: bool,
+) -> None:
+ """
+ Import a course or legacy library into a V2 library (or, a collection within a V2 library).
+ """
+ source, _ = ModulestoreSource.objects.get_or_create(key=source_key)
+ target_library = get_library(target_library_key)
+ if not (target_package_id := target_library.learning_package_id):
+ raise ValueError(
+ f"Cannot import {source_key} into library at {target_library_key} because the "
+ "library is not connected to a learning package"
+ )
+ target_collection_id = None
+ if target_collection_slug:
+ target_collection_id = get_collection(target_package_id, target_collection_slug).id
+ tasks.migrate_from_modulestore.delay(
+ user_id=user.id,
+ source_pk=source.id,
+ target_package_pk=target_package_id,
+ target_collection_pk=target_collection_id,
+ composition_level=composition_level,
+ replace_existing=replace_existing,
+ forward_source_to_target=forward_source_to_target,
+ )
diff --git a/cms/djangoapps/modulestore_migrator/apps.py b/cms/djangoapps/modulestore_migrator/apps.py
new file mode 100644
index 000000000000..37c2b66d6f27
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/apps.py
@@ -0,0 +1,13 @@
+"""
+App configurations
+"""
+
+from django.apps import AppConfig
+
+
+class ModulestoreMigratorConfig(AppConfig):
+ """
+ App for importing legacy content from the modulestore.
+ """
+
+ name = 'cms.djangoapps.modulestore_migrator'
diff --git a/cms/djangoapps/modulestore_migrator/constants.py b/cms/djangoapps/modulestore_migrator/constants.py
new file mode 100644
index 000000000000..34ce788504b9
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/constants.py
@@ -0,0 +1,7 @@
+"""
+Constants
+"""
+
+CONTENT_STAGING_PURPOSE = "modulestore_migrator"
+CONTENT_STAGING_PURPOSE_META = "modulestore_migrator_meta"
+META_BLOCK_TYPES: list[str] = ["about", "course_info", "static_tab"]
\ No newline at end of file
diff --git a/cms/djangoapps/modulestore_migrator/data.py b/cms/djangoapps/modulestore_migrator/data.py
new file mode 100644
index 000000000000..3b35b07c8937
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/data.py
@@ -0,0 +1,53 @@
+"""
+Value objects
+"""
+from __future__ import annotations
+
+from enum import Enum
+
+from openedx.core.djangoapps.content_libraries.api import ContainerType
+
+
+class CompositionLevel(Enum):
+ """
+ Enumeration of composition levels for legacy content.
+
+ Defined in increasing order of complexity so that `is_higher_than` works correctly.
+ """
+ # Components are individual XBlocks, e.g. Problem
+ Component = 'component'
+
+ # Container types currently supported by Content Libraries
+ Unit = ContainerType.Unit.value
+ Subsection = ContainerType.Subsection.value
+ Section = ContainerType.Section.value
+ OutlineRoot = ContainerType.OutlineRoot.value
+
+ # Import the outline root, as well as the weird meta blocks (about,
+ # course_info, static_tab) that exist as parent-less peers of the outline
+ # root, and get/create the Course instance. Unlike the other
+ # CompositionLevels, this level does not correspond to any particular kind of
+ # publishable entity.
+ CourseRun = "course_run"
+
+ @property
+ def is_complex(self) -> bool:
+ return self is not self.Component
+
+ def is_higher_than(self, other: 'CompositionLevel') -> bool:
+ """
+ Is this composition level 'above' (more complex than) the other?
+ """
+ levels: list[CompositionLevel] = list(self.__class__)
+ return levels.index(self) > levels.index(other)
+
+ @classmethod
+ def supported_choices(cls) -> list[tuple[str, str]]:
+ """
+ Returns all supported composition levels as a list of tuples,
+ for use in a Django Models ChoiceField.
+ """
+ return [
+ (composition_level.value, composition_level.name)
+ for composition_level in cls
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0001_initial.py b/cms/djangoapps/modulestore_migrator/migrations/0001_initial.py
new file mode 100644
index 000000000000..4d9bc62de01b
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0001_initial.py
@@ -0,0 +1,106 @@
+# Generated by Django 4.2.22 on 2025-06-10 04:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import model_utils.fields
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('oel_collections', '0005_alter_collection_options_alter_collection_enabled'),
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ('content_staging', '0005_stagedcontent_version_num'),
+ ('user_tasks', '0004_url_textfield'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='ModulestoreBlockMigration',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+ ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+ ('change_log_record', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelogrecord')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='ModulestoreMigration',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('composition_level', models.CharField(choices=[('component', 'Component'), ('unit', 'Unit'), ('subsection', 'Subsection'), ('section', 'Section')], default='component', help_text='Maximum hierachy level at which content should be aggregated in target library', max_length=255)),
+ ('replace_existing', models.BooleanField(default=False, help_text='If a piece of content already exists in the content library, should the import process replace it?')),
+ ('change_log', models.ForeignKey(help_text='Changelog entry in the target learning package which records this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelog')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='ModulestoreSource',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', opaque_keys.edx.django.models.LearningContextKeyField(help_text='Key of the content source (a course or a legacy library)', max_length=255, unique=True)),
+ ('forwarded_by', models.ForeignKey(help_text='If set, the system will forward references of this source over to the target of this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoremigration')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='source',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='migrations', to='modulestore_migrator.modulestoresource'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='staged_content',
+ field=models.OneToOneField(help_text='Modulestore content is processed and staged before importing it to a learning packge. We temporarily save the staged content to allow for troubleshooting of failed migrations.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='content_staging.stagedcontent'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='target',
+ field=models.ForeignKey(help_text='Content will be imported into this library', on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='target_collection',
+ field=models.ForeignKey(blank=True, help_text='Optional - Collection (within the target library) into which imported content will be grouped', null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_collections.collection'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='task_status',
+ field=models.OneToOneField(help_text='Tracks the status of the task which is executing this migration', on_delete=django.db.models.deletion.RESTRICT, to='user_tasks.usertaskstatus'),
+ ),
+ migrations.CreateModel(
+ name='ModulestoreBlockSource',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+ ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+ ('key', opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key of the XBlock that has been imported.', max_length=255)),
+ ('forwarded_by', models.ForeignKey(help_text='If set, the system will forward references of this block source over to the target of this block migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoreblockmigration')),
+ ('overall_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blocks', to='modulestore_migrator.modulestoresource')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='overall_migration',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='block_migrations', to='modulestore_migrator.modulestoremigration'),
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='source',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='modulestore_migrator.modulestoreblocksource'),
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='target',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishableentity'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='modulestoreblockmigration',
+ unique_together={('overall_migration', 'target'), ('overall_migration', 'source')},
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0002_source_forwardedby_blank.py b/cms/djangoapps/modulestore_migrator/migrations/0002_source_forwardedby_blank.py
new file mode 100644
index 000000000000..4d2d802ca788
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0002_source_forwardedby_blank.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.22 on 2025-06-11 15:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('modulestore_migrator', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='modulestoresource',
+ name='forwarded_by',
+ field=models.ForeignKey(blank=True, help_text='If set, the system will forward references of this source over to the target of this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoremigration'),
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0003_remove_everything.py b/cms/djangoapps/modulestore_migrator/migrations/0003_remove_everything.py
new file mode 100644
index 000000000000..3cbeed703ae5
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0003_remove_everything.py
@@ -0,0 +1,61 @@
+# Generated by Django 4.2.22 on 2025-06-19 18:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('modulestore_migrator', '0002_source_forwardedby_blank'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='modulestoreblocksource',
+ name='forwarded_by',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoreblocksource',
+ name='overall_source',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='change_log',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='source',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='staged_content',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='target',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='target_collection',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoremigration',
+ name='task_status',
+ ),
+ migrations.RemoveField(
+ model_name='modulestoresource',
+ name='forwarded_by',
+ ),
+ migrations.DeleteModel(
+ name='ModulestoreBlockMigration',
+ ),
+ migrations.DeleteModel(
+ name='ModulestoreBlockSource',
+ ),
+ migrations.DeleteModel(
+ name='ModulestoreMigration',
+ ),
+ migrations.DeleteModel(
+ name='ModulestoreSource',
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0004_initial.py b/cms/djangoapps/modulestore_migrator/migrations/0004_initial.py
new file mode 100644
index 000000000000..fb1537bf8016
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0004_initial.py
@@ -0,0 +1,107 @@
+# Generated by Django 4.2.22 on 2025-06-19 18:46
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import model_utils.fields
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('oel_collections', '0005_alter_collection_options_alter_collection_enabled'),
+ ('content_staging', '0005_stagedcontent_version_num'),
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ('user_tasks', '0004_url_textfield'),
+ ('modulestore_migrator', '0003_remove_everything'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='ModulestoreBlockMigration',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+ ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+ ('change_log_record', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelogrecord')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='ModulestoreMigration',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('composition_level', models.CharField(choices=[('component', 'Component'), ('unit', 'Unit'), ('subsection', 'Subsection'), ('section', 'Section'), ('outline_root', 'OutlineRoot')], default='component', help_text='Maximum hierachy level at which content should be aggregated in target library', max_length=255)),
+ ('replace_existing', models.BooleanField(default=False, help_text='If a piece of content already exists in the content library, should the import process replace it?')),
+ ('change_log', models.ForeignKey(help_text='Changelog entry in the target learning package which records this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.draftchangelog')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='ModulestoreSource',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', opaque_keys.edx.django.models.LearningContextKeyField(help_text='Key of the content source (a course or a legacy library)', max_length=255, unique=True)),
+ ('forwarded', models.OneToOneField(blank=True, help_text='If set, the system will forward references of this source over to the target of this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='forwards', to='modulestore_migrator.modulestoremigration')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='source',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='migrations', to='modulestore_migrator.modulestoresource'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='staged_content',
+ field=models.OneToOneField(help_text='Modulestore content is processed and staged before importing it to a learning packge. We temporarily save the staged content to allow for troubleshooting of failed migrations.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='content_staging.stagedcontent'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='target',
+ field=models.ForeignKey(help_text='Content will be imported into this library', on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='target_collection',
+ field=models.ForeignKey(blank=True, help_text='Optional - Collection (within the target library) into which imported content will be grouped', null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_collections.collection'),
+ ),
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='task_status',
+ field=models.OneToOneField(help_text='Tracks the status of the task which is executing this migration', on_delete=django.db.models.deletion.RESTRICT, to='user_tasks.usertaskstatus'),
+ ),
+ migrations.CreateModel(
+ name='ModulestoreBlockSource',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+ ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+ ('key', opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key of the XBlock that has been imported.', max_length=255)),
+ ('forwarded', models.OneToOneField(help_text='If set, the system will forward references of this block source over to the target of this block migration', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='forwards', to='modulestore_migrator.modulestoreblockmigration')),
+ ('overall_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blocks', to='modulestore_migrator.modulestoresource')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='overall_migration',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='block_migrations', to='modulestore_migrator.modulestoremigration'),
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='source',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='modulestore_migrator.modulestoreblocksource'),
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='target',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishableentity'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='modulestoreblockmigration',
+ unique_together={('overall_migration', 'source'), ('overall_migration', 'target')},
+ ),
+ ]
diff --git a/cms/djangoapps/import_from_modulestore/migrations/__init__.py b/cms/djangoapps/modulestore_migrator/migrations/__init__.py
similarity index 100%
rename from cms/djangoapps/import_from_modulestore/migrations/__init__.py
rename to cms/djangoapps/modulestore_migrator/migrations/__init__.py
diff --git a/cms/djangoapps/modulestore_migrator/models.py b/cms/djangoapps/modulestore_migrator/models.py
new file mode 100644
index 000000000000..576dfa4f2e82
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/models.py
@@ -0,0 +1,210 @@
+"""
+Models for the modulestore migration tool.
+"""
+from __future__ import annotations
+
+from django.contrib.auth import get_user_model
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+from user_tasks.models import UserTaskStatus
+
+from model_utils.models import TimeStampedModel
+from opaque_keys.edx.django.models import (
+ LearningContextKeyField,
+ UsageKeyField,
+)
+from openedx_learning.api.authoring_models import (
+ LearningPackage, PublishableEntity, Collection, DraftChangeLog, DraftChangeLogRecord
+)
+
+from openedx.core.djangoapps.content_staging.models import StagedContent
+from .data import CompositionLevel
+
+User = get_user_model()
+
+
+class ModulestoreSource(models.Model):
+ """
+ A legacy learning context (course or library) which can be a source of a migration.
+ """
+ key = LearningContextKeyField(
+ max_length=255,
+ unique=True,
+ help_text=_('Key of the content source (a course or a legacy library)'),
+ )
+ forwarded = models.OneToOneField(
+ 'modulestore_migrator.ModulestoreMigration',
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ help_text=_('If set, the system will forward references of this source over to the target of this migration'),
+ related_name="forwards",
+ )
+
+ def __str__(self):
+ return f"{self.__class__.__name__}('{self.key}')"
+
+ __repr__ = __str__
+
+
+class ModulestoreMigration(models.Model):
+ """
+ Tracks the action of a user importing a Modulestore-based course or legacy library into a
+ learning-core based learning package
+
+ Notes:
+ * As of Ulmo, a learning package is always associated with a v2 content library, but we
+ will not bake that assumption into this model)
+ * Each Migration is tied to a single UserTaskStatus, which connects it to a user and
+ contains the progress of the import.
+ * A single ModulestoreSource may very well have multiple ModulestoreMigrations; however,
+ at most one of them with be the "authoritative" migration, as indicated by `forwarded`.
+ """
+
+ ## MIGRATION SPECIFICATION
+ source = models.ForeignKey(
+ ModulestoreSource,
+ on_delete=models.CASCADE,
+ related_name="migrations",
+ )
+ composition_level = models.CharField(
+ max_length=255,
+ choices=CompositionLevel.supported_choices(),
+ default=CompositionLevel.Component.value,
+ help_text=_('Maximum hierachy level at which content should be aggregated in target library'),
+ )
+ replace_existing = models.BooleanField(
+ default=False,
+ help_text=_(
+ "If a piece of content already exists in the content library, should the import process replace it?"
+ ),
+ )
+ target = models.ForeignKey(
+ LearningPackage,
+ on_delete=models.CASCADE,
+ help_text=_('Content will be imported into this library'),
+ )
+ target_collection = models.ForeignKey(
+ Collection,
+ on_delete=models.SET_NULL,
+ null=True,
+ blank=True,
+ help_text=_('Optional - Collection (within the target library) into which imported content will be grouped'),
+ )
+
+ ## MIGRATION ARTIFACTS
+ task_status = models.OneToOneField(
+ UserTaskStatus,
+ on_delete=models.RESTRICT,
+ help_text=_("Tracks the status of the task which is executing this migration"),
+ )
+ change_log = models.ForeignKey(
+ DraftChangeLog,
+ on_delete=models.SET_NULL,
+ null=True,
+ help_text=_("Changelog entry in the target learning package which records this migration"),
+ )
+ staged_content = models.OneToOneField(
+ StagedContent,
+ null=True,
+ on_delete=models.SET_NULL, # Staged content is liable to be deleted in order to save space
+ help_text=_(
+ "Modulestore content is processed and staged before importing it to a learning packge. "
+ "We temporarily save the staged content to allow for troubleshooting of failed migrations."
+ )
+ )
+
+ def __str__(self):
+ return (
+ f"{self.__class__.__name__} #{self.pk}: "
+ f"{self.source.key} → {self.target_collection or self.target}"
+ )
+
+ def __repr__(self):
+ return (
+ f"{self.__class__.__name__}("
+ f"id={self.id}, source='{self.source}',"
+ f"target='{self.target_collection or self.target}')"
+ )
+
+
+class ModulestoreBlockSource(TimeStampedModel):
+ """
+ A legacy block usage (in a course or library) which can be a source of a block migration.
+ """
+ overall_source = models.ForeignKey(
+ ModulestoreSource,
+ on_delete=models.CASCADE,
+ related_name="blocks",
+ )
+ key = UsageKeyField(
+ max_length=255,
+ help_text=_('Original usage key of the XBlock that has been imported.'),
+ )
+ forwarded = models.OneToOneField(
+ 'modulestore_migrator.ModulestoreBlockMigration',
+ null=True,
+ on_delete=models.SET_NULL,
+ help_text=_(
+ 'If set, the system will forward references of this block source over to the target of this block migration'
+ ),
+ related_name="forwards",
+ )
+ unique_together = [("overall_source", "key")]
+
+ def __str__(self):
+ return f"{self.__class__.__name__}('{self.key}')"
+
+ __repr__ = __str__
+
+
+class ModulestoreBlockMigration(TimeStampedModel):
+ """
+ The migration of a single legacy block into a learning package.
+
+ Is always tied to a greater overall ModulestoreMigration.
+
+ Note:
+ * A single ModulestoreBlockSource may very well have multiple ModulestoreBlockMigrations; however,
+ at most one of them with be the "authoritative" migration, as indicated by `forwarded`.
+ This will coincide with the `overall_migration` being pointed to by `forwarded` as well.
+ """
+ overall_migration = models.ForeignKey(
+ ModulestoreMigration,
+ on_delete=models.CASCADE,
+ related_name="block_migrations",
+ )
+ source = models.ForeignKey(
+ ModulestoreBlockSource,
+ on_delete=models.CASCADE,
+ )
+ target = models.ForeignKey(
+ PublishableEntity,
+ on_delete=models.CASCADE,
+ )
+ change_log_record = models.OneToOneField(
+ DraftChangeLogRecord,
+ # a changelog record can be pruned, which would set this to NULL, but not delete the
+ # entire import record
+ null=True,
+ on_delete=models.SET_NULL,
+ )
+
+ class Meta:
+ unique_together = [
+ ('overall_migration', 'source'),
+ ('overall_migration', 'target'),
+ ]
+
+ def __str__(self):
+ return (
+ f"{self.__class__.__name__} #{self.pk}: "
+ f"{self.source.key} → {self.target}"
+ )
+
+ def __repr__(self):
+ return (
+ f"{self.__class__.__name__}("
+ f"id={self.id}, source='{self.source}',"
+ f"target='{self.target}')"
+ )
diff --git a/cms/djangoapps/import_from_modulestore/tests/__init__.py b/cms/djangoapps/modulestore_migrator/rest_api/__init__.py
similarity index 100%
rename from cms/djangoapps/import_from_modulestore/tests/__init__.py
rename to cms/djangoapps/modulestore_migrator/rest_api/__init__.py
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/urls.py
new file mode 100644
index 000000000000..b993a5952ce0
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/rest_api/urls.py
@@ -0,0 +1,13 @@
+"""
+Course to Library Import API URLs.
+"""
+
+from django.urls import include, path
+
+from .v0 import urls as v0_urls
+
+app_name = 'modulestore_migrator'
+
+urlpatterns = [
+ path('v0/', include(v0_urls)),
+]
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v0/__init__.py b/cms/djangoapps/modulestore_migrator/rest_api/v0/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v0/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/v0/urls.py
new file mode 100644
index 000000000000..eda8e0e3d43e
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/rest_api/v0/urls.py
@@ -0,0 +1,11 @@
+"""
+Course to Library Import API v0 URLs.
+"""
+
+from rest_framework.routers import SimpleRouter
+# from .views import ImportViewSet # @@TODO - re-anble this once API is fixed
+
+ROUTER = SimpleRouter()
+# ROUTER.register(r'imports', ImportViewSet)
+
+urlpatterns = ROUTER.urls
diff --git a/cms/djangoapps/modulestore_migrator/tasks.py b/cms/djangoapps/modulestore_migrator/tasks.py
new file mode 100644
index 000000000000..124fc2cbc216
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/tasks.py
@@ -0,0 +1,676 @@
+"""
+Tasks for the modulestore_migrator
+"""
+from __future__ import annotations
+
+import mimetypes
+import os
+import typing as t
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from enum import Enum
+
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from django.db import IntegrityError
+from django.core.exceptions import ObjectDoesNotExist
+from edx_django_utils.monitoring import set_code_owner_attribute_from_module
+from lxml import etree
+from lxml.etree import _ElementTree as XmlTree
+from opaque_keys.edx.keys import CourseKey, UsageKey
+from opaque_keys.edx.locator import (
+ CourseLocator, LibraryLocator,
+ LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator
+)
+from openedx_learning.api import authoring as authoring_api
+from openedx_learning.api.authoring_models import (
+ Collection,
+ Component,
+ Course,
+ CatalogCourse,
+ LearningPackage,
+ PublishableEntity,
+ PublishableEntityVersion,
+)
+from user_tasks.tasks import UserTask, UserTaskStatus
+from xblock.core import XBlock
+
+from openedx.core.djangoapps.content_libraries.api import ContainerType
+from openedx.core.djangoapps.content_libraries import api as libraries_api
+from openedx.core.djangoapps.content_libraries.models import ContentLibrary
+from openedx.core.djangoapps.content_staging import api as staging_api
+from openedx.core.djangoapps.xblock import models as xblock_models
+from openedx.core.djangoapps.xblock.api import create_xblock_field_data_for_container
+
+from xmodule.modulestore import exceptions as modulestore_exceptions
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.mixed import MixedModuleStore
+
+from .constants import CONTENT_STAGING_PURPOSE, CONTENT_STAGING_PURPOSE_META, META_BLOCK_TYPES
+from .data import CompositionLevel
+from .models import ModulestoreSource, ModulestoreMigration, ModulestoreBlockSource, ModulestoreBlockMigration
+
+
+log = get_task_logger(__name__)
+
+
+class MigrationStep(Enum):
+ """
+ Strings representation the state of an in-progress modulestore-to-learning-core import.
+
+ We use these values to set UserTaskStatus.state.
+ The other possible UserTaskStatus.state values are the built-in ones:
+ UserTaskStatus.{PENDING,FAILED,CANCELED,SUCCEEDED}.
+ """
+ VALIDATING_INPUT = 'Validating migration parameters'
+ CANCELLING_OLD = 'Cancelling any redundant migration tasks'
+ LOADING = 'Loading legacy content from ModulesStore'
+ STAGING = 'Staging legacy content for import'
+ PARSING = 'Parsing staged OLX'
+ IMPORTING_ASSETS = 'Importing staged files and resources'
+ IMPORTING_STRUCTURE = 'Importing staged content structure'
+ IMPORTING_META = 'Importing course info and other meta-components'
+ UNSTAGING = 'Cleaning staged content'
+ MAPPING_OLD_TO_NEW = 'Saving map of legacy content to migrated content'
+ FORWARDING = 'Forwarding legacy content to migrated content'
+ POPULATING_COLLECTION = 'Assigning imported items to the specified collection'
+
+
+class _MigrationTask(UserTask):
+ """
+ Base class for migrate_to_modulestore
+ """
+
+ @staticmethod
+ def calculate_total_steps(arguments_dict):
+ """
+ Get number of in-progress steps in importing process, as shown in the UI.
+ """
+ return len(list(MigrationStep))
+
+
+@shared_task(base=_MigrationTask, bind=True)
+# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin
+# does stack inspection and can't handle additional decorators.
+def migrate_from_modulestore(
+ self: _MigrationTask,
+ *,
+ user_id: int,
+ source_pk: int,
+ target_package_pk: int,
+ target_collection_pk: int,
+ replace_existing: bool,
+ composition_level: str,
+ forward_source_to_target: bool,
+) -> None:
+ """
+ Import a course or legacy library into a learning package.
+
+ Currently, the target learning package must be associated with a V2 content library, but that
+ restriction may be loosened in the future as more types of learning packages are developed.
+ """
+ # pylint: disable=too-many-statements
+ # This is a large function, but breaking it up futher would probably not
+ # make it any easier to understand.
+
+ set_code_owner_attribute_from_module(__name__)
+
+ status: UserTaskStatus = self.status
+ status.set_state(MigrationStep.VALIDATING_INPUT.value)
+ comp_level = CompositionLevel(composition_level)
+ try:
+ source = ModulestoreSource.objects.get(pk=source_pk)
+ target_package = LearningPackage.objects.get(pk=target_package_pk)
+ target_library = ContentLibrary.objects.get(learning_package_id=target_package_pk)
+ target_collection = Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None
+ except ObjectDoesNotExist as exc:
+ status.fail(str(exc))
+ return
+
+ course_lc_learning_context = None
+ if isinstance(source.key, CourseLocator):
+ source_root_usage_key = source.key.make_usage_key('course', 'course')
+
+ # Support SplitModuleStore shim from Learning Core, force it off for now because we need to build it using ModuleStore
+ course_lc_learning_context, _created = xblock_models.LearningCoreLearningContext.objects.get_or_create(key=source.key)
+ course_lc_learning_context.use_learning_core = False
+ course_lc_learning_context.save()
+
+ elif isinstance(source.key, LibraryLocator):
+ source_root_usage_key = source.key.make_usage_key('library', 'library')
+ else:
+ status.fail(
+ f"Not a valid source context key: {source.key}. "
+ "Source key must reference a course or a legacy library."
+ )
+ return
+
+ migration = ModulestoreMigration.objects.create(
+ source=source,
+ composition_level=composition_level,
+ replace_existing=replace_existing,
+ target=target_package,
+ target_collection=target_collection,
+ task_status=status,
+ )
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.CANCELLING_OLD.value)
+ # In order to prevent a user from accidentally starting a bunch of identical import tasks...
+ migrations_to_cancel = ModulestoreMigration.objects.filter(
+ # get all Migration tasks by this user with the same source and target
+ task_status__user=status.user,
+ source=source,
+ target=target_package,
+ ).select_related('task_status').exclude(
+ # (excluding that aren't running)
+ task_status__state__in=(UserTaskStatus.CANCELED, UserTaskStatus.FAILED, UserTaskStatus.SUCCEEDED)
+ ).exclude(
+ # (excluding this migration itself)
+ id=migration.id
+ )
+ # ... and cancel their tasks and clean away their staged content.
+ for migration_to_cancel in migrations_to_cancel:
+ if migration_to_cancel.task_status:
+ migration_to_cancel.task_status.cancel()
+ if migration_to_cancel.staged_content:
+ migration_to_cancel.staged_content.delete()
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.LOADING)
+ store: MixedModuleStore = modulestore()
+ try:
+ legacy_root = store.get_item(source_root_usage_key)
+ except modulestore_exceptions.ItemNotFoundError as exc:
+ status.fail(f"Failed to load source item '{source_root_usage_key}' from ModuleStore: {exc}")
+ return
+ if not legacy_root:
+ status.fail(f"Could not find source item '{source_root_usage_key}' in ModuleStore")
+ return
+ meta_blocks: list[XBlock] = (
+ store.get_items(source.key, qualifiers={"category": {"$in": META_BLOCK_TYPES}})
+ if comp_level == CompositionLevel.CourseRun
+ else []
+ )
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.STAGING.value)
+ staged_content = staging_api.stage_xblock_temporarily(
+ block=legacy_root,
+ user_id=status.user.pk,
+ purpose=CONTENT_STAGING_PURPOSE,
+ )
+ migration.staged_content = staged_content
+ staged_meta_contents = [
+ staging_api.stage_xblock_temporarily(
+ block=meta_block,
+ user_id=status.user.pk,
+ purpose=CONTENT_STAGING_PURPOSE_META,
+ )
+ for meta_block in meta_blocks
+ ]
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.PARSING.value)
+ parser = etree.XMLParser(strip_cdata=False)
+ try:
+ root_node = etree.fromstring(staged_content.olx, parser=parser)
+ except etree.ParseError as exc:
+ status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}")
+ return
+ meta_nodes = []
+ for staged_meta_content in staged_meta_contents:
+ meta_parser = etree.XMLParser(strip_cdata=False)
+ try:
+ meta_nodes.append(etree.fromstring(staged_meta_content.olx, parser=meta_parser))
+ except etree.ParseError as exc:
+ status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}")
+ return
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.IMPORTING_ASSETS.value)
+ content_by_filename: dict[str, int] = {}
+ now = datetime.now(tz=timezone.utc)
+ all_static_files: list[staging_api.StagedContentFileData] = [
+ static_file
+ for staged in [staged_content, *staged_meta_contents]
+ for static_file in staging_api.get_staged_content_static_files(staged.id)
+ ]
+ for staged_content_file_data in all_static_files:
+ old_path = staged_content_file_data.filename
+ file_data = staging_api.get_staged_content_static_file_data(staged_content.id, old_path)
+ if not file_data:
+ log.error(
+ f"Staged content {staged_content.id} included referenced file {old_path}, "
+ "but no file data was found."
+ )
+ continue
+ filename = os.path.basename(old_path)
+ media_type_str = mimetypes.guess_type(filename)[0] or "application/octet-stream"
+ media_type = authoring_api.get_or_create_media_type(media_type_str)
+ content_by_filename[filename] = authoring_api.get_or_create_file_content(
+ migration.target_id,
+ media_type.id,
+ data=file_data,
+ created=now,
+ ).id
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.IMPORTING_STRUCTURE.value)
+ now = datetime.now(timezone.utc)
+ with authoring_api.bulk_draft_changes_for(migration.target.id) as change_log:
+ root_migrated_node = _migrate_node(
+ content_by_filename=content_by_filename,
+ source_context_key=source.key,
+ source_node=root_node,
+ target_library_key=target_library.library_key,
+ target_package_id=target_package_pk,
+ replace_existing=replace_existing,
+ composition_level=comp_level,
+ created_at=now,
+ created_by=user_id,
+ )
+ migration.change_log = change_log
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.IMPORTING_META.value)
+ migrated_meta_nodes: list[_MigratedNode] = [
+ _migrate_node(
+ content_by_filename=content_by_filename,
+ source_context_key=source.key,
+ source_node=meta_node,
+ target_package_id=target_package_pk,
+ target_library_key=target_library.library_key,
+ replace_existing=replace_existing,
+ composition_level=comp_level,
+ created_at=now,
+ created_by=user_id,
+ )
+ for meta_node in meta_nodes
+ ]
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.UNSTAGING.value)
+ staged_content.delete()
+ status.increment_completed_steps()
+
+ # @@TODO We currently do the mapping in bulk in order to reduce the number of DB queries.
+ # But, as a user, it means that there are no artifacts (ModulestoreBlockMigrations)
+ # to be shown as the structure is being imported, which takes can take several minutes.
+ # Would be better to, instead, create each ModulestoreBlockSource and
+ # ModulestoreBlockMigration as we create its PublisahbleEntity(Version)? If
+ # we did this, we'd want to make sure that the objects are actually visible
+ # to the user mid-import (via django admin, or the library interface, or even just as
+ # as a "progress bar" field in the REST API), otherwise this would be pointless.
+ migrated_umbrella = _MigratedNode(
+ # This is a block-less pseudo-node representing an umbrella containing both
+ # (a) the outline and (b) all the meta blocks. @@TODO this might be too clever
+ # to leave in the production migrator... revisit.
+ source_to_target=None,
+ children=[root_migrated_node, *migrated_meta_nodes],
+ )
+ status.set_state(MigrationStep.MAPPING_OLD_TO_NEW.value)
+ block_source_keys_to_target_vers = dict(migrated_umbrella.all_source_to_target_pairs())
+ ModulestoreBlockSource.objects.bulk_create(
+ [
+ ModulestoreBlockSource(overall_source=source, key=source_usage_key)
+ for source_usage_key in block_source_keys_to_target_vers.keys()
+ ],
+ # Note: bulk_create will *not* return the primary keys of ModulestoreBlockSource
+ # objects that already exist (and thus were skipped by ignore_conflicts), so, in
+ # order to use ModulestoreBlockSource instances, we must re-load them in the next step.
+ ignore_conflicts=True
+ )
+ block_source_keys_to_sources: dict[UsageKey, ModulestoreBlockSource] = {
+ block_source.key: block_source for block_source in source.blocks.all()
+ }
+ # @@TODO We get an error when we try to use the change_log, because it hasn't
+ # been saved yet. Until we fix this, we won't be able to set
+ # ModuleBlockMigration.change_log_record.
+ # block_target_pks_to_change_log_record_pks: dict[int, int] = dict(
+ # change_log.records.values_list("target_entity_id", "id")
+ # )
+ #block_migrations = ModulestoreBlockMigration.objects.bulk_create(
+ ModulestoreBlockMigration.objects.bulk_create(
+ [
+ ModulestoreBlockMigration(
+ overall_migration=migration,
+ source=block_source_keys_to_sources[block_source_key],
+ target_id=block_target_ver.entity_id,
+ # @@TODO See above
+ # change_log_record_id=block_target_pks_to_change_log_record_pks[block_target_ver.entity_id],
+ )
+ for block_source_key, block_target_ver in block_source_keys_to_target_vers.items()
+ ],
+ )
+ block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=migration)
+
+ xblock_models.Block.objects.bulk_create(
+ [
+ xblock_models.Block(
+ learning_context=course_lc_learning_context,
+ key=block_source_key,
+ entity_id=block_target_ver.entity_id,
+ )
+ for block_source_key, block_target_ver in block_source_keys_to_target_vers.items()
+ ],
+ update_conflicts=True,
+ update_fields=["entity", "learning_context"],
+ )
+ status.increment_completed_steps()
+
+
+ status.set_state(MigrationStep.FORWARDING.value)
+ if forward_source_to_target:
+ block_sources_to_block_migrations = {
+ block_migration.source: block_migration for block_migration in block_migrations
+ }
+ for block_source, block_migration in block_sources_to_block_migrations.items():
+ block_source.forwarded = block_migration
+ block_source.save()
+ # ModulestoreBlockSource.objects.bulk_update(block_sources_to_block_migrations.keys(), ["forwarded"])
+ source.forwarded = migration
+ source.save()
+ if comp_level == CompositionLevel.CourseRun:
+ catalog_course, _ = CatalogCourse.objects.get_or_create(
+ org_id=source.key.org,
+ course_id=source.key.course,
+ )
+ try:
+ course = Course.objects.get(catalog_course=catalog_course, run=source.key.run)
+ except Course.DoesNotExist:
+ Course.objects.create(
+ catalog_course=catalog_course,
+ run=source.key.run,
+ learning_package=target_package,
+ outline_root=root_migrated_node.source_to_target[1].entity.container.outlineroot,
+ )
+ else:
+ course.learning_package = target_package
+ course.outline_root = root_migrated_node.source_to_target[1].entity.container.outlineroot
+ course.save()
+ status.increment_completed_steps()
+
+ status.set_state(MigrationStep.POPULATING_COLLECTION.value)
+ if target_collection:
+ # @@TODO This dict (block_target_pks_to_change_log_record_pks) was based on the change_log calculation
+ # above which is commented out. In order to add to target_collection, we either need to fix the
+ # change_log query above, or we need to calculate these target_pks some other way
+ block_target_pks: list[int] = [] # list(block_target_pks_to_change_log_record_pks.keys())
+ authoring_api.add_to_collection(
+ learning_package_id=target_package_pk,
+ key=target_collection.key,
+ entities_qset=PublishableEntity.objects.filter(id__in=block_target_pks),
+ created_by=user_id,
+ )
+ status.increment_completed_steps()
+
+ # Now have it use our Learning Core shim for Split instead of Mongo DB
+ course_lc_learning_context.use_learning_core = True
+ course_lc_learning_context.save()
+
+
+@dataclass(frozen=True)
+class _MigratedNode:
+ """
+ A node in the source tree, its target (if migrated), and any migrated children.
+
+ Note that target_version can equal None even when there migrated children.
+ This happens, particularly, if the node is above the requested composition level
+ but has descendents which are at or below sad level.
+ """
+ source_to_target: tuple[UsageKey, PublishableEntityVersion] | None
+ children: list[_MigratedNode]
+
+ def all_source_to_target_pairs(self) -> t.Iterable[tuple[UsageKey, PublishableEntityVersion]]:
+ """
+ Get all source_key->target_ver pairs via a pre-order traversal.
+ """
+ if self.source_to_target:
+ yield self.source_to_target
+ for child in self.children:
+ yield from child.all_source_to_target_pairs()
+
+
+def _migrate_node(
+ *,
+ content_by_filename: dict[str, int],
+ source_context_key: CourseKey, # Note: This includes legacy LibraryLocators, which are sneakily CourseKeys.
+ source_node: XmlTree,
+ target_package_id: int,
+ target_library_key: LibraryLocatorV2,
+ composition_level: CompositionLevel,
+ replace_existing: bool,
+ created_at: datetime,
+ created_by: int,
+) -> _MigratedNode:
+ """
+ Migration an OLX node (source_node) from a legacy course or library (source_context_key) to a
+ learning package (target_library). If the node is a container, create it in the target iff
+ it is at or above the requested composition_level; otherwise, just import its contents.
+ Recursively apply the same logic to all children.
+ """
+ # The OLX tag will map to one of the following...
+ # * A wiki tag --> Ignore
+ # * A recognized container type --> Migration children, and import container if requested.
+ # * A legacy library root --> Migration children, but NOT the root itself.
+ # * A course root --> Migration children, but NOT the root itself (for Teak, at least. Future
+ # releases may support treating the Course as an importable container).
+ # * Something else --> Try to import it as a component. If that fails, then it's either an un-
+ # supported component type, or it's an XBlock with dynamic children, which we
+ # do not support in libraries as of Teak.
+ should_migrate_node: bool
+ should_migrate_children: bool
+ container_type: ContainerType | None # if None, it's a Component
+ if source_node.tag == "wiki":
+ return _MigratedNode(None, [])
+ try:
+ container_type = ContainerType.from_source_olx_tag(source_node.tag)
+ except ValueError:
+ container_type = None
+ if source_node.tag == "library":
+ should_migrate_node = False
+ should_migrate_children = True
+ else:
+ should_migrate_node = True
+ should_migrate_children = False
+ else:
+ node_level = CompositionLevel(container_type.value)
+ should_migrate_node = not node_level.is_higher_than(composition_level)
+ should_migrate_children = True
+ migrated_children: list[_MigratedNode] = []
+ if should_migrate_children:
+ migrated_children = [
+ _migrate_node(
+ content_by_filename=content_by_filename,
+ source_context_key=source_context_key,
+ source_node=source_node_child,
+ target_package_id=target_package_id,
+ target_library_key=target_library_key,
+ composition_level=composition_level,
+ replace_existing=replace_existing,
+ created_by=created_by,
+ created_at=created_at,
+ )
+ for source_node_child in source_node.getchildren()
+ ]
+ source_to_target: tuple[UsageKey, PublishableEntityVersion] | None = None
+ if should_migrate_node:
+ source_olx = etree.tostring(source_node).decode('utf-8')
+ if source_block_id := source_node.get('url_name'):
+ source_key: UsageKey = source_context_key.make_usage_key(source_node.tag, source_block_id)
+ target_entity_version = (
+ _migrate_container(
+ source_key=source_key,
+ container_type=container_type,
+ title=source_node.get('display_name', source_block_id),
+ children=[
+ migrated_child.source_to_target[1]
+ for migrated_child in migrated_children if
+ migrated_child.source_to_target
+ ],
+ target_library_key=target_library_key,
+ replace_existing=replace_existing,
+ created_by=created_by,
+ created_at=created_at,
+ )
+ if container_type else
+ _migrate_component(
+ content_by_filename=content_by_filename,
+ source_key=source_key,
+ olx=source_olx,
+ target_package_id=target_package_id,
+ target_library_key=target_library_key,
+ replace_existing=replace_existing,
+ created_by=created_by,
+ created_at=created_at,
+ )
+ )
+ if target_entity_version:
+ source_to_target = (source_key, target_entity_version)
+ else:
+ log.warning(
+ f"Cannot migrate node from {source_context_key} to {target_library_key} "
+ f"because it lacks an url_name and thus has no identity: {source_olx}"
+ )
+ return _MigratedNode(source_to_target=source_to_target, children=migrated_children)
+
+
+def _migrate_container(
+ *,
+ source_key: UsageKey,
+ container_type: ContainerType,
+ title: str,
+ children: list[PublishableEntityVersion],
+ target_library_key: LibraryLocatorV2,
+ replace_existing: bool,
+ created_by: int,
+ created_at: datetime,
+) -> PublishableEntityVersion:
+ """
+ Create, update, or replace a container in a library based on a source key and children.
+
+ (We assume that the destination is a library rather than some other future kind of learning
+ package, but let's keep than an internal assumption.)
+ """
+ target_key = LibraryContainerLocator(
+ target_library_key, container_type.value, _slugify_source_usage_key(source_key)
+ )
+ try:
+ container = libraries_api.get_container(target_key)
+ container_exists = True
+ except libraries_api.ContentLibraryContainerNotFound:
+ container_exists = False
+ container = libraries_api.create_container(
+ library_key=target_library_key,
+ container_type=container_type,
+ slug=target_key.container_id,
+ title=title,
+ created=created_at,
+ user_id=created_by,
+ )
+ if container_exists and not replace_existing:
+ return PublishableEntityVersion.objects.get(
+ entity_id=container.container_pk,
+ version_num=container.draft_version_num,
+ )
+ next_container_version = authoring_api.create_next_container_version(
+ container.container_pk,
+ title=title,
+ entity_rows=[
+ authoring_api.ContainerEntityRow(entity_pk=child.entity_id, version_pk=None)
+ for child in children
+ ],
+ created=created_at,
+ created_by=created_by,
+ container_version_cls=container_type.container_model_classes[1],
+ )
+ create_xblock_field_data_for_container(next_container_version)
+ return next_container_version.publishable_entity_version
+
+
+def _migrate_component(
+ *,
+ content_by_filename: dict[str, int],
+ source_key: UsageKey,
+ olx: str,
+ target_package_id: int,
+ target_library_key: LibraryLocatorV2,
+ replace_existing: bool,
+ created_by: int,
+ created_at: datetime,
+) -> PublishableEntityVersion | None:
+ """
+ Create, update, or replace a component in a library based on a source key and OLX.
+
+ (We assume that the destination is a library rather than some other future kind of learning
+ package, but let's keep than an internal assumption.)
+ """
+ component_type = authoring_api.get_or_create_component_type("xblock.v1", source_key.block_type)
+ # mypy thinks LibraryUsageLocatorV2 is abstract. It's not.
+ target_key = LibraryUsageLocatorV2( # type: ignore[abstract]
+ target_library_key, source_key.block_type, _slugify_source_usage_key(source_key)
+ )
+ try:
+ component = authoring_api.get_components(target_package_id).get(
+ component_type=component_type,
+ local_key=target_key.block_id,
+ )
+ component_existed = True
+ except Component.DoesNotExist:
+ component_existed = False
+ try:
+ libraries_api.validate_can_add_block_to_library(
+ target_library_key, target_key.block_type, target_key.block_id
+ )
+ except libraries_api.IncompatibleTypesError as e:
+ log.error(f"Error validating block for library {target_library_key}: {e}")
+ return None
+ component = authoring_api.create_component(
+ target_package_id,
+ component_type=component_type,
+ local_key=target_key.block_id,
+ created=created_at,
+ created_by=created_by,
+ )
+ if component_existed and not replace_existing:
+ return component.versioning.draft.publishable_entity_version
+ component_version = libraries_api.set_library_block_olx(target_key, new_olx_str=olx)
+ for filename, content_pk in content_by_filename.items():
+ filename_no_ext, _ = os.path.splitext(filename)
+ if filename_no_ext not in olx:
+ continue
+ new_path = f"static/{filename}"
+ try:
+ authoring_api.create_component_version_content(component_version.pk, content_pk, key=new_path)
+ # @@TODO is there a way to determine content already exists that doesn't run the risk
+ # of swallowing other, unexpected IntegrityErrors?
+ except IntegrityError:
+ pass # Content already exists
+ return component_version.publishable_entity_version
+
+
+def _slugify_source_usage_key(key: UsageKey) -> str:
+ """
+ Return an appropriate slug (aka block_id, aka container_id) for the target entity.
+
+ Mix the legacy source context (course/library) information with the source block_id,
+ thus ensuring that migrations of two blocks with the same id from different source
+ contexts will not collide.
+
+ Note: We don't need to factor in block_type, because the target keys already do that.
+ Note: Slugs can't have plus signs, which are normally how legacy keys are delimted.
+
+ @@TODO -- Is this good enough? Conflicts are technically possible if a user has manually
+ formatted the key this way, but that seems super unlikely to happen by accident, right ... ?
+ Conflicts are also technically posssible for
+ """
+ context_key = key.course_key
+ if isinstance(context_key, LibraryLocator):
+ return f"{context_key.org}__{context_key.library}__{key.block_id}"
+ elif isinstance(context_key, CourseKey):
+ return f"{context_key.org}__{context_key.course}__{context_key.run}__{key.block_id}"
+ else:
+ raise ValueError(
+ f"Unexpected source usage key: {key}. Expected legacy course or library usage locator."
+ )
diff --git a/cms/envs/common.py b/cms/envs/common.py
index 38117b4e4a57..56407372d479 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -1636,7 +1636,7 @@
'openedx.core.djangoapps.course_groups', # not used in cms (yet), but tests run
'cms.djangoapps.xblock_config.apps.XBlockConfig',
'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig',
- 'cms.djangoapps.import_from_modulestore.apps.ImportFromModulestoreConfig',
+ 'cms.djangoapps.modulestore_migrator',
# New (Learning-Core-based) XBlock runtime
'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig',
@@ -1834,6 +1834,8 @@
"openedx_learning.apps.authoring.units",
"openedx_learning.apps.authoring.subsections",
"openedx_learning.apps.authoring.sections",
+ "openedx_learning.apps.authoring.outline_roots",
+ "openedx_learning.apps.authoring.courses",
]
diff --git a/cms/urls.py b/cms/urls.py
index f2c2c8b31a4a..28467cae221e 100644
--- a/cms/urls.py
+++ b/cms/urls.py
@@ -141,6 +141,8 @@
# rest api for course import/export
path('api/courses/', include('cms.djangoapps.contentstore.api.urls', namespace='courses_api')
),
+ path('api/modulestore_migrator/',
+ include('cms.djangoapps.modulestore_migrator.rest_api.urls', namespace='modulestore_migrator_api')),
re_path(fr'^export/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_handler,
name='export_handler'),
re_path(fr'^export_output/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_output_handler,
diff --git a/lms/djangoapps/courseware/toggles.py b/lms/djangoapps/courseware/toggles.py
index f9f083cad42e..b97684c2f37f 100644
--- a/lms/djangoapps/courseware/toggles.py
+++ b/lms/djangoapps/courseware/toggles.py
@@ -201,4 +201,6 @@ def courseware_disable_navigation_sidebar_blocks_caching(course_key=None):
"""
Return whether the courseware.disable_navigation_sidebar_blocks_caching flag is on.
"""
+ return True # For debugging the Learning core shim proof of concept
+
return COURSEWARE_MICROFRONTEND_NAVIGATION_SIDEBAR_BLOCKS_DISABLE_CACHING.is_enabled(course_key)
diff --git a/lms/envs/common.py b/lms/envs/common.py
index ea7f97556ca1..bde836c3938e 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -3375,6 +3375,8 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring
"openedx_learning.apps.authoring.units",
"openedx_learning.apps.authoring.subsections",
"openedx_learning.apps.authoring.sections",
+ "openedx_learning.apps.authoring.outline_roots",
+ "openedx_learning.apps.authoring.courses",
]
diff --git a/mypy.ini b/mypy.ini
index f0267364ffb0..982e500b3a93 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -9,7 +9,7 @@ files =
cms/lib/xblock/upstream_sync.py,
cms/lib/xblock/upstream_sync_container.py,
cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py,
- cms/djangoapps/import_from_modulestore,
+ cms/djangoapps/modulestore_migrator,
openedx/core/djangoapps/content/learning_sequences,
# FIXME: need to solve type issues and add 'search' app here:
# openedx/core/djangoapps/content/search,
diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py
index 49f423ce7ac3..78f04b6453f4 100644
--- a/openedx/core/djangoapps/content/block_structure/manager.py
+++ b/openedx/core/djangoapps/content/block_structure/manager.py
@@ -153,9 +153,11 @@ def _bulk_operations(self):
"""
A context manager for notifying the store of bulk operations.
"""
+ from xmodule.modulestore import ModuleStoreEnum
try:
course_key = self.root_block_usage_key.course_key
except AttributeError:
course_key = None
- with self.modulestore.bulk_operations(course_key):
- yield
+ with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, course_key):
+ with self.modulestore.bulk_operations(course_key):
+ yield
diff --git a/openedx/core/djangoapps/content/block_structure/store.py b/openedx/core/djangoapps/content/block_structure/store.py
index bb6359b9d3a3..a89b353eb2a1 100644
--- a/openedx/core/djangoapps/content/block_structure/store.py
+++ b/openedx/core/djangoapps/content/block_structure/store.py
@@ -102,6 +102,8 @@ def is_up_to_date(self, root_block_usage_key, modulestore):
Returns whether the data in storage for the given key is
already up-to-date with the version in the given modulestore.
"""
+ return False
+
try:
bs_model = self._get_model(root_block_usage_key)
root_block = modulestore.get_item(root_block_usage_key)
diff --git a/openedx/core/djangoapps/content/block_structure/tasks.py b/openedx/core/djangoapps/content/block_structure/tasks.py
index 5796b8167b2b..de23e1bfc5a4 100644
--- a/openedx/core/djangoapps/content/block_structure/tasks.py
+++ b/openedx/core/djangoapps/content/block_structure/tasks.py
@@ -59,6 +59,7 @@ def _update_course_in_cache(self, **kwargs):
"""
Updates the course blocks (mongo -> BlockStructure) for the specified course.
"""
+ log.info("Inner _update_course_in_cache called.")
_call_and_retry_if_needed(self, api.update_course_in_cache, **kwargs)
diff --git a/openedx/core/djangoapps/content/search/api.py b/openedx/core/djangoapps/content/search/api.py
index 389f6d21161f..d7cdd26cbafa 100644
--- a/openedx/core/djangoapps/content/search/api.py
+++ b/openedx/core/djangoapps/content/search/api.py
@@ -502,6 +502,8 @@ def index_container_batch(batch, num_done, library_key) -> int:
doc.update(searchable_doc_containers(container_key, "subsections"))
case lib_api.ContainerType.Subsection:
doc.update(searchable_doc_containers(container_key, "sections"))
+ case lib_api.ContainerType.Section:
+ doc.update(searchable_doc_containers(container_key, "outline_roots"))
docs.append(doc)
except Exception as err: # pylint: disable=broad-except
status_cb(f"Error indexing container {container.key}: {err}")
diff --git a/openedx/core/djangoapps/content/search/documents.py b/openedx/core/djangoapps/content/search/documents.py
index aaabb0b92a51..88ec128ddc0c 100644
--- a/openedx/core/djangoapps/content/search/documents.py
+++ b/openedx/core/djangoapps/content/search/documents.py
@@ -630,7 +630,7 @@ def get_child_keys(children) -> list[str]:
str(child.usage_key)
for child in children
]
- case lib_api.ContainerType.Subsection | lib_api.ContainerType.Section:
+ case lib_api.ContainerType.Subsection | lib_api.ContainerType.Section | lib_api.ContainerType.OutlineRoot:
return [
str(child.container_key)
for child in children
diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py
index 1457586301e1..d345bd22ebe0 100644
--- a/openedx/core/djangoapps/content_libraries/api/containers.py
+++ b/openedx/core/djangoapps/content_libraries/api/containers.py
@@ -25,10 +25,10 @@
)
from openedx_learning.api import authoring as authoring_api
from openedx_learning.api.authoring_models import Container, ContainerVersion, Component
+
from openedx.core.djangoapps.content_libraries.api.collections import library_collection_locator
from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts
-
-from openedx.core.djangoapps.xblock.api import get_component_from_usage_key
+from openedx.core.djangoapps.xblock.api import create_xblock_field_data_for_container, get_component_from_usage_key
from ..models import ContentLibrary
from .exceptions import ContentLibraryContainerNotFound
@@ -53,6 +53,9 @@
"update_container_children",
"get_containers_contains_item",
"publish_container_changes",
+
+ # Hacky XBlock data-for-containers
+ "create_xblock_field_data_for_container",
]
log = logging.getLogger(__name__)
@@ -65,6 +68,39 @@ class ContainerType(Enum):
Unit = "unit"
Subsection = "subsection"
Section = "section"
+ OutlineRoot = "outline_root"
+
+ @property
+ def container_model_classes(self) -> tuple[type[Container], type[ContainerVersion]]:
+ """
+ Get the container, containerversion subclasses associated with this type.
+
+ @@TODO Is this what we want, a hard mapping between container_types and Container classes?
+ * If so, then expand on this pattern, so that all ContainerType logic is contained within
+ this class, and get rid of the match-case statements that are all over the content_libraries
+ app.
+ * If not, then figure out what to do instead.
+ """
+ from openedx_learning.api.authoring_models import (
+ Unit,
+ UnitVersion,
+ Subsection,
+ SubsectionVersion,
+ Section,
+ SectionVersion,
+ OutlineRoot,
+ OutlineRootVersion,
+ )
+ match self:
+ case self.Unit:
+ return (Unit, UnitVersion)
+ case self.Subsection:
+ return (Subsection, SubsectionVersion)
+ case self.Section:
+ return (Section, SectionVersion)
+ case self.OutlineRoot:
+ return (OutlineRoot, OutlineRootVersion)
+ raise TypeError(f"unexpected ContainerType: {self!r}")
@property
def olx_tag(self) -> str:
@@ -84,6 +120,8 @@ def olx_tag(self) -> str:
return "sequential"
case self.Section:
return "chapter"
+ case self.OutlineRoot:
+ return "course"
raise TypeError(f"unexpected ContainerType: {self!r}")
@classmethod
@@ -168,6 +206,8 @@ def library_container_locator(
container_type = ContainerType.Subsection
elif hasattr(container, 'section'):
container_type = ContainerType.Section
+ elif hasattr(container, 'outlineroot'):
+ container_type = ContainerType.OutlineRoot
assert container_type is not None
@@ -252,12 +292,12 @@ def create_container(
created = datetime.now(tz=timezone.utc)
container: Container
- _initial_version: ContainerVersion
+ initial_version: ContainerVersion
# Then try creating the actual container:
match container_type:
case ContainerType.Unit:
- container, _initial_version = authoring_api.create_unit_and_version(
+ container, initial_version = authoring_api.create_unit_and_version(
content_library.learning_package_id,
key=slug,
title=title,
@@ -265,7 +305,7 @@ def create_container(
created_by=user_id,
)
case ContainerType.Subsection:
- container, _initial_version = authoring_api.create_subsection_and_version(
+ container, initial_version = authoring_api.create_subsection_and_version(
content_library.learning_package_id,
key=slug,
title=title,
@@ -273,7 +313,15 @@ def create_container(
created_by=user_id,
)
case ContainerType.Section:
- container, _initial_version = authoring_api.create_section_and_version(
+ container, initial_version = authoring_api.create_section_and_version(
+ content_library.learning_package_id,
+ key=slug,
+ title=title,
+ created=created,
+ created_by=user_id,
+ )
+ case ContainerType.OutlineRoot:
+ container, initial_version = authoring_api.create_outline_root_and_version(
content_library.learning_package_id,
key=slug,
title=title,
@@ -283,6 +331,8 @@ def create_container(
case _:
raise NotImplementedError(f"Library does not support {container_type} yet")
+ create_xblock_field_data_for_container(initial_version)
+
LIBRARY_CONTAINER_CREATED.send_event(
library_container=LibraryContainerData(
container_key=container_key,
@@ -333,12 +383,22 @@ def update_container(
created=created,
created_by=user_id,
)
-
- # The `affected_containers` are not obtained, because the sections are
+ affected_containers = get_containers_contains_item(container_key)
+ case ContainerType.OutlineRoot:
+ version = authoring_api.create_next_outline_root_version(
+ container.outlineroot,
+ title=display_name,
+ created=created,
+ created_by=user_id,
+ )
+ # The `affected_containers` are not obtained, because the outline_roots are
# not contained in any container.
case _:
raise NotImplementedError(f"Library does not support {container_type} yet")
+ # Let's add some XBlock data onto the container we just made...
+ create_xblock_field_data_for_container(version)
+
# Send event related to the updated container
LIBRARY_CONTAINER_UPDATED.send_event(
library_container=LibraryContainerData(
@@ -558,6 +618,23 @@ def update_container_children(
entities_action=entities_action,
)
+ for key in children_ids:
+ CONTENT_OBJECT_ASSOCIATIONS_CHANGED.send_event(
+ content_object=ContentObjectChangedData(
+ object_id=str(key),
+ changes=["sections"],
+ ),
+ )
+ case ContainerType.OutlineRoot:
+ subsections = [_get_container_from_key(key).outlineroot for key in children_ids] # type: ignore[arg-type]
+ new_version = authoring_api.create_next_outline_root_version(
+ container.outlineroot,
+ sections=sections, # type: ignore[arg-type]
+ created=created,
+ created_by=user_id,
+ entities_action=entities_action,
+ )
+
for key in children_ids:
CONTENT_OBJECT_ASSOCIATIONS_CHANGED.send_event(
content_object=ContentObjectChangedData(
@@ -568,6 +645,8 @@ def update_container_children(
case _:
raise ValueError(f"Invalid container type: {container_type}")
+ create_xblock_field_data_for_container(new_version)
+
LIBRARY_CONTAINER_UPDATED.send_event(
library_container=LibraryContainerData(
container_key=container_key,
diff --git a/openedx/core/djangoapps/content_libraries/signal_handlers.py b/openedx/core/djangoapps/content_libraries/signal_handlers.py
index fe5489493641..4aa006a1d7c5 100644
--- a/openedx/core/djangoapps/content_libraries/signal_handlers.py
+++ b/openedx/core/djangoapps/content_libraries/signal_handlers.py
@@ -17,7 +17,12 @@
LIBRARY_COLLECTION_UPDATED
)
from openedx_learning.api.authoring import get_components, get_containers
-from openedx_learning.api.authoring_models import Collection, CollectionPublishableEntity, PublishableEntity
+from openedx_learning.api.authoring_models import (
+ Collection,
+ CollectionPublishableEntity,
+ PublishableEntity,
+ PublishLog,
+)
from lms.djangoapps.grades.api import signals as grades_signals
diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py
index 72d2d1b0884f..fc5d812dd09a 100644
--- a/openedx/core/djangoapps/content_libraries/tasks.py
+++ b/openedx/core/djangoapps/content_libraries/tasks.py
@@ -124,6 +124,11 @@ def wait_for_post_publish_events(publish_log: PublishLog, library_key: LibraryLo
up to some reasonable timeout, and then finish anything remaining
asynchonrously.
"""
+ from openedx.core.djangoapps.xblock.api import handle_library_publish
+
+ # Learning Core Shim code (we really want the publish_log)
+ handle_library_publish(publish_log)
+
# Update the search index (and anything else) for the affected blocks
result = send_events_after_publish.apply_async(args=(publish_log.pk, str(library_key)))
# Try waiting a bit for those post-publish events to be handled:
diff --git a/openedx/core/djangoapps/xblock/admin.py b/openedx/core/djangoapps/xblock/admin.py
new file mode 100644
index 000000000000..bc55a01f3171
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/admin.py
@@ -0,0 +1,109 @@
+"""Django admin interface for XBlock field data models."""
+import json
+
+from django.contrib import admin
+from django.urls import reverse
+from django.utils.html import format_html
+from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin
+
+from .models import XBlockVersionFieldData, LearningCoreLearningContext
+
+@admin.register(LearningCoreLearningContext)
+class LearningCoreLearningContextAdmin(admin.ModelAdmin):
+ list_display = [
+ "key",
+ "use_learning_core",
+ ]
+
+
+@admin.register(XBlockVersionFieldData)
+class XBlockVersionFieldDataAdmin(ReadOnlyModelAdmin):
+ """
+ Admin interface for XBlock field data records.
+ """
+
+ list_display = [
+ 'component',
+ 'version_num',
+ 'uuid',
+ 'created',
+ ]
+
+ fields = [
+ 'publishable_entity_version_link',
+ 'component_link',
+ 'version_num',
+ 'uuid',
+ 'created',
+ 'content_display',
+ 'settings_display',
+ ]
+
+ readonly_fields = fields
+
+ def get_queryset(self, request):
+ """Optimize queries by selecting related objects."""
+ queryset = super().get_queryset(request)
+ return queryset.select_related(
+ 'publishable_entity_version',
+ 'publishable_entity_version__componentversion',
+ 'publishable_entity_version__componentversion__component',
+ )
+
+ def component(self, obj):
+ """Display the component key."""
+ try:
+ return obj.publishable_entity_version.componentversion.component.key
+ except AttributeError:
+ return "N/A"
+
+ def version_num(self, obj):
+ """Display the version number."""
+ try:
+ return obj.publishable_entity_version.version_num
+ except AttributeError:
+ return "N/A"
+
+ def uuid(self, obj):
+ """Display the UUID."""
+ try:
+ return obj.publishable_entity_version.uuid
+ except AttributeError:
+ return "N/A"
+
+ @admin.display(description="Publishable Entity Version")
+ def publishable_entity_version_link(self, obj):
+ """Link to the related publishable entity version in admin."""
+ admin_url = reverse('admin:oel_components_componentversion_change', args=[obj.publishable_entity_version.pk])
+ return format_html('{}', admin_url, obj.publishable_entity_version)
+
+ @admin.display(description="Component")
+ def component_link(self, obj):
+ """Link to the related component in admin."""
+ admin_url = reverse(
+ "admin:oel_components_component_change",
+ args=[obj.publishable_entity_version.componentversion.component.pk],
+ )
+ return format_html('{}', admin_url, obj.publishable_entity_version.componentversion.component)
+
+ @admin.display(description="Content Fields")
+ def content_display(self, obj):
+ """Display formatted content fields."""
+ if not obj.content:
+ return "No content fields"
+
+ return format_html(
+ '{}',
+ json.dumps(obj.content, indent=2),
+ )
+
+ @admin.display(description="Settings Fields")
+ def settings_display(self, obj):
+ """Display formatted settings fields."""
+ if not obj.settings:
+ return "No settings fields"
+
+ return format_html(
+ '{}',
+ json.dumps(obj.settings, indent=2),
+ )
diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py
index c806fefc87c5..17b3e5188dd8 100644
--- a/openedx/core/djangoapps/xblock/api.py
+++ b/openedx/core/djangoapps/xblock/api.py
@@ -13,11 +13,12 @@
import logging
import threading
+import bson.tz_util
from django.core.exceptions import PermissionDenied
from django.urls import reverse
from django.utils.translation import gettext as _
from openedx_learning.api import authoring as authoring_api
-from openedx_learning.api.authoring_models import Component, ComponentVersion
+from openedx_learning.api.authoring_models import Component, ComponentVersion, ContainerVersion, PublishLog
from opaque_keys.edx.keys import UsageKeyV2
from opaque_keys.edx.locator import LibraryUsageLocatorV2
from rest_framework.exceptions import NotFound
@@ -330,3 +331,350 @@ def get_handler_url(
# can be called by the XBlock from python as well and in that case we don't
# have access to the request.
return site_root_url + path
+
+
+from django.template.defaultfilters import filesizeformat
+from opaque_keys.edx.keys import CourseKey
+from xmodule.modulestore import BlockData
+from xmodule.modulestore.split_mongo import BlockKey
+from datetime import datetime, timezone
+import bson
+from bson import ObjectId
+from bson.codec_options import CodecOptions
+import zlib
+from openedx.core.lib.cache_utils import request_cached
+
+
+from .models import (
+ LearningCoreCourseStructure,
+ LearningCoreLearningContext,
+ XBlockVersionFieldData,
+)
+
+def get_structure_for_course(course_key: CourseKey):
+ """Just gets the published version for now, need to update to do both branches later"""
+ lookup_key = course_key.replace(branch=None, version_guid=None)
+ lccs = LearningCoreCourseStructure.objects.get(course_key=lookup_key)
+ uncompressed_data = zlib.decompress(lccs.structure)
+ return bson.decode(uncompressed_data, codec_options=CodecOptions(tz_aware=True))
+
+
+def update_learning_core_course(course_key: CourseKey):
+ """
+ This is going to write to LearningCoreCourseStructure.
+
+ Pass 0 of this: just push hardcoded data into the shim
+
+ """
+ writer = LearningCoreCourseShimWriter(course_key)
+ structure = writer.make_structure()
+
+ import pprint
+
+ with open("lc_struct.txt", "w") as struct_file:
+ printer = pprint.PrettyPrinter(indent=2, stream=struct_file)
+ printer.pprint(structure)
+
+ # Structure doc is so repetitive that we get a 4-5X reduction in file size
+ num_blocks = len(structure['blocks'])
+ encoded_structure = zlib.compress(bson.encode(structure, codec_options=CodecOptions(tz_aware=True)))
+
+ lccs, _created = LearningCoreCourseStructure.objects.get_or_create(course_key=course_key)
+ lccs.structure = encoded_structure
+ lccs.save()
+
+ log.info(f"Updated Learning Core Structure (for Split) on course {course_key}.")
+ log.info(f"Structure size: {filesizeformat(len(encoded_structure))} for {num_blocks} blocks.")
+
+ from xmodule.modulestore.django import SignalHandler
+ log.info(f"Emitting course_published signal for {course_key}")
+ SignalHandler.course_published.send_robust(sender=update_learning_core_course, course_key=course_key)
+
+
+@request_cached()
+def learning_core_backend_enabled_for_course(course_key: CourseKey):
+ try:
+ lookup_key = course_key.replace(branch=None, version_guid=None)
+ lc_context = LearningCoreLearningContext.objects.get(key=lookup_key)
+ return lc_context.use_learning_core
+ except LearningCoreLearningContext.DoesNotExist:
+ return False
+
+
+def get_definition_doc(def_id: ObjectId):
+ try:
+ xb_field_data = XBlockVersionFieldData.objects.get(definition_object_id=str(def_id))
+ except XBlockVersionFieldData.DoesNotExist:
+ return None
+
+ return {
+ '_id': ObjectId(xb_field_data.definition_object_id),
+ 'block_type': None,
+ 'fields': xb_field_data.content,
+ 'edit_info': {
+ 'edited_by': xb_field_data.publishable_entity_version.created_by_id,
+ 'edited_on': xb_field_data.publishable_entity_version.created,
+
+ # These are supposed to be the ObjectIds of the structure docs that
+ # represent the last time this block was edited and the original
+ # version at the time of creation. It's actually a common occurrence
+ # for these values to get pruned in Split, so we're making dummy
+ # ObjectIds--i.e. we're making it look like this was created a while
+ # ago and the versions for both the original creation and last
+ # update are no longer available.
+ 'previous_version': ObjectId(),
+ 'original_version': ObjectId(),
+ },
+ 'schema_version': 1,
+ }
+
+
+def handle_library_publish(publish_log: PublishLog):
+ affected_course_keys = set(
+ key
+ for key in publish_log.records.values_list('entity__block__learning_context__key', flat=True)
+ if key
+ )
+ log.info(f"Affected Courses to update in LC shim: {affected_course_keys}")
+ for course_key in affected_course_keys:
+ log.info(f"Type of course_key: {type(course_key)}")
+ update_learning_core_course(course_key)
+
+
+def create_xblock_field_data_for_container(version: ContainerVersion):
+ # this whole thing should be in xblock.api instead of here.
+ from openedx.core.djangoapps.xblock.models import Block
+
+ entity = version.publishable_entity_version.entity
+
+ # If this PublishableEntity isn't associated with an Learning Core backed
+ # XBlock, then we can't write anything. Note: This is going to be an edge
+ # case later, when we want to add an existing container to a container that
+ # was imported from a course.
+ if not hasattr(entity, 'block'):
+ log.error(f"No Block detected for entity {entity.key}???")
+ return
+
+ parent_block = entity.block
+ container_usage_key = parent_block.key
+ course_key = container_usage_key.course_key
+
+ # Generic values for all container types
+ content_scoped_fields = {}
+ settings_scoped_fields = {
+ 'display_name': version.publishable_entity_version.title
+ }
+ children = []
+
+ # Things specific to the course root...
+ if container_usage_key.block_type == "course":
+ content_scoped_fields['license'] = None
+ content_scoped_fields['wiki_slug'] = f'{course_key.org}.{course_key.course}.{course_key.run}'
+ settings_scoped_fields.update(
+ _course_block_entry(container_usage_key)
+ )
+
+ for child_entity_row in version.entity_list.entitylistrow_set.select_related('entity__block').all():
+ log.info(f"Iterating children: {child_entity_row.entity}")
+
+ # If it's not a container and it doesn't have field data, we won't know what to do with it in the structure doc,
+ # so just skip it. This can happen when you have OLX with no corresponding XBlock class.
+ if not hasattr(child_entity_row.entity, 'container') and not hasattr(child_entity_row.entity, 'xblockfielddata'):
+ continue
+
+ if not hasattr(child_entity_row.entity, 'block'):
+ # This can happen if we add a new component in a library to a
+ # container that was imported from a course.
+ match(container_usage_key.block_type):
+ case "course":
+ child_block_type = "chapter"
+ child_block_id = child_entity_row.entity.key
+ case "chapter":
+ child_block_type = "sequential"
+ child_block_id = child_entity_row.entity.key
+ case "sequential":
+ child_block_type = "vertical"
+ child_block_id = child_entity_row.entity.key
+ case "vertical":
+ child_block_type = child_entity_row.entity.component.component_type.name
+ child_block_id = child_entity_row.entity.component.local_key
+
+ log.info(f"Creating child usage key: {child_usage_key}")
+ child_usage_key = course_key.make_usage_key(child_block_type, child_block_id)
+ child_block = Block.objects.create(
+ learning_context_id=parent_block.learning_context_id,
+ entity=child_entity_row.entity,
+ key=child_usage_key,
+ )
+ else:
+ child_block = child_entity_row.entity.block
+ child_usage_key = child_block.key
+
+ if child_usage_key.block_type != "discussion":
+ children.append(
+ [child_usage_key.block_type, child_usage_key.block_id]
+ )
+
+ field_data = XBlockVersionFieldData.objects.create(
+ pk=version.pk,
+ content=content_scoped_fields,
+ settings=settings_scoped_fields,
+ children=children,
+ )
+ log.info(f"Wrote XBlock Data for Container: {version}: {field_data}")
+
+
+def _course_block_entry(usage_key):
+ return {
+ 'allow_anonymous': True,
+ 'allow_anonymous_to_peers': False,
+ 'cert_html_view_enabled': True,
+ 'discussion_blackouts': [],
+ 'discussion_topics': {'General': {'id': 'course'}},
+ 'discussions_settings': {
+ 'enable_graded_units': False,
+ 'enable_in_context': True,
+ 'openedx': { 'group_at_subsection': False},
+ 'posting_restrictions': 'disabled',
+ 'provider_type': 'openedx',
+ 'unit_level_visibility': True
+ },
+ 'end': None,
+ 'language': 'en',
+
+ ## HARDCODED START DATE
+ 'start': datetime(2020, 1, 1, 0, 0, tzinfo=timezone.utc),
+ 'static_asset_path': 'course',
+ 'tabs': [
+ {
+ 'course_staff_only': False,
+ 'name': 'Course',
+ 'type': 'courseware'
+ },
+ {
+ 'course_staff_only': False,
+ 'name': 'Progress',
+ 'type': 'progress'
+ },
+ {
+ 'course_staff_only': False,
+ 'name': 'Dates',
+ 'type': 'dates'
+ },
+ {
+ 'course_staff_only': False,
+ 'name': 'Discussion',
+ 'type': 'discussion'
+ },
+ {
+ 'course_staff_only': False,
+ 'is_hidden': True,
+ 'name': 'Wiki',
+ 'type': 'wiki'
+ },
+ {
+ 'course_staff_only': False,
+ 'name': 'Textbooks',
+ 'type': 'textbooks'
+ }
+ ],
+ 'xml_attributes': {
+ 'filename': [ f'course/{usage_key.run}.xml', f'course/{usage_key.run}.xml']
+ }
+ }
+
+
+class LearningCoreCourseShimWriter:
+ def __init__(self, course_key: CourseKey):
+ self.course_key = course_key
+ self.structure_obj_id = bson.ObjectId()
+
+ self.edited_on = datetime.now(tz=timezone.utc)
+ self.user_id = -1 # This is "the system did it"
+
+ def make_structure(self):
+ structure = self.base_structure()
+
+ context = LearningCoreLearningContext.objects.get(key=self.course_key)
+ blocks = (
+ context.blocks
+ .select_related(
+ 'entity__published__version__xblockversionfielddata',
+ 'entity__draft__version__xblockversionfielddata',
+ )
+ )
+ for block in blocks:
+ entity_version = block.entity.published.version
+ if not hasattr(entity_version, 'xblockversionfielddata'):
+ log.error(f"MISSING XBlockVersionFieldData for {block.key}")
+ continue # Just give up on this block.
+
+ if block.key.block_type == "discussion":
+ # hacky hacky prototype...
+ log.error(f"Skipping discussion block {block.key} because we don't seem to handle inline discussions right")
+ continue
+
+ field_data = entity_version.xblockversionfielddata
+ block_entry = self.base_block_entry(
+ block.key.block_type,
+ block.key.block_id,
+ ObjectId(field_data.definition_object_id),
+ )
+ block_entry['fields'].update(field_data.settings)
+
+ # This is a hack for when discussion data's already gotten into our saved children, even though we don't
+ # have any field data associated with it.
+ if field_data.children:
+ filtered_children = [
+ entry for entry in field_data.children if entry[0] != "discussion"
+ ]
+ if filtered_children:
+ block_entry['fields']['children'] = filtered_children
+
+ structure['blocks'].append(block_entry)
+
+ return structure
+
+ def base_structure(self):
+ doc_id = bson.ObjectId()
+
+ return {
+ '_id': doc_id,
+ 'blocks': [],
+ 'schema_version': 1, # LOL
+
+ 'root': ['course', 'course'], # Root is always the CourseBlock
+ 'edited_by': self.user_id,
+ 'edited_on': self.edited_on,
+
+ # We're always going to be the "first" version for now, from Split's
+ # perspective.
+ 'previous_version': None,
+ 'original_version': doc_id
+ }
+
+ def base_block_entry(self, block_type: str, block_id: str, definition_object_id: ObjectId):
+ return {
+ 'asides': {}, # We are *so* not doing asides in this prototype
+ 'block_id': block_id,
+ 'block_type': block_type,
+ 'defaults': {},
+ 'fields': {'children': []}, # Even blocks without children are written this way.
+ 'definition': definition_object_id,
+ 'edit_info': self.base_edit_info()
+ }
+
+ def base_edit_info(self):
+ return {
+ 'edited_by': self.user_id,
+ 'edited_on': self.edited_on,
+
+ # This is v1 libraries data that we're faking
+ 'original_usage': None,
+ 'original_usage_vesion': None,
+
+ # Edit history, all of which we're faking
+ 'previous_version': None,
+ 'source_version': self.structure_obj_id,
+ 'update_version': self.structure_obj_id,
+ }
diff --git a/openedx/core/djangoapps/xblock/management/__init__.py b/openedx/core/djangoapps/xblock/management/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/openedx/core/djangoapps/xblock/management/commands/__init__.py b/openedx/core/djangoapps/xblock/management/commands/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/openedx/core/djangoapps/xblock/management/commands/migrate_xblock_field_data.py b/openedx/core/djangoapps/xblock/management/commands/migrate_xblock_field_data.py
new file mode 100644
index 000000000000..7c28b83f3717
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/management/commands/migrate_xblock_field_data.py
@@ -0,0 +1,173 @@
+"""
+Management command to migrate existing XBlock OLX content to the field data model.
+
+This command processes Components and uses the XBlock runtime to generate and save field data for both published
+and draft versions, ensuring that XBlockVersionFieldData records exist for faster field access.
+"""
+import logging
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from django.db.models import Q
+
+from openedx_learning.api.authoring_models import Component
+from xblock.exceptions import NoSuchUsage
+
+from openedx.core.djangoapps.xblock.models import XBlockVersionFieldData
+from openedx.core.djangoapps.xblock.api import get_runtime
+from openedx.core.djangoapps.xblock.data import AuthoredDataMode, LatestVersion
+from opaque_keys.edx.keys import UsageKeyV2
+
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+ """
+ Management command to populate XBlockVersionFieldData from existing Components.
+
+ This command processes Components, using the XBlock runtime's get_block method
+ to generate and save field data for both published and draft versions.
+ This is intended to pre-populate the field data cache for performance.
+ """
+
+ help = "Migrate existing XBlock content to the field data model using the runtime."
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '--learning-package-id',
+ type=int,
+ help='Only process Components from this learning package.'
+ )
+ parser.add_argument(
+ '--batch-size',
+ type=int,
+ default=100,
+ help='Number of Components to process in each batch (default: 100).'
+ )
+ parser.add_argument(
+ '--dry-run',
+ action='store_true',
+ help='Show what would be migrated without actually creating records.'
+ )
+ parser.add_argument(
+ '--force',
+ action='store_true',
+ help='Recreate field data even if it already exists.'
+ )
+ parser.add_argument(
+ '--block-type',
+ type=str,
+ help='Only process Components of this block type (e.g., "problem", "html").'
+ )
+
+ def handle(self, *args, **options):
+ """Process Components and create field data records."""
+
+ filters = Q()
+ if options['learning_package_id']:
+ filters &= Q(learning_package_id=options['learning_package_id'])
+ if options['block_type']:
+ filters &= Q(component_type__name=options['block_type'])
+
+ try:
+ components = Component.objects.select_related('learning_package').filter(filters).order_by('pk')
+ total_count = components.count()
+ self.stdout.write(f"Found {total_count} Components to process")
+ except Exception as e: # pylint: disable=broad-exception-caught
+ raise CommandError(f"Failed to query Components: {e}") from e
+
+ if options['dry_run']:
+ self.stdout.write(self.style.WARNING("DRY RUN - No changes will be made"))
+
+ if total_count == 0:
+ self.stdout.write(self.style.SUCCESS("No Components to process."))
+ return
+
+ batch_size = options['batch_size']
+ processed_components = 0
+ processed_versions = 0
+ errors = 0
+
+ self.stdout.write(f"Processing in batches of {batch_size}...")
+
+ # We do not need a user instance to generate XBlock's field data from `Scope.content` and `Scope.settings`.
+ runtime = get_runtime(None) # type: ignore
+ # Set the mode to Authoring to access draft versions, in case somebody runs this from LMS.
+ runtime.authored_data_mode = AuthoredDataMode.DEFAULT_DRAFT
+
+ for batch_start in range(0, total_count, batch_size):
+ batch_end = min(batch_start + batch_size, total_count)
+ component_batch = components[batch_start:batch_end]
+
+ for component in component_batch:
+ processed_components += 1
+ key = None
+ try:
+ # FIXME: There must be a better way to construct the key.
+ key = UsageKeyV2.from_string(
+ f"lb{component.learning_package.key.lstrip('lib')}{component.key.lstrip('xblock.v1')}"
+ )
+
+ # If --force is used, delete existing field data first.
+ if options['force']:
+ versions_to_clear = []
+ if component.versioning.draft:
+ versions_to_clear.append(component.versioning.draft.pk)
+ if component.versioning.published:
+ versions_to_clear.append(component.versioning.published.pk)
+
+ if versions_to_clear:
+ if options['dry_run']:
+ self.stdout.write(
+ f"DRY RUN: Would delete existing field data for {key}"
+ )
+ else:
+ with transaction.atomic():
+ deleted_count, _ = XBlockVersionFieldData.objects.filter(
+ publishable_entity_version_id__in=versions_to_clear
+ ).delete()
+ if deleted_count > 0:
+ log.info(
+ "Deleted %d field data records for %s due to --force", deleted_count, key
+ )
+
+ for version in (LatestVersion.PUBLISHED, LatestVersion.DRAFT):
+ if options['dry_run']:
+ if version == LatestVersion.DRAFT and not component.versioning.draft:
+ continue
+ if version == LatestVersion.PUBLISHED and not component.versioning.published:
+ continue
+ self.stdout.write(f"DRY RUN: Would process {key} version {version.name}")
+ continue
+
+ try:
+ # This call will parse the OLX and create a new database record if it doesn't exist.
+ runtime.get_block(key, version=version)
+ processed_versions += 1
+ except NoSuchUsage:
+ # This is expected if a component doesn't have a draft or published version.
+ pass
+ except Exception as e: # pylint: disable=broad-exception-caught
+ errors += 1
+ log.exception("Error processing %s version %s: %s", key, version.name, e)
+ self.stdout.write(
+ self.style.ERROR(f"Error on {key} v{version.name}: {e}")
+ )
+
+ except Exception as e: # pylint: disable=broad-exception-caught
+ errors += 1
+ error_key_str = key or f"Component PK {component.pk}"
+ log.exception("Error processing %s: %s", error_key_str, e)
+ self.stdout.write(
+ self.style.ERROR(f"Error on {error_key_str}: {e}")
+ )
+
+ if processed_components % 100 == 0 and processed_components > 0:
+ self.stdout.write(f"Processed {processed_components}/{total_count} components...")
+
+ self.stdout.write(
+ self.style.SUCCESS(
+ f"Migration complete: {processed_versions} block versions processed, "
+ f"{processed_components} total components processed, {errors} errors"
+ )
+ )
diff --git a/openedx/core/djangoapps/xblock/management/commands/update_lc_course.py b/openedx/core/djangoapps/xblock/management/commands/update_lc_course.py
new file mode 100644
index 000000000000..3226673d90fb
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/management/commands/update_lc_course.py
@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from opaque_keys.edx.keys import CourseKey
+
+from ...api import update_learning_core_course
+
+class Command(BaseCommand):
+ """
+ Invoke with:
+
+ python manage.py cms update_lc_course
+ """
+ help = "Updates a single course to read from a hybrid LC/Modulestore interface."
+
+ def add_arguments(self, parser):
+ parser.add_argument('course_key')
+
+ def handle(self, *args, **options):
+ course_key = CourseKey.from_string(options['course_key'])
+ update_learning_core_course(course_key)
diff --git a/openedx/core/djangoapps/xblock/migrations/0001_add_xblock_version_field_data.py b/openedx/core/djangoapps/xblock/migrations/0001_add_xblock_version_field_data.py
new file mode 100644
index 000000000000..5e189b888fb7
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0001_add_xblock_version_field_data.py
@@ -0,0 +1,29 @@
+# Generated by Django 4.2.22 on 2025-06-16 18:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='XBlockVersionFieldData',
+ fields=[
+ ('publishable_entity_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentityversion')),
+ ('content', jsonfield.fields.JSONField(default=dict, help_text='XBlock content scope fields as JSON')),
+ ('settings', jsonfield.fields.JSONField(default=dict, help_text='XBlock settings scope fields as JSON')),
+ ],
+ options={
+ 'verbose_name': 'XBlock Version Field Data',
+ 'verbose_name_plural': 'XBlock Version Field Data',
+ },
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0002_learningcorecoursestructure_and_more.py b/openedx/core/djangoapps/xblock/migrations/0002_learningcorecoursestructure_and_more.py
new file mode 100644
index 000000000000..19207cf017ae
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0002_learningcorecoursestructure_and_more.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.22 on 2025-06-20 01:21
+
+from django.db import migrations, models
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0001_add_xblock_version_field_data'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='LearningCoreCourseStructure',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('course_key', opaque_keys.edx.django.models.CourseKeyField(max_length=255)),
+ ('structure', models.BinaryField()),
+ ],
+ ),
+ migrations.AddConstraint(
+ model_name='learningcorecoursestructure',
+ constraint=models.UniqueConstraint(models.F('course_key'), name='xblock_lccs_uniq_course_key'),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0003_xblockversionfielddata_definition_object_id.py b/openedx/core/djangoapps/xblock/migrations/0003_xblockversionfielddata_definition_object_id.py
new file mode 100644
index 000000000000..31ab67fdf139
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0003_xblockversionfielddata_definition_object_id.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.22 on 2025-06-20 02:05
+
+from django.db import migrations, models
+import openedx.core.djangoapps.xblock.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0002_learningcorecoursestructure_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='xblockversionfielddata',
+ name='definition_object_id',
+ field=models.CharField(
+ default=openedx.core.djangoapps.xblock.models.XBlockVersionFieldData.generate_object_id_str,
+ max_length=24,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0004_generate_def_ids.py b/openedx/core/djangoapps/xblock/migrations/0004_generate_def_ids.py
new file mode 100644
index 000000000000..75bc5b451584
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0004_generate_def_ids.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.22 on 2025-06-20 02:10
+
+from django.db import migrations
+from bson import ObjectId
+
+
+def gen_definition_object_ids(apps, schema_editor):
+ XBlockVersionFieldData = apps.get_model("xblock_new", "XBlockVersionFieldData")
+ for row in XBlockVersionFieldData.objects.all():
+ row.definition_object_id = str(ObjectId())
+ row.save(update_fields=["definition_object_id"])
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0003_xblockversionfielddata_definition_object_id'),
+ ]
+
+ operations = [
+ migrations.RunPython(gen_definition_object_ids, reverse_code=migrations.RunPython.noop),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0005_unique_def_ids.py b/openedx/core/djangoapps/xblock/migrations/0005_unique_def_ids.py
new file mode 100644
index 000000000000..ecc32010801e
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0005_unique_def_ids.py
@@ -0,0 +1,24 @@
+# Generated by Django 4.2.22 on 2025-06-20 14:07
+
+from django.db import migrations, models
+import openedx.core.djangoapps.xblock.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0004_generate_def_ids'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="XBlockVersionFieldData",
+ name="definition_object_id",
+ field=models.CharField(
+ max_length=24,
+ unique=True,
+ null=False,
+ default=openedx.core.djangoapps.xblock.models.XBlockVersionFieldData.generate_object_id_str,
+ )
+ )
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0006_block_learningcontext_block_learning_context.py b/openedx/core/djangoapps/xblock/migrations/0006_block_learningcontext_block_learning_context.py
new file mode 100644
index 000000000000..801d3c8c197a
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0006_block_learningcontext_block_learning_context.py
@@ -0,0 +1,38 @@
+# Generated by Django 4.2.22 on 2025-06-21 02:57
+
+from django.db import migrations, models
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ('xblock_new', '0005_unique_def_ids'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Block',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)),
+ ('children', models.JSONField(default=None, null=True)),
+ ('entity', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentity')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='LearningContext',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', opaque_keys.edx.django.models.LearningContextKeyField(max_length=255, unique=True)),
+ ('root', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='xblock_new.block')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='block',
+ name='learning_context',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='xblock_new.learningcontext'),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0007_remove_block_children_and_more.py b/openedx/core/djangoapps/xblock/migrations/0007_remove_block_children_and_more.py
new file mode 100644
index 000000000000..792735db52e7
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0007_remove_block_children_and_more.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.22 on 2025-06-21 03:24
+
+from django.db import migrations
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0006_block_learningcontext_block_learning_context'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='block',
+ name='children',
+ ),
+ migrations.AddField(
+ model_name='xblockversionfielddata',
+ name='children',
+ field=jsonfield.fields.JSONField(default=None, help_text='XBlock children scope fields as JSON'),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0008_learningcorelearningcontext_and_more.py b/openedx/core/djangoapps/xblock/migrations/0008_learningcorelearningcontext_and_more.py
new file mode 100644
index 000000000000..e2d8b5628e94
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0008_learningcorelearningcontext_and_more.py
@@ -0,0 +1,32 @@
+# Generated by Django 4.2.22 on 2025-06-21 03:39
+
+from django.db import migrations, models
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0007_remove_block_children_and_more'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='LearningCoreLearningContext',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', opaque_keys.edx.django.models.LearningContextKeyField(max_length=255, unique=True)),
+ ('use_learning_core', models.BooleanField(default=True)),
+ ('root', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='xblock_new.block')),
+ ],
+ ),
+ migrations.AlterField(
+ model_name='block',
+ name='learning_context',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='xblock_new.learningcorelearningcontext'),
+ ),
+ migrations.DeleteModel(
+ name='LearningContext',
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0009_alter_block_entity.py b/openedx/core/djangoapps/xblock/migrations/0009_alter_block_entity.py
new file mode 100644
index 000000000000..385d9851605e
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0009_alter_block_entity.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.22 on 2025-06-21 04:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ('xblock_new', '0008_learningcorelearningcontext_and_more'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='block',
+ name='entity',
+ field=models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentity'),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0010_alter_block_learning_context.py b/openedx/core/djangoapps/xblock/migrations/0010_alter_block_learning_context.py
new file mode 100644
index 000000000000..edcdf9e24fbf
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0010_alter_block_learning_context.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.22 on 2025-06-21 05:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0009_alter_block_entity'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='block',
+ name='learning_context',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='blocks', to='xblock_new.learningcorelearningcontext'),
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/0011_remove_learningcorelearningcontext_root.py b/openedx/core/djangoapps/xblock/migrations/0011_remove_learningcorelearningcontext_root.py
new file mode 100644
index 000000000000..9e33e1964f8e
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/migrations/0011_remove_learningcorelearningcontext_root.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.22 on 2025-06-23 13:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('xblock_new', '0010_alter_block_learning_context'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='learningcorelearningcontext',
+ name='root',
+ ),
+ ]
diff --git a/openedx/core/djangoapps/xblock/migrations/__init__.py b/openedx/core/djangoapps/xblock/migrations/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/openedx/core/djangoapps/xblock/models.py b/openedx/core/djangoapps/xblock/models.py
new file mode 100644
index 000000000000..c323e0cbf525
--- /dev/null
+++ b/openedx/core/djangoapps/xblock/models.py
@@ -0,0 +1,82 @@
+"""Models for XBlock runtime."""
+from django.db import models
+from jsonfield.fields import JSONField
+from openedx_learning.api.authoring_models import PublishableEntity, PublishableEntityVersionMixin
+from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField, UsageKeyField
+
+import bson
+
+class XBlockVersionFieldData(PublishableEntityVersionMixin):
+ """
+ Optimized storage of parsed XBlock field data.
+
+ This model stores the parsed field data from XBlock OLX content to avoid repeated XML parsing on every block load.
+ It maintains a 1:1 relationship with PublishableEntityVersion and caches both content and settings scope fields.
+ When block field data changes, a new ComponentVersion and corresponding XBlockVersionFieldData record
+ are created by the LearningCoreXBlockRuntime.
+ """
+ def generate_object_id_str():
+ # TODO: This should be a proper field type
+ return str(bson.ObjectId())
+
+ # This exists entirely for the Modulestore shim layer. We can get rid of it
+ # when we've moved entirely off of SplitModuleStore.
+ definition_object_id = models.CharField(
+ max_length=24,
+ null=False,
+ unique=True,
+ default=generate_object_id_str,
+ )
+
+ content = JSONField(
+ default=dict,
+ help_text="XBlock content scope fields as JSON"
+ )
+
+ settings = JSONField(
+ default=dict,
+ help_text="XBlock settings scope fields as JSON"
+ )
+
+ children = JSONField(
+ default=None,
+ help_text="XBlock children scope fields as JSON"
+ )
+
+ class Meta:
+ verbose_name = "XBlock Version Field Data"
+ verbose_name_plural = "XBlock Version Field Data"
+
+ def __str__(self):
+ return f"Field data for {self.publishable_entity_version}"
+
+
+class LearningCoreCourseStructure(models.Model):
+ course_key = CourseKeyField(max_length=255)
+ structure = models.BinaryField()
+
+ class Meta:
+ constraints = [
+ models.UniqueConstraint("course_key", name="xblock_lccs_uniq_course_key")
+ ]
+
+
+class LearningCoreLearningContext(models.Model):
+ key = LearningContextKeyField(max_length=255, unique=True)
+
+ # This is a way for us to turn off LC as a backend both for rollback
+ # purposes, but also to temporarily disable when doing a re-import.
+ use_learning_core = models.BooleanField(default=True)
+
+ def __str__(self):
+ return str(self.key)
+
+
+class Block(models.Model):
+ learning_context = models.ForeignKey(
+ LearningCoreLearningContext,
+ on_delete=models.CASCADE,
+ related_name="blocks",
+ )
+ key = UsageKeyField(max_length=255, unique=True)
+ entity = models.OneToOneField(PublishableEntity, on_delete=models.RESTRICT)
diff --git a/openedx/core/djangoapps/xblock/rest_api/views.py b/openedx/core/djangoapps/xblock/rest_api/views.py
index a71fb6cfd5e2..881821e7ca46 100644
--- a/openedx/core/djangoapps/xblock/rest_api/views.py
+++ b/openedx/core/djangoapps/xblock/rest_api/views.py
@@ -37,7 +37,7 @@
render_block_view as _render_block_view,
get_block_olx,
)
-from ..utils import validate_secure_token_for_xblock_handler
+from ..utils import get_explicitly_set_fields_by_scope, validate_secure_token_for_xblock_handler
from .url_converters import VersionConverter
from .serializers import XBlockOlxSerializer
@@ -294,7 +294,7 @@ def get(self, request, usage_key: UsageKeyV2, version: LatestVersion | int = Lat
# fields except "data".
block_dict = {
"display_name": get_block_display_name(block), # note this is also present in metadata
- "metadata": self.get_explicitly_set_fields_by_scope(block, Scope.settings),
+ "metadata": get_explicitly_set_fields_by_scope(block, Scope.settings),
}
if hasattr(block, "data"):
block_dict["data"] = block.data
@@ -312,8 +312,8 @@ def post(self, request, usage_key, version: LatestVersion | int = LatestVersion.
data = request.data.get("data")
metadata = request.data.get("metadata")
- old_metadata = self.get_explicitly_set_fields_by_scope(block, Scope.settings)
- old_content = self.get_explicitly_set_fields_by_scope(block, Scope.content)
+ old_metadata = get_explicitly_set_fields_by_scope(block, Scope.settings)
+ old_content = get_explicitly_set_fields_by_scope(block, Scope.content)
# only update data if it was passed
if data is not None:
@@ -349,23 +349,8 @@ def post(self, request, usage_key, version: LatestVersion | int = LatestVersion.
block_dict = {
"id": str(block.usage_key),
"display_name": get_block_display_name(block), # note this is also present in metadata
- "metadata": self.get_explicitly_set_fields_by_scope(block, Scope.settings),
+ "metadata": get_explicitly_set_fields_by_scope(block, Scope.settings),
}
if hasattr(block, "data"):
block_dict["data"] = block.data
return Response(block_dict)
-
- def get_explicitly_set_fields_by_scope(self, block, scope=Scope.content):
- """
- Get a dictionary of the fields for the given scope which are set explicitly on the given xblock.
-
- (Including any set to None.)
- """
- result = {}
- for field in block.fields.values(): # lint-amnesty, pylint: disable=no-member
- if field.scope == scope and field.is_set_on(block):
- try:
- result[field.name] = field.read_json(block)
- except TypeError as exc:
- raise TypeError(f"Unable to read field {field.name} from block {block.usage_key}") from exc
- return result
diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py
index 5f9cba6c3a22..42e67254b090 100644
--- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py
+++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py
@@ -6,6 +6,7 @@
import logging
from collections import defaultdict
from datetime import datetime, timezone
+from typing import TYPE_CHECKING
from urllib.parse import unquote
from django.core.exceptions import ObjectDoesNotExist, ValidationError
@@ -25,10 +26,14 @@
from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core
from openedx.core.lib.xblock_serializer.data import StaticFile
from ..data import AuthoredDataMode, LatestVersion
-from ..utils import get_auto_latest_version
+from ..models import XBlockVersionFieldData
+from ..utils import get_auto_latest_version, get_explicitly_set_fields_by_scope
from ..learning_context.manager import get_learning_context_impl
from .runtime import XBlockRuntime
+if TYPE_CHECKING:
+ from openedx_learning.api.authoring_models import PublishableEntityVersionMixin
+
log = logging.getLogger(__name__)
@@ -42,6 +47,9 @@ class LearningCoreFieldData(FieldData):
``NotImplementedError``. This class does NOT support the parent and children
scopes.
+ This class includes optimization for loading field data from the
+ XBlockVersionFieldData model to avoid parsing OLX on every block load.
+
LearningCoreFieldData should only live for the duration of one request. The
interaction between LearningCoreXBlockRuntime and LearningCoreFieldData is
as follows:
@@ -110,9 +118,13 @@ def get(self, block, name):
usage_key = block.scope_ids.usage_id
return self.field_data[usage_key][name]
- def set(self, block, name, value):
+ def set(self, block, name, value) -> bool:
"""
Set a field for a block to a value.
+
+ Returns:
+ True if the value was changed.
+ False if the field was already set to the same value.
"""
self._check_field(block, name)
usage_key = block.scope_ids.usage_id
@@ -121,10 +133,11 @@ def set(self, block, name, value):
# without doing anything.
block_fields = self.field_data[usage_key]
if (name in block_fields) and (block_fields[name] == value):
- return
+ return False
block_fields[name] = value
self.changed.add(usage_key)
+ return True
def has_changes(self, block):
"""
@@ -146,8 +159,11 @@ def _getfield(self, block, name):
def _check_field(self, block, name):
"""
- Given a block and the name of one of its fields, check that we will be
- able to read/write it.
+ Given a block and the name of one of its fields, check that we will be able to read/write it.
+
+ Raises:
+ KeyError: If the field does not exist for the block.
+ NotImplementedError: if the field exists but its scope is not supported.
"""
field = self._getfield(block, name)
if field.scope not in (Scope.content, Scope.settings):
@@ -157,6 +173,47 @@ def _check_field(self, block, name):
" and settings scopes."
)
+ def load_field_data(self, block: XBlock, field_data: XBlockVersionFieldData):
+ """
+ Populate the block with field data from the database.
+
+ Args:
+ block: The XBlock instance to load data into.
+ field_data: The XBlockVersionFieldData instance containing the field data.
+ """
+ for field_name, value in {**field_data.content, **field_data.settings}.items():
+ try:
+ # Use the `set` method to ensure field validation and scope checks.
+ if self.set(block, field_name, value):
+ # Also set the value in the block instance.
+ setattr(block, field_name, value)
+ except (KeyError, NotImplementedError):
+ log.warning("Skipping unsupported field %s for block %s", field_name, block.scope_ids.usage_id)
+
+ def save_field_data(self, block: XBlock, component_version: PublishableEntityVersionMixin):
+ """
+ Persist field data to the database model for future loads.
+
+ Args:
+ block: The XBlock instance to extract field data from.
+ component_version: The version to associate with the field data.
+ """
+ content_fields = get_explicitly_set_fields_by_scope(block, Scope.content)
+ settings_fields = get_explicitly_set_fields_by_scope(block, Scope.settings)
+
+ try:
+ XBlockVersionFieldData.objects.create(
+ publishable_entity_version=component_version.publishable_entity_version,
+ content=content_fields,
+ settings=settings_fields,
+ )
+ except Exception as e: # pylint: disable=broad-exception-caught
+ # This method may be triggered while loading an XBlock, so we don't want to raise an error.
+ log.exception(
+ "Failed to save field data for component version %s: %s",
+ component_version.pk, e,
+ )
+
class LearningCoreXBlockRuntime(XBlockRuntime):
"""
@@ -167,14 +224,18 @@ class LearningCoreXBlockRuntime(XBlockRuntime):
(eventually) asset storage.
"""
+ authored_data_store: LearningCoreFieldData
+
def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion = LatestVersion.AUTO):
"""
Fetch an XBlock from Learning Core data models.
- This method will find the OLX for the content in Learning Core, parse it
- into an XBlock (with mixins) instance, and properly initialize our
- internal LearningCoreFieldData instance with the field values from the
- parsed OLX.
+ This method will try to load the field data from the database.
+
+ If the field data is not found, it will find the OLX for the content in Learning Core, parse it into an XBlock
+ (with mixins) instance, and properly initialize our internal LearningCoreFieldData instance with the field
+ values from the parsed OLX. Then, it will save the current field data from the parsed OLX into the database
+ for future loads.
"""
# We can do this more efficiently in a single query later, but for now
# just get it the easy way.
@@ -193,37 +254,47 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion
if component_version is None:
raise NoSuchUsage(usage_key)
- content = component_version.contents.get(
- componentversioncontent__key="block.xml"
- )
- xml_node = etree.fromstring(content.text)
block_type = usage_key.block_type
keys = ScopeIds(self.user_id, block_type, None, usage_key)
+ block_class = self.mixologist.mix(self.load_block_type(block_type))
- if xml_node.get("url_name", None):
- log.warning("XBlock at %s should not specify an old-style url_name attribute.", usage_key)
+ # Try to load field data from the database.
+ try:
+ field_data = component_version.publishable_entity_version.xblockversionfielddata # type: ignore
+ block = self.construct_xblock_from_class(block_class, keys)
+ self.authored_data_store.load_field_data(block, field_data)
- block_class = self.mixologist.mix(self.load_block_type(block_type))
+ # Retrieve field data from XML and save it to the database.
+ except ObjectDoesNotExist:
+ content = component_version.contents.get(
+ componentversioncontent__key="block.xml"
+ )
+ xml_node = etree.fromstring(content.text)
- if hasattr(block_class, 'parse_xml_new_runtime'):
- # This is a (former) XModule with messy XML parsing code; let its parse_xml() method continue to work
- # as it currently does in the old runtime, but let this parse_xml_new_runtime() method parse the XML in
- # a simpler way that's free of tech debt, if defined.
- # In particular, XmlMixin doesn't play well with this new runtime, so this is mostly about
- # bypassing that mixin's code.
- # When a former XModule no longer needs to support the old runtime, its parse_xml_new_runtime method
- # should be removed and its parse_xml() method should be simplified to just call the super().parse_xml()
- # plus some minor additional lines of code as needed.
- block = block_class.parse_xml_new_runtime(xml_node, runtime=self, keys=keys)
- else:
- block = block_class.parse_xml(xml_node, runtime=self, keys=keys)
+ if xml_node.get("url_name", None):
+ log.warning("XBlock at %s should not specify an old-style url_name attribute.", usage_key)
+
+ if hasattr(block_class, 'parse_xml_new_runtime'):
+ # This is a (former) XModule with messy XML parsing code; let its parse_xml() method continue to work
+ # as it currently does in the old runtime, but let this parse_xml_new_runtime() method parse the XML in
+ # a simpler way that's free of tech debt, if defined.
+ # In particular, XmlMixin doesn't play well with this new runtime, so this is mostly about
+ # bypassing that mixin's code.
+ # When a former XModule no longer needs to support the old runtime, its parse_xml_new_runtime method
+ # should be removed and its parse_xml() method should be simplified to just call the super().parse_xml()
+ # plus some minor additional lines of code as needed.
+ block = block_class.parse_xml_new_runtime(xml_node, runtime=self, keys=keys)
+ else:
+ block = block_class.parse_xml(xml_node, runtime=self, keys=keys)
+
+ # Update field data with parsed values. We can't call .save() because it will call save_block(), below.
+ block.force_save_fields(block._get_fields_to_save()) # pylint: disable=protected-access
+
+ self.authored_data_store.save_field_data(block, component_version) # type: ignore
# Store the version request on the block so we can retrieve it when needed for generating handler URLs etc.
block._runtime_requested_version = version # pylint: disable=protected-access
- # Update field data with parsed values. We can't call .save() because it will call save_block(), below.
- block.force_save_fields(block._get_fields_to_save()) # pylint: disable=protected-access
-
# We've pre-loaded the fields for this block, so the FieldData shouldn't
# consider these values "changed" in its sense of "you have to persist
# these because we've altered the field values from what was stored".
@@ -304,7 +375,7 @@ def save_block(self, block):
text=serialized.olx_str,
created=now,
)
- authoring_api.create_next_version(
+ new_component_version = authoring_api.create_next_version(
component.pk,
title=block.display_name,
content_to_replace={
@@ -313,6 +384,10 @@ def save_block(self, block):
created=now,
created_by=self.user.id if self.user else None
)
+
+ # Create the field data record for the new version.
+ self.authored_data_store.save_field_data(block, new_component_version)
+
self.authored_data_store.mark_unchanged(block)
# Signal that we've modified this block
diff --git a/openedx/core/djangoapps/xblock/utils.py b/openedx/core/djangoapps/xblock/utils.py
index b4ae054cf498..fa7b7e4de9a7 100644
--- a/openedx/core/djangoapps/xblock/utils.py
+++ b/openedx/core/djangoapps/xblock/utils.py
@@ -10,6 +10,7 @@
import crum
from django.conf import settings
+from xblock.fields import Scope
from openedx.core.djangoapps.xblock.apps import get_xblock_app_config
@@ -186,3 +187,19 @@ def get_auto_latest_version(version: int | LatestVersion) -> int | LatestVersion
else LatestVersion.PUBLISHED
)
return version
+
+
+def get_explicitly_set_fields_by_scope(block, scope=Scope.content):
+ """
+ Get a dictionary of the fields for the given scope which are set explicitly on the given xblock.
+
+ (Including any set to None.)
+ """
+ result = {}
+ for field in block.fields.values(): # lint-amnesty, pylint: disable=no-member
+ if field.scope == scope and field.is_set_on(block):
+ try:
+ result[field.name] = field.read_json(block)
+ except TypeError as exc:
+ raise TypeError(f"Unable to read field {field.name} from block {block.usage_key}") from exc
+ return result
diff --git a/openedx/core/types/user.py b/openedx/core/types/user.py
index 9eb63edba358..e680a33b42ae 100644
--- a/openedx/core/types/user.py
+++ b/openedx/core/types/user.py
@@ -7,4 +7,5 @@
import django.contrib.auth.models
+AuthUser: t.TypeAlias = django.contrib.auth.models.User
User: t.TypeAlias = django.contrib.auth.models.User | django.contrib.auth.models.AnonymousUser
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
index 3badf5490964..7804eaf3bca7 100644
--- a/requirements/constraints.txt
+++ b/requirements/constraints.txt
@@ -67,11 +67,6 @@ libsass==0.10.0
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126
numpy<2.0.0
-# Date: 2023-09-18
-# pinning this version to avoid updates while the library is being developed
-# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269
-openedx-learning==0.26.0
-
# Date: 2023-11-29
# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35268
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index ce3567fcebdf..10c7fa8aa27b 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -840,7 +840,7 @@ openedx-filters==2.1.0
# ora2
openedx-forum==0.3.0
# via -r requirements/edx/kernel.in
-openedx-learning==0.26.0
+-e git+https://github.com/openedx/openedx-learning.git@lc-course-prototype#egg=openedx-learning
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/kernel.in
diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
index 66201eb8771f..853f829fc85a 100644
--- a/requirements/edx/development.txt
+++ b/requirements/edx/development.txt
@@ -1400,7 +1400,7 @@ openedx-forum==0.3.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
-openedx-learning==0.26.0
+-e git+https://github.com/openedx/openedx-learning.git@lc-course-prototype#egg=openedx-learning
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/doc.txt
diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt
index 675592a5360a..89f102ac8202 100644
--- a/requirements/edx/doc.txt
+++ b/requirements/edx/doc.txt
@@ -1005,7 +1005,7 @@ openedx-filters==2.1.0
# ora2
openedx-forum==0.3.0
# via -r requirements/edx/base.txt
-openedx-learning==0.26.0
+-e git+https://github.com/openedx/openedx-learning.git@lc-course-prototype#egg=openedx-learning
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
diff --git a/requirements/edx/github.in b/requirements/edx/github.in
index 6ec36d3a0681..666c37fd5e1c 100644
--- a/requirements/edx/github.in
+++ b/requirements/edx/github.in
@@ -73,6 +73,7 @@
#
# * Organize the URL into one of the two categories below:
+-e git+https://github.com/openedx/openedx-learning.git@lc-course-prototype#egg=openedx-learning
##############################################################################
# Release candidates being tested.
diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
index 0bbbe3f30180..d002f38b768a 100644
--- a/requirements/edx/testing.txt
+++ b/requirements/edx/testing.txt
@@ -1064,7 +1064,7 @@ openedx-filters==2.1.0
# ora2
openedx-forum==0.3.0
# via -r requirements/edx/base.txt
-openedx-learning==0.26.0
+-e git+https://github.com/openedx/openedx-learning.git@lc-course-prototype#egg=openedx-learning
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
diff --git a/xmodule/modulestore/split_mongo/mongo_connection.py b/xmodule/modulestore/split_mongo/mongo_connection.py
index 4654511cfe01..36b7019b593a 100644
--- a/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -22,6 +22,7 @@
from pymongo.errors import DuplicateKeyError # pylint: disable=unused-import
from edx_django_utils import monitoring
from edx_django_utils.cache import RequestCache
+from opaque_keys.edx.keys import CourseKey
from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex
from xmodule.exceptions import HeartbeatFailure
@@ -153,6 +154,13 @@ def structure_from_mongo(structure, course_context=None):
course_context (CourseKey): For metrics gathering, the CourseKey
for the course that this data is being processed for.
"""
+ import pprint
+
+ #with open("raw_struct.txt", "w") as struct_file:
+ # struct_file.write(f"Course: {course_context}\n\n")
+ # printer = pprint.PrettyPrinter(indent=2, stream=struct_file)
+ # printer.pprint(structure)
+
with TIMER.timer('structure_from_mongo', course_context) as tagger:
tagger.measure('blocks', len(structure['blocks']))
@@ -164,6 +172,11 @@ def structure_from_mongo(structure, course_context=None):
new_blocks[BlockKey(block['block_type'], block.pop('block_id'))] = BlockData(**block)
structure['blocks'] = new_blocks
+ #with open("struct.txt", "w") as struct_file:
+ # struct_file.write(f"Course: {course_context}\n\n")
+ # printer = pprint.PrettyPrinter(indent=2, stream=struct_file)
+ # printer.pprint(structure)
+
return structure
@@ -341,13 +354,25 @@ def get_structure(self, key, course_context=None):
cache = CourseStructureCache()
structure = cache.get(key, course_context)
+
+ structure = None # force cache miss for now
+
tagger_get_structure.tag(from_cache=str(bool(structure)).lower())
if not structure:
# Always log cache misses, because they are unexpected
tagger_get_structure.sample_rate = 1
with TIMER.timer("get_structure.find_one", course_context) as tagger_find_one:
- doc = self.structures.find_one({'_id': key})
+ # Reminder: course_context includes the branch information
+ from openedx.core.djangoapps.xblock.api import get_structure_for_course, learning_core_backend_enabled_for_course
+
+ if learning_core_backend_enabled_for_course(course_context):
+ log.info(f"Getting Structure doc from Learning Core: {course_context}: {key}")
+ doc = get_structure_for_course(course_context)
+ else:
+ log.info(f"Getting Structure doc from ModuleStore: {course_context}: {key}")
+ doc = self.structures.find_one({'_id': key})
+
if doc is None:
log.warning(
"doc was None when attempting to retrieve structure for item with key %s",
@@ -537,12 +562,31 @@ def delete_course_index(self, course_key):
}
return self.course_index.delete_one(query)
- def get_definition(self, key, course_context=None):
+ def get_definition(self, key, course_context: CourseKey | None=None):
"""
Get the definition from the persistence mechanism whose id is the given key
"""
+ from openedx.core.djangoapps.xblock.api import get_definition_doc, learning_core_backend_enabled_for_course
+
+ log.info(f"Fetching Definition: {key}")
with TIMER.timer("get_definition", course_context) as tagger:
- definition = self.definitions.find_one({'_id': key})
+ # Note that sometimes course_context comes in with version/branch
+ # information, and sometimes it doesn't. So we can't rely on that to
+ # only enable the LC shim for the published branch. We also can't do
+ # switching from Studio to LMS because Studio needs to build things
+ # off of course publish.
+ definition = None
+ if learning_core_backend_enabled_for_course(course_context):
+ log.info(f"Getting Definition doc from Learning Core: {course_context}: {key}")
+ definition = get_definition_doc(key)
+
+ if not definition:
+ # This fallback exists for the random standalone blocks that
+ # courses expect. Change this to an "else" branch when we're
+ # importing those for real.
+ log.info(f"Getting Definition doc from ModuleStore: {course_context}: {key}")
+ definition = self.definitions.find_one({'_id': key})
+
tagger.measure("fields", len(definition['fields']))
tagger.tag(block_type=definition['block_type'])
return definition
@@ -551,6 +595,7 @@ def get_definitions(self, definitions, course_context=None):
"""
Retrieve all definitions listed in `definitions`.
"""
+ log.info(f"Fetching Definitions: {definitions}")
with TIMER.timer("get_definitions", course_context) as tagger:
tagger.measure('definitions', len(definitions))
definitions = self.definitions.find({'_id': {'$in': definitions}})
diff --git a/xmodule/modulestore/split_mongo/split.py b/xmodule/modulestore/split_mongo/split.py
index e69a2ca0e53c..60d68f4b2af8 100644
--- a/xmodule/modulestore/split_mongo/split.py
+++ b/xmodule/modulestore/split_mongo/split.py
@@ -432,6 +432,7 @@ def get_definitions(self, course_key, ids):
ids (list): A list of definition ids
"""
definitions = []
+ print(ids)
ids = set(ids)
bulk_write_record = self._get_bulk_ops_record(course_key)