Prototype Writing to SplitModuleStore-shim layer from Learning Core Libraries

cms/djangoapps/modulestore_migrator/constants.py

@@ -2,5 +2,6 @@
 Constants
 """
 
-CONTENT_STAGING_PURPOSE_PREFIX = "modulestore_migrator"
-CONTENT_STAGING_PURPOSE_TEMPLATE = CONTENT_STAGING_PURPOSE_PREFIX + "({source_key})"
+CONTENT_STAGING_PURPOSE = "modulestore_migrator"
+CONTENT_STAGING_PURPOSE_META = "modulestore_migrator_meta"
+META_BLOCK_TYPES: list[str] = ["about", "course_info", "static_tab"]

cms/djangoapps/modulestore_migrator/data.py

@@ -23,8 +23,15 @@ class CompositionLevel(Enum):
     Section = ContainerType.Section.value
     OutlineRoot = ContainerType.OutlineRoot.value
 
+    # Import the outline root, as well as the weird meta blocks (about,
+    # course_info, static_tab) that exist as parent-less peers of the outline
+    # root, and get/create the Course instance. Unlike the other
+    # CompositionLevels, this level does not correspond to any particular kind of
+    # publishable entity.
+    CourseRun = "course_run"
+
     @property
-    def is_container(self) -> bool:
+    def is_complex(self) -> bool:
         return self is not self.Component
 
     def is_higher_than(self, other: 'CompositionLevel') -> bool:

cms/djangoapps/modulestore_migrator/tasks.py

@@ -26,21 +26,27 @@
 from openedx_learning.api.authoring_models import (
     Collection,
     Component,
-    ContainerVersion,
+    Course,
+    CatalogCourse,
     LearningPackage,
     PublishableEntity,
     PublishableEntityVersion,
 )
 from user_tasks.tasks import UserTask, UserTaskStatus
+from xblock.core import XBlock
 
 from openedx.core.djangoapps.content_libraries.api import ContainerType
 from openedx.core.djangoapps.content_libraries import api as libraries_api
 from openedx.core.djangoapps.content_libraries.models import ContentLibrary
 from openedx.core.djangoapps.content_staging import api as staging_api
+from openedx.core.djangoapps.xblock import models as xblock_models
+from openedx.core.djangoapps.xblock.api import create_xblock_field_data_for_container
+
 from xmodule.modulestore import exceptions as modulestore_exceptions
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.mixed import MixedModuleStore
 
-from .constants import CONTENT_STAGING_PURPOSE_TEMPLATE
+from .constants import CONTENT_STAGING_PURPOSE, CONTENT_STAGING_PURPOSE_META, META_BLOCK_TYPES
 from .data import CompositionLevel
 from .models import ModulestoreSource, ModulestoreMigration, ModulestoreBlockSource, ModulestoreBlockMigration
 
@@ -63,6 +69,7 @@ class MigrationStep(Enum):
     PARSING = 'Parsing staged OLX'
     IMPORTING_ASSETS = 'Importing staged files and resources'
     IMPORTING_STRUCTURE = 'Importing staged content structure'
+    IMPORTING_META = 'Importing course info and other meta-components'
     UNSTAGING = 'Cleaning staged content'
     MAPPING_OLD_TO_NEW = 'Saving map of legacy content to migrated content'
     FORWARDING = 'Forwarding legacy content to migrated content'
@@ -110,6 +117,7 @@ def migrate_from_modulestore(
 
     status: UserTaskStatus = self.status
     status.set_state(MigrationStep.VALIDATING_INPUT.value)
+    comp_level = CompositionLevel(composition_level)
     try:
         source = ModulestoreSource.objects.get(pk=source_pk)
         target_package = LearningPackage.objects.get(pk=target_package_pk)
@@ -118,8 +126,16 @@ def migrate_from_modulestore(
     except ObjectDoesNotExist as exc:
         status.fail(str(exc))
         return
+
+    course_lc_learning_context = None
     if isinstance(source.key, CourseLocator):
         source_root_usage_key = source.key.make_usage_key('course', 'course')
+
+        # Support SplitModuleStore shim from Learning Core, force it off for now because we need to build it using ModuleStore
+        course_lc_learning_context, _created = xblock_models.LearningCoreLearningContext.objects.get_or_create(key=source.key)
+        course_lc_learning_context.use_learning_core = False
+        course_lc_learning_context.save()
+
     elif isinstance(source.key, LibraryLocator):
         source_root_usage_key = source.key.make_usage_key('library', 'library')
     else:
@@ -128,6 +144,7 @@ def migrate_from_modulestore(
             "Source key must reference a course or a legacy library."
         )
         return
+
     migration = ModulestoreMigration.objects.create(
         source=source,
         composition_level=composition_level,
@@ -161,23 +178,37 @@ def migrate_from_modulestore(
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.LOADING)
+    store: MixedModuleStore = modulestore()
     try:
-        legacy_root = modulestore().get_item(source_root_usage_key)
+        legacy_root = store.get_item(source_root_usage_key)
     except modulestore_exceptions.ItemNotFoundError as exc:
         status.fail(f"Failed to load source item '{source_root_usage_key}' from ModuleStore: {exc}")
         return
     if not legacy_root:
         status.fail(f"Could not find source item '{source_root_usage_key}' in ModuleStore")
         return
+    meta_blocks: list[XBlock] = (
+        store.get_items(source.key, qualifiers={"category": {"$in": META_BLOCK_TYPES}})
+        if comp_level == CompositionLevel.CourseRun
+        else []
+    )
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.STAGING.value)
     staged_content = staging_api.stage_xblock_temporarily(
         block=legacy_root,
         user_id=status.user.pk,
-        purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source.key),
+        purpose=CONTENT_STAGING_PURPOSE,
     )
     migration.staged_content = staged_content
+    staged_meta_contents = [
+        staging_api.stage_xblock_temporarily(
+            block=meta_block,
+            user_id=status.user.pk,
+            purpose=CONTENT_STAGING_PURPOSE_META,
+        )
+        for meta_block in meta_blocks
+    ]
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.PARSING.value)
@@ -186,12 +217,26 @@ def migrate_from_modulestore(
         root_node = etree.fromstring(staged_content.olx, parser=parser)
     except etree.ParseError as exc:
         status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}")
+        return
+    meta_nodes = []
+    for staged_meta_content in staged_meta_contents:
+        meta_parser = etree.XMLParser(strip_cdata=False)
+        try:
+            meta_nodes.append(etree.fromstring(staged_meta_content.olx, parser=meta_parser))
+        except etree.ParseError as exc:
+            status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}")
+            return
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.IMPORTING_ASSETS.value)
     content_by_filename: dict[str, int] = {}
     now = datetime.now(tz=timezone.utc)
-    for staged_content_file_data in staging_api.get_staged_content_static_files(staged_content.id):
+    all_static_files: list[staging_api.StagedContentFileData] = [
+        static_file
+        for staged in [staged_content, *staged_meta_contents]
+        for static_file in staging_api.get_staged_content_static_files(staged.id)
+    ]
+    for staged_content_file_data in all_static_files:
         old_path = staged_content_file_data.filename
         file_data = staging_api.get_staged_content_static_file_data(staged_content.id, old_path)
         if not file_data:
@@ -212,21 +257,39 @@ def migrate_from_modulestore(
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.IMPORTING_STRUCTURE.value)
+    now = datetime.now(timezone.utc)
     with authoring_api.bulk_draft_changes_for(migration.target.id) as change_log:
         root_migrated_node = _migrate_node(
             content_by_filename=content_by_filename,
-            source_context_key=source_root_usage_key.course_key,
+            source_context_key=source.key,
             source_node=root_node,
             target_library_key=target_library.library_key,
             target_package_id=target_package_pk,
             replace_existing=replace_existing,
-            composition_level=CompositionLevel(composition_level),
-            created_at=datetime.now(timezone.utc),
-            created_by=status.user_id,
+            composition_level=comp_level,
+            created_at=now,
+            created_by=user_id,
         )
     migration.change_log = change_log
     status.increment_completed_steps()
 
+    status.set_state(MigrationStep.IMPORTING_META.value)
+    migrated_meta_nodes: list[_MigratedNode] = [
+        _migrate_node(
+            content_by_filename=content_by_filename,
+            source_context_key=source.key,
+            source_node=meta_node,
+            target_package_id=target_package_pk,
+            target_library_key=target_library.library_key,
+            replace_existing=replace_existing,
+            composition_level=comp_level,
+            created_at=now,
+            created_by=user_id,
+        )
+        for meta_node in meta_nodes
+    ]
+    status.increment_completed_steps()
+
     status.set_state(MigrationStep.UNSTAGING.value)
     staged_content.delete()
     status.increment_completed_steps()
@@ -239,8 +302,15 @@ def migrate_from_modulestore(
     #        we did this, we'd want to make sure that the objects are actually visible
     #        to the user mid-import (via django admin, or the library interface, or even just as
     #        as a "progress bar" field in the REST API), otherwise this would be pointless.
+    migrated_umbrella = _MigratedNode(
+        # This is a block-less pseudo-node representing an umbrella containing both
+        # (a) the outline and (b) all the meta blocks. @@TODO this might be too clever
+        # to leave in the production migrator... revisit.
+        source_to_target=None,
+        children=[root_migrated_node, *migrated_meta_nodes],
+    )
     status.set_state(MigrationStep.MAPPING_OLD_TO_NEW.value)
-    block_source_keys_to_target_vers = dict(root_migrated_node.all_source_to_target_pairs())
+    block_source_keys_to_target_vers = dict(migrated_umbrella.all_source_to_target_pairs())
     ModulestoreBlockSource.objects.bulk_create(
         [
             ModulestoreBlockSource(overall_source=source, key=source_usage_key)
@@ -274,8 +344,22 @@ def migrate_from_modulestore(
         ],
     )
     block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=migration)
+
+    xblock_models.Block.objects.bulk_create(
+        [
+            xblock_models.Block(
+                learning_context=course_lc_learning_context,
+                key=block_source_key,
+                entity_id=block_target_ver.entity_id,
+            )
+            for block_source_key, block_target_ver in block_source_keys_to_target_vers.items()
+        ],
+        update_conflicts=True,
+        update_fields=["entity", "learning_context"],
+    )
     status.increment_completed_steps()
 
+
     status.set_state(MigrationStep.FORWARDING.value)
     if forward_source_to_target:
         block_sources_to_block_migrations = {
@@ -287,6 +371,24 @@ def migrate_from_modulestore(
         # ModulestoreBlockSource.objects.bulk_update(block_sources_to_block_migrations.keys(), ["forwarded"])
         source.forwarded = migration
         source.save()
+        if comp_level == CompositionLevel.CourseRun:
+            catalog_course, _ = CatalogCourse.objects.get_or_create(
+                org_id=source.key.org,
+                course_id=source.key.course,
+            )
+            try:
+                course = Course.objects.get(catalog_course=catalog_course, run=source.key.run)
+            except Course.DoesNotExist:
+                Course.objects.create(
+                    catalog_course=catalog_course,
+                    run=source.key.run,
+                    learning_package=target_package,
+                    outline_root=root_migrated_node.source_to_target[1].entity.container.outlineroot,
+                )
+            else:
+                course.learning_package = target_package
+                course.outline_root = root_migrated_node.source_to_target[1].entity.container.outlineroot
+                course.save()
     status.increment_completed_steps()
 
     status.set_state(MigrationStep.POPULATING_COLLECTION.value)
@@ -303,6 +405,10 @@ def migrate_from_modulestore(
         )
     status.increment_completed_steps()
 
+    # Now have it use our Learning Core shim for Split instead of Mongo DB
+    course_lc_learning_context.use_learning_core = True
+    course_lc_learning_context.save()
+
 
 @dataclass(frozen=True)
 class _MigratedNode:
@@ -468,7 +574,7 @@ def _migrate_container(
             entity_id=container.container_pk,
             version_num=container.draft_version_num,
         )
-    return authoring_api.create_next_container_version(
+    next_container_version = authoring_api.create_next_container_version(
         container.container_pk,
         title=title,
         entity_rows=[
@@ -478,7 +584,9 @@ def _migrate_container(
         created=created_at,
         created_by=created_by,
         container_version_cls=container_type.container_model_classes[1],
-    ).publishable_entity_version
+    )
+    create_xblock_field_data_for_container(next_container_version)
+    return next_container_version.publishable_entity_version
 
 
 def _migrate_component(

lms/djangoapps/courseware/toggles.py

@@ -201,4 +201,6 @@ def courseware_disable_navigation_sidebar_blocks_caching(course_key=None):
     """
     Return whether the courseware.disable_navigation_sidebar_blocks_caching flag is on.
     """
+    return True  # For debugging the Learning core shim proof of concept
+
     return COURSEWARE_MICROFRONTEND_NAVIGATION_SIDEBAR_BLOCKS_DISABLE_CACHING.is_enabled(course_key)

openedx/core/djangoapps/content/block_structure/manager.py

@@ -153,9 +153,11 @@ def _bulk_operations(self):
         """
         A context manager for notifying the store of bulk operations.
         """
+        from xmodule.modulestore import ModuleStoreEnum
         try:
             course_key = self.root_block_usage_key.course_key
         except AttributeError:
             course_key = None
-        with self.modulestore.bulk_operations(course_key):
-            yield
+        with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, course_key):
+            with self.modulestore.bulk_operations(course_key):
+                yield

openedx/core/djangoapps/content/block_structure/store.py

@@ -102,6 +102,8 @@ def is_up_to_date(self, root_block_usage_key, modulestore):
         Returns whether the data in storage for the given key is
         already up-to-date with the version in the given modulestore.
         """
+        return False
+
         try:
             bs_model = self._get_model(root_block_usage_key)
             root_block = modulestore.get_item(root_block_usage_key)

openedx/core/djangoapps/content/block_structure/tasks.py

@@ -59,6 +59,7 @@ def _update_course_in_cache(self, **kwargs):
     """
     Updates the course blocks (mongo -> BlockStructure) for the specified course.
     """
+    log.info("Inner _update_course_in_cache called.")
     _call_and_retry_if_needed(self, api.update_course_in_cache, **kwargs)