diff --git a/cms/envs/common.py b/cms/envs/common.py index 179eede100b1..e7a47b3ce6b0 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1811,9 +1811,21 @@ # alternative swagger generator for CMS API 'drf_spectacular', + 'openedx_events', + + # Learning Core Apps, used by v2 content libraries (content_libraries app) + "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.contents.apps.ContentsConfig", + "openedx_learning.core.publishing.apps.PublishingConfig", ] +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +} + ################# EDX MARKETING SITE ################################## diff --git a/lms/envs/common.py b/lms/envs/common.py index 0bad10728565..d64de1992531 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -40,7 +40,6 @@ # and throws spurious errors. Therefore, we disable invalid-name checking. # pylint: disable=invalid-name - import importlib.util import sys import os @@ -3316,9 +3315,22 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # Notifications 'openedx.core.djangoapps.notifications', + + 'openedx_events', + + # Learning Core Apps, used by v2 content libraries (content_libraries app) + "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.contents.apps.ContentsConfig", + "openedx_learning.core.publishing.apps.PublishingConfig", ] +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +} + ######################### CSRF ######################################### # Forwards-compatibility with Django 1.7 diff --git a/openedx/core/djangoapps/content_libraries/admin.py b/openedx/core/djangoapps/content_libraries/admin.py index 559d2471cee3..e715841187e6 100644 --- a/openedx/core/djangoapps/content_libraries/admin.py +++ b/openedx/core/djangoapps/content_libraries/admin.py @@ -2,7 +2,7 @@ Admin site for content libraries """ from django.contrib import admin -from .models import ContentLibrary, ContentLibraryPermission +from .models import ContentLibrary, ContentLibraryLearningPackage, ContentLibraryPermission class ContentLibraryPermissionInline(admin.TabularInline): @@ -40,3 +40,7 @@ def get_readonly_fields(self, request, obj=None): return ["library_key", "org", "slug", "bundle_uuid"] else: return ["library_key", ] + +@admin.register(ContentLibraryLearningPackage) +class ContentLibraryLearningPackageAdmin(admin.ModelAdmin): + pass diff --git a/openedx/core/djangoapps/content_libraries/management/commands/migrate_lib_to_learning_core.py b/openedx/core/djangoapps/content_libraries/management/commands/migrate_lib_to_learning_core.py new file mode 100644 index 000000000000..925d4f473bd0 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/management/commands/migrate_lib_to_learning_core.py @@ -0,0 +1,234 @@ +""" +Command to import Blockstore-backed v2 Libraries to Learning Core data models. + +This will hopefully be very short-lived code. +""" +from datetime import datetime, timezone +from xml.etree import ElementTree as ET +import logging + +from django.db import transaction +from django.core.management import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist + +from opaque_keys.edx.locator import LibraryLocatorV2 +from openedx.core.djangoapps.content_libraries import api as lib_api +from openedx.core.djangoapps.content_libraries import models as lib_models +from openedx.core.djangoapps.content_libraries import constants as lib_constants +from openedx.core.lib.blockstore_api import ( + get_bundle, + get_bundle_file_data, + get_bundle_files_dict, +) +from openedx_learning.core.publishing import api as publishing_api +from openedx_learning.core.components import api as components_api +from openedx_learning.core.contents import api as contents_api + + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Create a new LearningPackage and initialize with contents from Library. + + If you run this and specify a Library that already has a LearningPackage + (using -f), this command will delete that LearningPackage and create a new + one to associate with the Libary. It does not modify the existing one. + + All the work is done in a transaction, so errors partway through the + process shouldn't cause state inconsistency in the database. A partly- + imported course *can* cause data to end up in Django Storages. + """ + + def add_arguments(self, parser): + """ + Add arguments to the argument parser. + """ + parser.add_argument( + 'library-key', + type=LibraryLocatorV2.from_string, + help=('Content Library Key to import content from.'), + ) + parser.add_argument( + '-f', + '--force', + action='store_true', + default=False, + ) + + def handle(self, *args, **options): + """ + Does the work of parsing content from Blockstore and writing it into + openedx-learning core models (publishing, components, contents). + """ + # Search for the library. + try: + lib_key = options['library-key'] + lib_data = lib_api.get_library(lib_key) + lib = lib_models.ContentLibrary.objects.get_by_key(lib_key) + except ObjectDoesNotExist: + raise CommandError(f"Library not found: {lib_key}") + + COMPONENT_NAMESPACE = 'xblock.v1' + + learning_package_already_exists = ( + hasattr(lib, 'contents') and + lib.contents.learning_package is not None + ) + + if learning_package_already_exists and not options['force']: + raise CommandError( + f"Learning Package already exists for {lib_key} (use -f to overwrite)" + ) + + with transaction.atomic(): + # This is a migration script and we're assuming there's no important + # state attached to the LearningPackage yet. That makes it safe to + # just wipe out everything and recreate it. + if learning_package_already_exists: + lp = lib.contents.learning_package + log.info(f"Deleting existing LearningPackage {lp.key} ({lp.uuid})") + lib.contents.delete() + lp.delete() + + # Initialize a new LearningPackage + learning_package = publishing_api.create_learning_package( + key=lib_key, + title=lib_data.title, + ) + log.info(f"Created LearningPackage {learning_package.key} ({learning_package.uuid})") + lib_models.ContentLibraryLearningPackage.objects.create( + content_library=lib, + learning_package=learning_package, + ) + + # We don't need the full history stored in Blockstore, just the most + # recently published version and the most recent draft. + bundle = get_bundle(lib.bundle_uuid) + published_files = get_bundle_files_dict(lib.bundle_uuid) + + now = datetime.now(timezone.utc) + + # First get the published version into openedx-learning models. On + # the openedx-learning side, we'll create them as Drafts and then + # publish at the end. + published_metadata_dict = {} + published_component_pks = {} + published_definition_files = { + file_path: metadata + for file_path, metadata in published_files.items() + if file_path.endswith('/definition.xml') # This is the OLX + } + for file_path, metadata in published_definition_files.items(): + block_type, block_id, _def_xml = file_path.split('/') + published_metadata_dict[file_path] = metadata + xml_bytes = get_bundle_file_data(bundle.uuid, file_path) + display_name = extract_display_name(xml_bytes, file_path) + + component, component_version = components_api.create_component_and_version( + learning_package_id=learning_package.id, + namespace=COMPONENT_NAMESPACE, + type=block_type, + local_key=block_id, + title=display_name, + created=now, + created_by=None, + ) + published_component_pks[file_path] = component.pk + text_content, _created = contents_api.get_or_create_text_content_from_bytes( + learning_package_id=learning_package.id, + data_bytes=xml_bytes, + mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", + created=now, + ) + components_api.add_content_to_component_version( + component_version, + raw_content_id=text_content.pk, + key="definition.xml", + learner_downloadable=False + ) + # Publish all the Draft versions we created. + publishing_api.publish_all_drafts( + learning_package_id=learning_package.id, + message="Initial import from Blockstore", + published_at=now, + ) + + # Now grab the draft version from blockstore, and copy those... + draft_files = get_bundle_files_dict(lib.bundle_uuid, use_draft=lib_constants.DRAFT_NAME) + draft_definition_files = { + file_path: metadata + for file_path, metadata in draft_files.items() + if file_path.endswith("definition.xml") + } + for file_path, draft_metadata in draft_definition_files.items(): + published_metadata = published_metadata_dict.get(file_path) + if draft_metadata.modified: + block_type, block_id, _def_xml = file_path.split('/') + xml_bytes = get_bundle_file_data(bundle.uuid, file_path, use_draft=lib_constants.DRAFT_NAME) + display_name = extract_display_name(xml_bytes, file_path) + + # If this is newly created in the draft, we have to create a + # whole new Component... + if published_metadata is None: + component = components_api.create_component( + learning_package_id=learning_package.id, + namespace=COMPONENT_NAMESPACE, + type=block_type, + local_key=block_id, + created=now, + created_by=None, + ) + component_pk = component.pk + version_num = 1 + # Otherwise, it's just been modified... + else: + component_pk = published_component_pks[file_path] + version_num = 2 + + component_version = components_api.create_component_version( + component_pk=component_pk, + version_num=version_num, + title=display_name, + created=now, + created_by=None, + ) + text_content, _created = contents_api.get_or_create_text_content_from_bytes( + learning_package_id=learning_package.id, + data_bytes=xml_bytes, + mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", + created=now, + ) + components_api.add_content_to_component_version( + component_version, + raw_content_id=text_content.pk, + key="definition.xml", + learner_downloadable=False + ) + + # Now remove stuff that was present in the published set but was + # deleted in the draft. + deleted_definition_files = set(published_definition_files) - set(draft_definition_files) + for deleted_definition_file in deleted_definition_files: + log.info(f"Deleting {deleted_definition_file} from draft") + component_pk = published_component_pks[deleted_definition_file] + publishing_api.set_draft_version(component_pk, None) + + +def extract_display_name(xml_bytes, file_path): + """ + Parse the display_name out of the XML. + + This will return an empty string if no display_name is specified, or if + there is a parsing error. + """ + try: + xml_str = xml_bytes.decode('utf-8') + block_root = ET.fromstring(xml_str) + display_name = block_root.attrib.get("display_name", "") + except ET.ParseError as err: + log.error(f"Parse error for {file_path}: {err}") + display_name = "" + + return display_name diff --git a/openedx/core/djangoapps/content_libraries/migrations/0010_contentlibrarylearningpackage.py b/openedx/core/djangoapps/content_libraries/migrations/0010_contentlibrarylearningpackage.py new file mode 100644 index 000000000000..0549e09b2497 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/migrations/0010_contentlibrarylearningpackage.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.22 on 2023-10-24 15:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_publishing', '0002_alter_fk_on_delete'), + ('content_libraries', '0009_alter_contentlibrary_authorized_lti_configs'), + ] + + operations = [ + migrations.CreateModel( + name='ContentLibraryLearningPackage', + fields=[ + ('content_library', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='contents', serialize=False, to='content_libraries.contentlibrary')), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.learningpackage')), + ], + ), + ] diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index 7bf8792699f9..e64aa16c19ae 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -56,6 +56,7 @@ LIBRARY_TYPES, COMPLEX, LICENSE_OPTIONS, ALL_RIGHTS_RESERVED, ) +from openedx_learning.core.publishing.models import LearningPackage from organizations.models import Organization # lint-amnesty, pylint: disable=wrong-import-order from .apps import ContentLibrariesConfig @@ -527,3 +528,30 @@ def update_score(self, weighted_earned, weighted_possible, timestamp): def __str__(self): return str(self.usage_key) + + +class ContentLibraryLearningPackage(models.Model): + """ + Associates ContentLibrary with a LearningPackage. + + This essentially maps the abstract, catalog concept of a Content Library + from the actual bucket of content that is stored in that library. + + If the referenced ContentLibrary is deleted, this association is also + deleted, but the LearningPackage being referenced is _not_ deleted. + + This association actively prevents the referenced LearningPackage from being + deleted, i.e. you cannot just remove the content that is backing a + ContentLibrary by mistake. You have to explicitly disassociate it from the + ContentLibrary first. + """ + content_library = models.OneToOneField( + ContentLibrary, + primary_key=True, + on_delete=models.CASCADE, + related_name='contents', + ) + learning_package = models.ForeignKey( + LearningPackage, + on_delete=models.RESTRICT, + )