diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index cb5456a717d9..aeb2b50d1bd8 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -1,14 +1,15 @@ """ -Python API for content libraries. +Python API for content libraries +================================ -Via 'views.py', most of these API methods are also exposed as a REST API. +Via ``views.py``, most of these API methods are also exposed as a REST API. The API methods in this file are focused on authoring and specific to content libraries; they wouldn't necessarily apply or work in other learning contexts such as courses, blogs, "pathways," etc. ** As this is an authoring-focused API, all API methods in this file deal with -the DRAFT version of the content library. ** +the DRAFT version of the content library.** Some of these methods will work and may be used from the LMS if needed (mostly for test setup; other use is discouraged), but some of the implementation @@ -17,29 +18,49 @@ Any APIs that use/affect content libraries but are generic enough to work in other learning contexts too are in the core XBlock python/REST API at - openedx.core.djangoapps.xblock.api/rest_api +``openedx.core.djangoapps.xblock.api/rest_api``. + +For example, to render a content library XBlock as HTML, one can use the +generic: -For example, to render a content library XBlock as HTML, one can use the generic render_block_view(block, view_name, user) -API in openedx.core.djangoapps.xblock.api (use it from Studio for the draft -version, from the LMS for published version). + +That is an API in ``openedx.core.djangoapps.xblock.api`` (use it from Studio for +the draft version, from the LMS for published version). There are one or two methods in this file that have some overlap with the core -XBlock API; for example, this content library API provides a get_library_block() -which returns metadata about an XBlock; it's in this API because it also returns -data about whether or not the XBlock has unpublished edits, which is an -authoring-only concern. Likewise, APIs for getting/setting an individual -XBlock's OLX directly seem more appropriate for small, reusable components in -content libraries and may not be appropriate for other learning contexts so they -are implemented here in the library API only. In the future, if we find a need -for these in most other learning contexts then those methods could be promoted -to the core XBlock API and made generic. +XBlock API; for example, this content library API provides a +``get_library_block()`` which returns metadata about an XBlock; it's in this API +because it also returns data about whether or not the XBlock has unpublished +edits, which is an authoring-only concern. Likewise, APIs for getting/setting +an individual XBlock's OLX directly seem more appropriate for small, reusable +components in content libraries and may not be appropriate for other learning +contexts so they are implemented here in the library API only. In the future, +if we find a need for these in most other learning contexts then those methods +could be promoted to the core XBlock API and made generic. + +Import from Courseware +---------------------- + +Content Libraries can import blocks from Courseware (Modulestore). The import +can be done per-course, by listing its content, and supports both access to +remote platform instances as well as local modulestore APIs. Additionally, +there are Celery-based interfaces suitable for background processing controlled +through RESTful APIs (see :mod:`.views`). """ -from uuid import UUID + + +import abc +import collections from datetime import datetime +from uuid import UUID +import base64 +import hashlib import logging import attr +import requests + from django.conf import settings from django.contrib.auth.models import AbstractUser, Group from django.core.exceptions import PermissionDenied @@ -48,17 +69,21 @@ from django.utils.translation import ugettext as _ from elasticsearch.exceptions import ConnectionError as ElasticConnectionError from lxml import etree -from opaque_keys.edx.keys import LearningContextKey +from opaque_keys.edx.keys import LearningContextKey, UsageKey from opaque_keys.edx.locator import BundleDefinitionLocator, LibraryLocatorV2, LibraryUsageLocatorV2 from organizations.models import Organization from xblock.core import XBlock from xblock.exceptions import XBlockNotFoundError - +from edx_rest_api_client.client import OAuthAPIClient from openedx.core.djangoapps.content_libraries import permissions from openedx.core.djangoapps.content_libraries.constants import DRAFT_NAME, COMPLEX from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer -from openedx.core.djangoapps.content_libraries.models import ContentLibrary, ContentLibraryPermission +from openedx.core.djangoapps.content_libraries.models import ( + ContentLibrary, + ContentLibraryPermission, + ContentLibraryBlockImportTask, +) from openedx.core.djangoapps.content_libraries.signals import ( CONTENT_LIBRARY_CREATED, CONTENT_LIBRARY_UPDATED, @@ -67,6 +92,7 @@ LIBRARY_BLOCK_UPDATED, LIBRARY_BLOCK_DELETED, ) +from openedx.core.djangoapps.olx_rest_api.block_serializer import XBlockSerializer from openedx.core.djangoapps.xblock.api import get_block_display_name, load_block from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl from openedx.core.djangoapps.xblock.runtime.olx_parsing import XBlockInclude @@ -86,10 +112,18 @@ ) from openedx.core.djangolib import blockstore_cache from openedx.core.djangolib.blockstore_cache import BundleCache +from xmodule.modulestore.django import modulestore + +from . import tasks + log = logging.getLogger(__name__) -# Exceptions: + +# Exceptions +# ========== + + ContentLibraryNotFound = ContentLibrary.DoesNotExist @@ -121,7 +155,9 @@ class LibraryPermissionIntegrityError(IntegrityError): """ Thrown when an operation would cause insane permissions. """ -# Models: +# Models +# ====== + @attr.s class ContentLibraryMetadata: @@ -229,6 +265,10 @@ class AccessLevel: # lint-amnesty, pylint: disable=function-redefined NO_ACCESS = None +# General APIs +# ============ + + def get_libraries_for_user(user, org=None, library_type=None): """ Return content libraries that the user has permission to view. @@ -1051,3 +1091,301 @@ def revert_changes(library_key): return # If there is no draft, no action is needed. LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME).cache.clear() CONTENT_LIBRARY_UPDATED.send(sender=None, library_key=library_key, update_blocks=True) + + +# Import from Courseware +# ====================== + + +class BaseEdxImportClient(abc.ABC): + """ + Base class for all courseware import clients. + + Import clients are wrappers tailored to implement the steps used in the + import APIs and can leverage different backends. It is not aimed towards + being a generic API client for Open edX. + """ + + EXPORTABLE_BLOCK_TYPES = { + "drag-and-drop-v2", + "problem", + "html", + "video", + } + + def __init__(self, library_key=None, library=None): + """ + Initialize an import client for a library. + + The method accepts either a library object or a key to a library object. + """ + if bool(library_key) == bool(library): + raise ValueError('Provide at least one of `library_key` or ' + '`library`, but not both.') + if library is None: + library = ContentLibrary.objects.get_by_key(library_key) + self.library = library + + @abc.abstractmethod + def get_block_data(self, block_key): + """ + Get the block's OLX and static files, if any. + """ + + @abc.abstractmethod + def get_export_keys(self, course_key): + """ + Get all exportable block keys of a given course. + """ + + @abc.abstractmethod + def get_block_static_data(self, asset_file): + """ + Get the contents of an asset_file.. + """ + + def import_block(self, modulestore_key): + """ + Import a single modulestore block. + """ + + block_data = self.get_block_data(modulestore_key) + + # Get or create the block in the library. + # + # To dedup blocks from different courses with the same ID, we hash the + # course key into the imported block id. + + course_key_id = base64.b32encode( + hashlib.blake2s( + str(modulestore_key.course_key).encode() + ).digest() + )[:16].decode().lower() + # Prepend 'c' to allow changing hash without conflicts. + block_id = f"{modulestore_key.block_id}_c{course_key_id}" + log.info('Importing to library block: id=%s', block_id) + try: + library_block = create_library_block( + self.library.library_key, + modulestore_key.block_type, + block_id, + ) + blockstore_key = library_block.usage_key + except LibraryBlockAlreadyExists: + blockstore_key = LibraryUsageLocatorV2( + lib_key=self.library.library_key, + block_type=modulestore_key.block_type, + usage_id=block_id, + ) + get_library_block(blockstore_key) + log.warning('Library block already exists: Appending static files ' + 'and overwriting OLX: %s', str(blockstore_key)) + + # Handle static files. + + files = [ + f.path for f in + get_library_block_static_asset_files(blockstore_key) + ] + for filename, static_file in block_data.get('static_files', {}).items(): + if filename in files: + # Files already added, move on. + continue + file_content = self.get_block_static_data(static_file) + add_library_block_static_asset_file( + blockstore_key, filename, file_content) + files.append(filename) + + # Import OLX. + + set_library_block_olx(blockstore_key, block_data['olx']) + + def import_blocks_from_course(self, course_key, progress_callback): + """ + Import all eligible blocks from course key. + + Progress is reported through ``progress_callback``, guaranteed to be + called within an exception handler if ``exception is not None``. + """ + + # Query the course and rerieve all course blocks. + + export_keys = self.get_export_keys(course_key) + if not export_keys: + raise ValueError(f"The courseware course {course_key} does not have " + "any exportable content. No action taken.") + + # Import each block, skipping the ones that fail. + + for index, block_key in enumerate(export_keys): + try: + log.info('Importing block: %s/%s: %s', index + 1, len(export_keys), block_key) + self.import_block(block_key) + except Exception as exc: # pylint: disable=broad-except + log.exception("Error importing block: %s", block_key) + progress_callback(block_key, index + 1, len(export_keys), exc) + else: + log.info('Successfully imported: %s/%s: %s', index + 1, len(export_keys), block_key) + progress_callback(block_key, index + 1, len(export_keys), None) + + log.info("Publishing library: %s", self.library.library_key) + publish_changes(self.library.library_key) + + +class EdxModulestoreImportClient(BaseEdxImportClient): + """ + An import client based on the local instance of modulestore. + """ + + def __init__(self, modulestore_instance=None, **kwargs): + """ + Initialize the client with a modulestore instance. + """ + super().__init__(**kwargs) + self.modulestore = modulestore_instance or modulestore() + + def get_block_data(self, block_key): + """ + Get block OLX by serializing it from modulestore directly. + """ + block = self.modulestore.get_item(block_key) + data = XBlockSerializer(block) + return {'olx': data.olx_str, + 'static_files': {s.name: s for s in data.static_files}} + + def get_export_keys(self, course_key): + """ + Retrieve the course from modulestore and traverse its content tree. + """ + course = self.modulestore.get_course(course_key) + export_keys = set() + blocks_q = collections.deque(course.get_children()) + while blocks_q: + block = blocks_q.popleft() + usage_id = block.scope_ids.usage_id + if usage_id in export_keys: + continue + if usage_id.block_type in self.EXPORTABLE_BLOCK_TYPES: + export_keys.add(usage_id) + if block.has_children: + blocks_q.extend(block.get_children()) + return list(export_keys) + + def get_block_static_data(self, asset_file): + """ + Get static content from its URL if available, otherwise from its data. + """ + if asset_file.data: + return asset_file.data + resp = requests.get(f"http://{settings.CMS_BASE}" + asset_file.url) + resp.raise_for_status() + return resp.content + + +class EdxApiImportClient(BaseEdxImportClient): + """ + An import client based on a remote Open Edx API interface. + """ + + URL_COURSES = "/api/courses/v1/courses/{course_key}" + + URL_MODULESTORE_BLOCK_OLX = "/api/olx-export/v1/xblock/{block_key}/" + + def __init__(self, lms_url, studio_url, oauth_key, oauth_secret, *args, **kwargs): + """ + Initialize the API client with URLs and OAuth keys. + """ + super().__init__(**kwargs) + self.lms_url = lms_url + self.studio_url = studio_url + self.oauth_client = OAuthAPIClient( + self.lms_url, + oauth_key, + oauth_secret, + ) + + def get_block_data(self, block_key): + """ + See parent's docstring. + """ + olx_path = self.URL_MODULESTORE_BLOCK_OLX.format(block_key=block_key) + resp = self._get(self.studio_url + olx_path) + return resp['blocks'][str(block_key)] + + def get_export_keys(self, course_key): + """ + See parent's docstring. + """ + course_blocks_url = self._get_course(course_key)['blocks_url'] + course_blocks = self._get( + course_blocks_url, + params={'all_blocks': True, 'depth': 'all'})['blocks'] + export_keys = [] + for block_info in course_blocks.values(): + if block_info['type'] in self.EXPORTABLE_BLOCK_TYPES: + export_keys.append(UsageKey.from_string(block_info['id'])) + return export_keys + + def get_block_static_data(self, asset_file): + """ + See parent's docstring. + """ + if (asset_file['url'].startswith(self.studio_url) + and 'export-file' in asset_file['url']): + # We must call download this file with authentication. But + # we only want to pass the auth headers if this is the same + # studio instance, or else we could leak credentials to a + # third party. + path = asset_file['url'][len(self.studio_url):] + resp = self._call('get', path) + else: + resp = requests.get(asset_file['url']) + resp.raise_for_status() + return resp.content + + def _get(self, *args, **kwargs): + """ + Perform a get request to the client. + """ + return self._json_call('get', *args, **kwargs) + + def _get_course(self, course_key): + """ + Request details for a course. + """ + course_url = self.lms_url + self.URL_COURSES.format(course_key=course_key) + return self._get(course_url) + + def _json_call(self, method, *args, **kwargs): + """ + Wrapper around request calls that ensures valid json responses. + """ + return self._call(method, *args, **kwargs).json() + + def _call(self, method, *args, **kwargs): + """ + Wrapper around request calls. + """ + response = getattr(self.oauth_client, method)(*args, **kwargs) + response.raise_for_status() + return response + + +def import_blocks_create_task(library_key, course_key): + """ + Create a new import block task. + + This API will schedule a celery task to perform the import, and it returns a + import task object for polling. + """ + library = ContentLibrary.objects.get_by_key(library_key) + import_task = ContentLibraryBlockImportTask.objects.create( + library=library, + course_id=course_key, + ) + result = tasks.import_blocks_from_course.apply_async( + args=(import_task.pk, str(course_key)) + ) + log.info(f"Import block task created: import_task={import_task} " + f"celery_task={result.id}") + return import_task diff --git a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py new file mode 100644 index 000000000000..ca1b6e9d0fbf --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py @@ -0,0 +1,146 @@ +""" +Command to import modulestore content into Content Libraries. +""" + +import argparse +import logging + +from django.conf import settings +from django.core.management import BaseCommand, CommandError + +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocatorV2 +from openedx.core.djangoapps.content_libraries import api as contentlib_api + + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Import modulestore content, references by a course, into a Content Libraries + library. + """ + + def add_arguments(self, parser): + """ + Add arguments to the argument parser. + """ + parser.add_argument( + 'library-key', + type=LibraryLocatorV2.from_string, + help=('Usage key of the Content Library to import content into.'), + ) + parser.add_argument( + 'course-key', + type=CourseKey.from_string, + help=('The Course Key string, used to identify the course to import ' + 'content from.'), + ) + subparser = parser.add_subparsers( + title='Courseware location and methods', + dest='method', + description=('Select the method and location to locate the course and ' + 'its contents.') + ) + api_parser = subparser.add_parser( + 'api', + + help=('Query and retrieve course blocks from a remote instance using ' + 'Open edX course and OLX export APIs. You need to enable API access ' + 'on the instance.') + ) + api_parser.add_argument( + '--lms-url', + default=settings.LMS_ROOT_URL, + help=("The LMS URL, used to retrieve course content (default: " + "'%(default)s')."), + ) + api_parser.add_argument( + '--studio-url', + default=f"https://{settings.CMS_BASE}", + help=("The Studio URL, used to retrieve block OLX content (default: " + "'%(default)s')"), + ) + oauth_group = api_parser.add_mutually_exclusive_group(required=False) + oauth_group.add_argument( + '--oauth-creds-file', + type=argparse.FileType('r'), + help=('The edX OAuth credentials in a filename. The first line is ' + 'the OAuth key, second line is the OAuth secret. This is ' + 'preferred compared to passing the credentials in the command ' + 'line.'), + ) + oauth_group.add_argument( + '--oauth-creds', + nargs=2, + help=('The edX OAuth credentials in the command line. The first ' + 'argument is the OAuth secret, the second argument is the ' + 'OAuth key. Notice that command line arguments are insecure, ' + 'see `--oauth-creds-file`.'), + ) + subparser.add_parser( + 'modulestore', + help=("Use a local modulestore instance to retrieve blocks database on " + "the instance where the command is being run. You don't need " + "to enable API access.") + ) + + def handle(self, *args, **options): + """ + Collect all blocks from a course that are "importable" and write them to the + a blockstore library. + """ + + # Search for the library. + + try: + contentlib_api.get_library(options['library-key']) + except contentlib_api.ContentLibraryNotFound as exc: + raise CommandError("The library specified does not exist: " + f"{options['library-key']}") from exc + + # Validate the method and its arguments, instantiate the openedx client. + + if options['method'] == 'api': + if options['oauth_creds_file']: + with options['oauth_creds_file'] as creds_f: + oauth_key, oauth_secret = [v.strip() for v in creds_f.readlines()] + elif options['oauth_creds']: + oauth_key, oauth_secret = options['oauth_creds'] + else: + raise CommandError("Method 'api' requires one of the " + "--oauth-* options, and none was specified.") + edx_client = contentlib_api.EdxApiImportClient( + options['lms_url'], + options['studio_url'], + oauth_key, + oauth_secret, + library_key=options['library-key'], + ) + elif options['method'] == 'modulestore': + edx_client = contentlib_api.EdxModulestoreImportClient( + library_key=options['library-key'], + ) + else: + raise CommandError(f"Method not supported: {options['method']}") + + failed_blocks = [] + + def on_progress(block_key, block_num, block_count, exception=None): + self.stdout.write(f"{block_num}/{block_count}: {block_key}: ", ending='') + # In case stdout is a term and line buffered: + self.stdout.flush() + if exception: + self.stdout.write(self.style.ERROR('❌')) + log.error('Failed to import block: %s', block_key, exc_info=exception) + failed_blocks.append(block_key) + else: + self.stdout.write(self.style.SUCCESS('✓')) + + edx_client.import_blocks_from_course(options['course-key'], on_progress) + + if failed_blocks: + self.stdout.write(self.style.ERROR(f"❌ {len(failed_blocks)} failed:")) + for key in failed_blocks: + self.stdout.write(self.style.ERROR(str(key))) diff --git a/openedx/core/djangoapps/content_libraries/migrations/0005_contentlibraryblockimporttask.py b/openedx/core/djangoapps/content_libraries/migrations/0005_contentlibraryblockimporttask.py new file mode 100644 index 000000000000..ab852cf14c7e --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/migrations/0005_contentlibraryblockimporttask.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.24 on 2021-07-16 23:20 + +from django.db import migrations, models +import django.db.models.deletion +from opaque_keys.edx.django.models import CourseKeyField + + +class Migration(migrations.Migration): + + dependencies = [ + ('content_libraries', '0004_contentlibrary_license'), + ] + + operations = [ + migrations.CreateModel( + name='ContentLibraryBlockImportTask', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('state', models.CharField(choices=[('created', 'Task was created, but not queued to run.'), ('pending', 'Task was created and queued to run.'), ('running', 'Task is running.'), ('failed', 'Task finished, but some blocks failed to import.'), ('successful', 'Task finished successfully.')], default='created', help_text='The state of the block import task.', max_length=30, verbose_name='state')), + ('progress', models.FloatField(default=0.0, help_text='A float from 0.0 to 1.0 representing the task progress.', verbose_name='progress')), + ('course_id', CourseKeyField(max_length=255, db_index=True, verbose_name='course ID', help_text='ID of the imported course.')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='import_tasks', to='content_libraries.ContentLibrary')), + ], + options={'ordering': ['-created_at', '-updated_at']}, + ), + ] diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index 24c69bb3a83a..709aa8d99b8a 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -1,11 +1,16 @@ """ -Models for new Content Libraries +Models for new Content Libraries. """ + +import contextlib + from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import ugettext_lazy as _ + +from opaque_keys.edx.django.models import CourseKeyField from opaque_keys.edx.locator import LibraryLocatorV2 from openedx.core.djangoapps.content_libraries.constants import ( LIBRARY_TYPES, COMPLEX, LICENSE_OPTIONS, @@ -127,3 +132,82 @@ def save(self, *args, **kwargs): # lint-amnesty, pylint: disable=arguments-diff def __str__(self): who = self.user.username if self.user else self.group.name return f"ContentLibraryPermission ({self.access_level} for {who})" + + +class ContentLibraryBlockImportTask(models.Model): + """ + Model of a task to import blocks from an external source (e.g. modulestore). + """ + + library = models.ForeignKey( + ContentLibrary, + on_delete=models.CASCADE, + related_name='import_tasks', + ) + + TASK_CREATED = 'created' + TASK_PENDING = 'pending' + TASK_RUNNING = 'running' + TASK_FAILED = 'failed' + TASK_SUCCESSFUL = 'successful' + + TASK_STATE_CHOICES = ( + (TASK_CREATED, _('Task was created, but not queued to run.')), + (TASK_PENDING, _('Task was created and queued to run.')), + (TASK_RUNNING, _('Task is running.')), + (TASK_FAILED, _('Task finished, but some blocks failed to import.')), + (TASK_SUCCESSFUL, _('Task finished successfully.')), + ) + + state = models.CharField( + choices=TASK_STATE_CHOICES, + default=TASK_CREATED, + max_length=30, + verbose_name=_('state'), + help_text=_('The state of the block import task.'), + ) + + progress = models.FloatField( + default=0.0, + verbose_name=_('progress'), + help_text=_('A float from 0.0 to 1.0 representing the task progress.'), + ) + + course_id = CourseKeyField( + max_length=255, + db_index=True, + verbose_name=_('course ID'), + help_text=_('ID of the imported course.'), + ) + + created_at = models.DateTimeField(auto_now_add=True) + + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ['-created_at', '-updated_at'] + + @classmethod + @contextlib.contextmanager + def execute(cls, import_task_id): + """ + A context manager to manage a task that is being executed. + """ + self = cls.objects.get(pk=import_task_id) + self.state = self.TASK_RUNNING + self.save() + try: + yield self + self.state = self.TASK_SUCCESSFUL + except: # pylint: disable=broad-except + self.state = self.TASK_FAILED + raise + finally: + self.save() + + def save_progress(self, progress): + self.progress = progress + self.save(update_fields=['progress', 'updated_at']) + + def __str__(self): + return f'{self.course_id} to {self.library} #{self.pk}' diff --git a/openedx/core/djangoapps/content_libraries/serializers.py b/openedx/core/djangoapps/content_libraries/serializers.py index 0d4d80566fa5..96ee8dc59c05 100644 --- a/openedx/core/djangoapps/content_libraries/serializers.py +++ b/openedx/core/djangoapps/content_libraries/serializers.py @@ -11,8 +11,12 @@ ALL_RIGHTS_RESERVED, LICENSE_OPTIONS, ) -from openedx.core.djangoapps.content_libraries.models import ContentLibraryPermission +from openedx.core.djangoapps.content_libraries.models import ( + ContentLibraryPermission, ContentLibraryBlockImportTask +) from openedx.core.lib import blockstore_api +from openedx.core.lib.api.serializers import CourseKeyField + DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' @@ -85,12 +89,18 @@ class ContentLibraryPermissionSerializer(ContentLibraryPermissionLevelSerializer group_name = serializers.CharField(source="group.name", allow_null=True, allow_blank=False, default=None) -class ContentLibraryFilterSerializer(serializers.Serializer): +class BaseFilterSerializer(serializers.Serializer): """ - Serializer for filtering library listings. + Base serializer for filtering listings on the content library APIs. """ text_search = serializers.CharField(default=None, required=False) org = serializers.CharField(default=None, required=False) + + +class ContentLibraryFilterSerializer(BaseFilterSerializer): + """ + Serializer for filtering library listings. + """ type = serializers.ChoiceField(choices=LIBRARY_TYPES, default=None, required=False) @@ -190,3 +200,30 @@ class LibraryXBlockStaticFilesSerializer(serializers.Serializer): Serializes a LibraryXBlockStaticFile (or a BundleFile) """ files = LibraryXBlockStaticFileSerializer(many=True) + + +class ContentLibraryBlockImportTaskSerializer(serializers.ModelSerializer): + """ + Serializer for a Content Library block import task. + """ + + org = serializers.SerializerMethodField() + + def get_org(self, obj): + return obj.course_id.org + + class Meta: + model = ContentLibraryBlockImportTask + fields = '__all__' + + +class ContentLibraryBlockImportTaskCreateSerializer(serializers.Serializer): + """ + Serializer to create a new block import task. + + The serializer accepts the following parameter: + + - The courseware course key to import blocks from. + """ + + course_key = CourseKeyField() diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py new file mode 100644 index 000000000000..1cb097fad57b --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -0,0 +1,40 @@ +""" +Celery tasks for Content Libraries. +""" + + +import logging + +from celery import shared_task +from celery_utils.logged_task import LoggedTask + +from opaque_keys.edx.keys import CourseKey + +from . import api +from .models import ContentLibraryBlockImportTask + + +logger = logging.getLogger(__name__) + + +@shared_task(base=LoggedTask) +def import_blocks_from_course(import_task_id, course_key_str): + """ + A Celery task to import blocks from a course through modulestore. + """ + + course_key = CourseKey.from_string(course_key_str) + + with ContentLibraryBlockImportTask.execute(import_task_id) as import_task: + + def on_progress(block_key, block_num, block_count, exception=None): + if exception: + logger.exception('Import block failed: %s', block_key) + else: + logger.info('Import block succesful: %s', block_key) + import_task.save_progress(block_num / block_count) + + edx_client = api.EdxModulestoreImportClient(library=import_task.library) + edx_client.import_blocks_from_course( + course_key, on_progress + ) diff --git a/openedx/core/djangoapps/content_libraries/tests/test_api.py b/openedx/core/djangoapps/content_libraries/tests/test_api.py new file mode 100644 index 000000000000..87ae180d291e --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/tests/test_api.py @@ -0,0 +1,243 @@ +""" +Tests for Content Library internal api. +""" + +import base64 +import hashlib +from unittest import mock + +from django.test import TestCase + +from opaque_keys.edx.keys import ( + CourseKey, + UsageKey, +) +from opaque_keys.edx.locator import LibraryLocatorV2 + +from .. import api + + +class EdxModulestoreImportClientTest(TestCase): + """ + Tests for course importing APIs. + """ + + def setUp(self): + """ + Setup mocks and the test client. + """ + super().setUp() + self.mock_library = mock.MagicMock() + self.modulestore_mock = mock.MagicMock() + self.client = api.EdxModulestoreImportClient( + modulestore_instance=self.modulestore_mock, + library=self.mock_library + ) + + def test_instantiate_without_args(self): + """ + When instantiated without args, + Then raises. + """ + with self.assertRaises(ValueError): + api.EdxModulestoreImportClient() + + def test_import_blocks_from_course_without_course(self): + """ + Given no course, + Then raises. + """ + self.modulestore_mock.get_course.return_value.get_children.return_value = [] + with self.assertRaises(ValueError): + self.client.import_blocks_from_course('foobar', lambda *_: None) + + @mock.patch('openedx.core.djangoapps.content_libraries.api.create_library_block') + @mock.patch('openedx.core.djangoapps.content_libraries.api.get_library_block') + @mock.patch('openedx.core.djangoapps.content_libraries.api.get_library_block_static_asset_files') + @mock.patch('openedx.core.djangoapps.content_libraries.api.publish_changes') + @mock.patch('openedx.core.djangoapps.content_libraries.api.set_library_block_olx') + def test_import_blocks_from_course_on_block_with_olx( + self, + mock_set_library_block_olx, + mock_publish_changes, + mock_get_library_block_static_asset_files, + mock_get_library_block, + mock_create_library_block, + ): + """ + Given a course with one block + When called + Then extract OLX, write to library and publish. + """ + + usage_key_str = 'lb:foo:bar:foobar:1234' + library_key_str = 'lib:foo:bar' + + self.client.get_export_keys = mock.MagicMock(return_value=[UsageKey.from_string(usage_key_str)]) + self.client.get_block_data = mock.MagicMock(return_value={'olx': 'fake-olx'}) + + mock_create_library_block.side_effect = api.LibraryBlockAlreadyExists + self.mock_library.library_key = LibraryLocatorV2.from_string(library_key_str) + + self.client.import_blocks_from_course('foobar', lambda *_: None) + + mock_get_library_block.assert_called_once() + mock_get_library_block_static_asset_files.called_once() + mock_set_library_block_olx.assert_called_once_with( + mock.ANY, 'fake-olx') + mock_publish_changes.assert_called_once() + + @mock.patch('openedx.core.djangoapps.content_libraries.api.create_library_block') + @mock.patch('openedx.core.djangoapps.content_libraries.api.get_library_block_static_asset_files') + @mock.patch('openedx.core.djangoapps.content_libraries.api.set_library_block_olx') + def test_import_block_when_called_twice_same_block_but_different_course( + self, + mock_set_library_block_olx, + mock_get_library_block_static_asset_files, + mock_create_library_block, + ): + """ + Given an block used by one course + And another block with same id use by a different course + And import_block() was called on the first block + When import_block() is called on the second block + Then create a library block for the second block + """ + course_key_str = 'block-v1:FakeCourse+FakeOrg+FakeRun+type@a-fake-block-type+block@fake-block-id' + + modulestore_usage_key = UsageKey.from_string(course_key_str) + expected_course_key_hash = base64.b32encode( + hashlib.blake2s( + str(modulestore_usage_key.course_key).encode() + ).digest() + )[:16].decode().lower() + expected_usage_id = f"{modulestore_usage_key.block_id}_c{expected_course_key_hash}" + + self.client.get_block_data = mock.MagicMock() + self.client.import_block(modulestore_usage_key) + + mock_create_library_block.assert_called_with( + self.client.library.library_key, + modulestore_usage_key.block_type, + expected_usage_id) + mock_get_library_block_static_asset_files.assert_called_once() + mock_set_library_block_olx.assert_called_once() + + +@mock.patch('openedx.core.djangoapps.content_libraries.api.OAuthAPIClient') +class EdxApiImportClientTest(TestCase): + """ + Tests for EdxApiImportClient. + """ + + LMS_URL = 'https://foobar_lms.example.com/' + + STUDIO_URL = 'https://foobar_studio.example.com/' + + library_key_str = 'lib:foobar_content:foobar_library' + + course_key_str = 'course-v1:AFakeCourse+FooBar+1' + + def create_mock_library(self, *, course_id=None, course_key_str=None): + """ + Create a library mock. + """ + mock_library = mock.MagicMock() + mock_library.library_key = LibraryLocatorV2.from_string( + self.library_key_str + ) + if course_key_str is None: + course_key_str = self.course_key_str + if course_id is None: + course_id = CourseKey.from_string(course_key_str) + type(mock_library).course_id = mock.PropertyMock(return_value=course_id) + return mock_library + + def create_client(self, *, mock_library=None): + """ + Create a edX API import client mock. + """ + return api.EdxApiImportClient( + self.LMS_URL, + self.STUDIO_URL, + 'foobar_oauth_key', + 'foobar_oauth_secret', + library=(mock_library or self.create_mock_library()), + ) + + def mock_oauth_client_response(self, mock_oauth_client, *, content=None, exception=None): + """ + Setup a mock response for oauth client GET calls. + """ + mock_response = mock.MagicMock() + mock_content = None + if exception: + mock_response.raise_for_status.side_effect = exception + if content: + mock_content = mock.PropertyMock(return_value='foobar_file_content') + type(mock_response).content = mock_content + mock_oauth_client.get.return_value = mock_response + if mock_content: + return mock_response, mock_content + return mock_response + + @mock.patch('openedx.core.djangoapps.content_libraries.api.add_library_block_static_asset_file') + @mock.patch('openedx.core.djangoapps.content_libraries.api.create_library_block') + @mock.patch('openedx.core.djangoapps.content_libraries.api.get_library_block_static_asset_files') + @mock.patch('openedx.core.djangoapps.content_libraries.api.publish_changes') + @mock.patch('openedx.core.djangoapps.content_libraries.api.set_library_block_olx') + def test_import_block_when_url_is_from_studio( + self, + mock_set_library_block_olx, + mock_publish_changes, + mock_get_library_block_static_asset_files, + mock_create_library_block, + mock_add_library_block_static_asset_file, + mock_oauth_client_class, + ): + """ + Given an block with one asset provided by a studio. + When import_block() is called on the block. + Then a GET to the API endpoint is. + """ + + # Setup mocks. + + static_filename = 'foobar_filename' + static_content = 'foobar_file_content' + block_olx = 'foobar-olx' + usage_key = UsageKey.from_string('lb:foo:bar:foobar:1234') + # We ensure ``export-file`` belongs to the URL. + asset_studio_url = f"{self.STUDIO_URL}/foo/bar/export-file/foo/bar" + block_data = { + 'olx': block_olx, + 'static_files': {static_filename: {'url': asset_studio_url}} + } + _, mock_content = self.mock_oauth_client_response( + mock_oauth_client_class.return_value, + content=static_content, + ) + mock_create_library_block.return_value.usage_key = usage_key + + # Create client and call. + + client = self.create_client() + client.get_block_data = mock.MagicMock(return_value=block_data) + client.import_block(usage_key) + + # Assertions. + + client.get_block_data.assert_called_once_with(usage_key) + mock_create_library_block.assert_called_once() + mock_get_library_block_static_asset_files.assert_called_once() + mock_content.assert_called() + mock_add_library_block_static_asset_file.assert_called_once_with( + usage_key, + static_filename, + static_content + ) + mock_set_library_block_olx.assert_called_once_with( + usage_key, + block_olx + ) + mock_publish_changes.assert_not_called() diff --git a/openedx/core/djangoapps/content_libraries/tests/test_command_content_libraries_import.py b/openedx/core/djangoapps/content_libraries/tests/test_command_content_libraries_import.py new file mode 100644 index 000000000000..0f57b8f2d815 --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/tests/test_command_content_libraries_import.py @@ -0,0 +1,47 @@ +""" +Unit tests for content_libraries_import command. +""" + + +from unittest import mock +from io import StringIO + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase + + +@mock.patch('openedx.core.djangoapps.content_libraries.management.commands.content_libraries_import.contentlib_api') +class ContentLibrariesImportTest(TestCase): + """ + Unit tests for content_libraries_import command. + """ + + library_key_str = 'lib:foo:bar' + + course_key_str = 'course-v1:foo+bar+foobar' + + def call_command(self, *args, **kwargs): + """ + Call command with default test paramters. + """ + out = StringIO() + kwargs['stdout'] = out + library_key = kwargs.pop('library_key', self.library_key_str) + course_key = kwargs.pop('course_key', self.course_key_str) + call_command('content_libraries_import', library_key, course_key, + 'api', + '--oauth-creds', 'fake-key', 'fake-secret', + *args, **kwargs) + return out + + def test_call_without_library(self, api_mock): + """ + Given library does not exists + Then raises command error + """ + from openedx.core.djangoapps.content_libraries.api import ContentLibraryNotFound + api_mock.ContentLibraryNotFound = ContentLibraryNotFound + api_mock.get_library.side_effect = ContentLibraryNotFound + with self.assertRaises(CommandError): + self.call_command() diff --git a/openedx/core/djangoapps/content_libraries/urls.py b/openedx/core/djangoapps/content_libraries/urls.py index 4066d72cc8dd..3cd02a9ac5c0 100644 --- a/openedx/core/djangoapps/content_libraries/urls.py +++ b/openedx/core/djangoapps/content_libraries/urls.py @@ -1,16 +1,27 @@ """ URL configuration for Studio's Content Libraries REST API """ + from django.conf.urls import include, url +from rest_framework import routers + from . import views +# Django application name. + app_name = 'openedx.core.djangoapps.content_libraries' +# Router for importing blocks from courseware. + +import_blocks_router = routers.DefaultRouter() +import_blocks_router.register(r'tasks', views.LibraryImportTaskViewSet, basename='import-block-task') + # These URLs are only used in Studio. The LMS already provides all the # API endpoints needed to serve XBlocks from content libraries using the # standard XBlock REST API (see openedx.core.django_apps.xblock.rest_api.urls) + urlpatterns = [ url(r'^api/libraries/v2/', include([ # list of libraries / create a library: @@ -34,6 +45,8 @@ url(r'^team/user/(?P[^/]+)/$', views.LibraryTeamUserView.as_view()), # Add/Edit (PUT) or remove (DELETE) a group's permission to use this library url(r'^team/group/(?P[^/]+)/$', views.LibraryTeamGroupView.as_view()), + # Import blocks into this library. + url(r'^import_blocks/', include(import_blocks_router.urls)), ])), url(r'^blocks/(?P[^/]+)/', include([ # Get metadata about a specific XBlock in this library, or delete the block: diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 150b9a47b8b8..2b9598d43418 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -19,14 +19,17 @@ from rest_framework.parsers import MultiPartParser from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework.viewsets import ViewSet from openedx.core.djangoapps.content_libraries import api, permissions from openedx.core.djangoapps.content_libraries.serializers import ( + ContentLibraryBlockImportTaskCreateSerializer, + ContentLibraryBlockImportTaskSerializer, + ContentLibraryFilterSerializer, ContentLibraryMetadataSerializer, - ContentLibraryUpdateSerializer, ContentLibraryPermissionLevelSerializer, ContentLibraryPermissionSerializer, - ContentLibraryFilterSerializer, + ContentLibraryUpdateSerializer, LibraryXBlockCreationSerializer, LibraryXBlockMetadataSerializer, LibraryXBlockTypeSerializer, @@ -39,6 +42,7 @@ ) from openedx.core.lib.api.view_utils import view_auth_classes + User = get_user_model() log = logging.getLogger(__name__) @@ -689,3 +693,64 @@ def delete(self, request, usage_key_str, file_path): except ValueError: raise ValidationError("Invalid file path") # lint-amnesty, pylint: disable=raise-missing-from return Response(status=status.HTTP_204_NO_CONTENT) + + +@view_auth_classes() +class LibraryImportTaskViewSet(ViewSet): + """ + Import blocks from Courseware through modulestore. + """ + + @convert_exceptions + def list(self, request, lib_key_str): + """ + List all import tasks for this library. + """ + library_key = LibraryLocatorV2.from_string(lib_key_str) + api.require_permission_for_library_key( + library_key, + request.user, + permissions.CAN_VIEW_THIS_CONTENT_LIBRARY + ) + queryset = api.ContentLibrary.objects.get_by_key(library_key).import_tasks + result = ContentLibraryBlockImportTaskSerializer(queryset, many=True).data + paginator = LibraryApiPagination() + return paginator.get_paginated_response( + paginator.paginate_queryset(result, request) + ) + + @convert_exceptions + def create(self, request, lib_key_str): + """ + Create and queue an import tasks for this library. + """ + + library_key = LibraryLocatorV2.from_string(lib_key_str) + api.require_permission_for_library_key( + library_key, + request.user, + permissions.CAN_EDIT_THIS_CONTENT_LIBRARY, + ) + + serializer = ContentLibraryBlockImportTaskCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + course_key = serializer.validated_data['course_key'] + + import_task = api.import_blocks_create_task(library_key, course_key) + return Response(ContentLibraryBlockImportTaskSerializer(import_task).data) + + @convert_exceptions + def retrieve(self, request, lib_key_str, pk=None): + """ + Retrieve a import task for inspection. + """ + + library_key = LibraryLocatorV2.from_string(lib_key_str) + api.require_permission_for_library_key( + library_key, + request.user, + permissions.CAN_VIEW_THIS_CONTENT_LIBRARY, + ) + + import_task = api.ContentLibraryBlockImportTask.objects.get(pk=pk) + return Response(ContentLibraryBlockImportTaskSerializer(import_task).data) diff --git a/openedx/core/djangoapps/olx_rest_api/adapters.py b/openedx/core/djangoapps/olx_rest_api/adapters.py index f5cf4aa823cd..5c37ba639fb8 100644 --- a/openedx/core/djangoapps/olx_rest_api/adapters.py +++ b/openedx/core/djangoapps/olx_rest_api/adapters.py @@ -86,7 +86,7 @@ def collect_assets_from_text(text, course_id, include_content=False): path = path[8:] info = { 'path': path, - 'url': '/' + str(course_id.make_asset_key("asset", path)), + 'url': '/' + str(StaticContent.compute_location(course_id, path)), } if include_content: content = get_asset_content_from_path(course_id, path)