Skip to content

Add a content libraries import from courseware API and management command [SE-4619] #27715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bradenmacdonald merged 1 commit into from
Aug 17, 2021
Merged

Add a content libraries import from courseware API and management command [SE-4619] #27715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
382 changes: 360 additions & 22 deletions openedx/core/djangoapps/content_libraries/api.py
View file
Open in desktop

Large diffs are not rendered by default.

146 changes: 146 additions & 0 deletions openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
"""
Command to import modulestore content into Content Libraries.
"""

import argparse
import logging

from django.conf import settings
from django.core.management import BaseCommand, CommandError

from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocatorV2
from openedx.core.djangoapps.content_libraries import api as contentlib_api


log = logging.getLogger(__name__)


class Command(BaseCommand):
"""
Import modulestore content, references by a course, into a Content Libraries
library.
"""

def add_arguments(self, parser):
"""
Add arguments to the argument parser.
"""
parser.add_argument(
'library-key',
type=LibraryLocatorV2.from_string,
help=('Usage key of the Content Library to import content into.'),
)
parser.add_argument(
'course-key',
type=CourseKey.from_string,
help=('The Course Key string, used to identify the course to import '
'content from.'),
)
subparser = parser.add_subparsers(
title='Courseware location and methods',
dest='method',
description=('Select the method and location to locate the course and '
'its contents.')
)
api_parser = subparser.add_parser(
'api',

help=('Query and retrieve course blocks from a remote instance using '
'Open edX course and OLX export APIs. You need to enable API access '
'on the instance.')
)
api_parser.add_argument(
'--lms-url',
default=settings.LMS_ROOT_URL,
help=("The LMS URL, used to retrieve course content (default: "
"'%(default)s')."),
)
api_parser.add_argument(
'--studio-url',
default=f"https://{settings.CMS_BASE}",
help=("The Studio URL, used to retrieve block OLX content (default: "
"'%(default)s')"),
)
oauth_group = api_parser.add_mutually_exclusive_group(required=False)
oauth_group.add_argument(
'--oauth-creds-file',
type=argparse.FileType('r'),
help=('The edX OAuth credentials in a filename. The first line is '
'the OAuth key, second line is the OAuth secret. This is '
'preferred compared to passing the credentials in the command '
'line.'),
)
oauth_group.add_argument(
'--oauth-creds',
nargs=2,
help=('The edX OAuth credentials in the command line. The first '
'argument is the OAuth secret, the second argument is the '
'OAuth key. Notice that command line arguments are insecure, '
'see `--oauth-creds-file`.'),
)
subparser.add_parser(
'modulestore',
help=("Use a local modulestore instance to retrieve blocks database on "
"the instance where the command is being run. You don't need "
"to enable API access.")
)

def handle(self, *args, **options):
"""
Collect all blocks from a course that are "importable" and write them to the
a blockstore library.
"""

# Search for the library.

try:
contentlib_api.get_library(options['library-key'])
except contentlib_api.ContentLibraryNotFound as exc:
raise CommandError("The library specified does not exist: "
f"{options['library-key']}") from exc

# Validate the method and its arguments, instantiate the openedx client.

if options['method'] == 'api':
if options['oauth_creds_file']:
with options['oauth_creds_file'] as creds_f:
oauth_key, oauth_secret = [v.strip() for v in creds_f.readlines()]
elif options['oauth_creds']:
oauth_key, oauth_secret = options['oauth_creds']
else:
raise CommandError("Method 'api' requires one of the "
"--oauth-* options, and none was specified.")
edx_client = contentlib_api.EdxApiImportClient(
options['lms_url'],
options['studio_url'],
oauth_key,
oauth_secret,
library_key=options['library-key'],
)
elif options['method'] == 'modulestore':
edx_client = contentlib_api.EdxModulestoreImportClient(
library_key=options['library-key'],
)
else:
raise CommandError(f"Method not supported: {options['method']}")

failed_blocks = []

def on_progress(block_key, block_num, block_count, exception=None):
self.stdout.write(f"{block_num}/{block_count}: {block_key}: ", ending='')
# In case stdout is a term and line buffered:
self.stdout.flush()
if exception:
self.stdout.write(self.style.ERROR('❌'))
log.error('Failed to import block: %s', block_key, exc_info=exception)
failed_blocks.append(block_key)
else:
self.stdout.write(self.style.SUCCESS('✓'))

edx_client.import_blocks_from_course(options['course-key'], on_progress)

if failed_blocks:
self.stdout.write(self.style.ERROR(f"❌ {len(failed_blocks)} failed:"))
for key in failed_blocks:
self.stdout.write(self.style.ERROR(str(key)))
28 changes: 28 additions & 0 deletions openedx/core/djangoapps/content_libraries/migrations/0005_contentlibraryblockimporttask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Generated by Django 2.2.24 on 2021-07-16 23:20

from django.db import migrations, models
import django.db.models.deletion
from opaque_keys.edx.django.models import CourseKeyField


class Migration(migrations.Migration):

dependencies = [
('content_libraries', '0004_contentlibrary_license'),
]

operations = [
migrations.CreateModel(
name='ContentLibraryBlockImportTask',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('state', models.CharField(choices=[('created', 'Task was created, but not queued to run.'), ('pending', 'Task was created and queued to run.'), ('running', 'Task is running.'), ('failed', 'Task finished, but some blocks failed to import.'), ('successful', 'Task finished successfully.')], default='created', help_text='The state of the block import task.', max_length=30, verbose_name='state')),
('progress', models.FloatField(default=0.0, help_text='A float from 0.0 to 1.0 representing the task progress.', verbose_name='progress')),
('course_id', CourseKeyField(max_length=255, db_index=True, verbose_name='course ID', help_text='ID of the imported course.')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='import_tasks', to='content_libraries.ContentLibrary')),
],
options={'ordering': ['-created_at', '-updated_at']},
),
]
86 changes: 85 additions & 1 deletion openedx/core/djangoapps/content_libraries/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
"""
Models for new Content Libraries
Models for new Content Libraries.
"""

import contextlib

from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.translation import ugettext_lazy as _

from opaque_keys.edx.django.models import CourseKeyField
from opaque_keys.edx.locator import LibraryLocatorV2
from openedx.core.djangoapps.content_libraries.constants import (
LIBRARY_TYPES, COMPLEX, LICENSE_OPTIONS,
Expand Down Expand Up @@ -127,3 +132,82 @@ def save(self, *args, **kwargs): # lint-amnesty, pylint: disable=arguments-diff
def __str__(self):
who = self.user.username if self.user else self.group.name
return f"ContentLibraryPermission ({self.access_level} for {who})"


class ContentLibraryBlockImportTask(models.Model):
"""
Model of a task to import blocks from an external source (e.g. modulestore).
"""

library = models.ForeignKey(
ContentLibrary,
on_delete=models.CASCADE,
related_name='import_tasks',
)

TASK_CREATED = 'created'
TASK_PENDING = 'pending'
TASK_RUNNING = 'running'
TASK_FAILED = 'failed'
TASK_SUCCESSFUL = 'successful'

TASK_STATE_CHOICES = (
(TASK_CREATED, _('Task was created, but not queued to run.')),
(TASK_PENDING, _('Task was created and queued to run.')),
(TASK_RUNNING, _('Task is running.')),
(TASK_FAILED, _('Task finished, but some blocks failed to import.')),
(TASK_SUCCESSFUL, _('Task finished successfully.')),
)

state = models.CharField(
choices=TASK_STATE_CHOICES,
default=TASK_CREATED,
max_length=30,
verbose_name=_('state'),
help_text=_('The state of the block import task.'),
)

progress = models.FloatField(
default=0.0,
verbose_name=_('progress'),
help_text=_('A float from 0.0 to 1.0 representing the task progress.'),
)

course_id = CourseKeyField(
max_length=255,
db_index=True,
verbose_name=_('course ID'),
help_text=_('ID of the imported course.'),
)

created_at = models.DateTimeField(auto_now_add=True)

updated_at = models.DateTimeField(auto_now=True)

class Meta:
ordering = ['-created_at', '-updated_at']

@classmethod
@contextlib.contextmanager
def execute(cls, import_task_id):
"""
A context manager to manage a task that is being executed.
"""
self = cls.objects.get(pk=import_task_id)
self.state = self.TASK_RUNNING
self.save()
try:
yield self
self.state = self.TASK_SUCCESSFUL
except: # pylint: disable=broad-except
self.state = self.TASK_FAILED
raise
finally:
self.save()

def save_progress(self, progress):
self.progress = progress
self.save(update_fields=['progress', 'updated_at'])

def __str__(self):
return f'{self.course_id} to {self.library} #{self.pk}'
43 changes: 40 additions & 3 deletions openedx/core/djangoapps/content_libraries/serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,12 @@
ALL_RIGHTS_RESERVED,
LICENSE_OPTIONS,
)
from openedx.core.djangoapps.content_libraries.models import ContentLibraryPermission
from openedx.core.djangoapps.content_libraries.models import (
ContentLibraryPermission, ContentLibraryBlockImportTask
)
from openedx.core.lib import blockstore_api
from openedx.core.lib.api.serializers import CourseKeyField


DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'

Expand Down Expand Up @@ -85,12 +89,18 @@ class ContentLibraryPermissionSerializer(ContentLibraryPermissionLevelSerializer
group_name = serializers.CharField(source="group.name", allow_null=True, allow_blank=False, default=None)


class ContentLibraryFilterSerializer(serializers.Serializer):
class BaseFilterSerializer(serializers.Serializer):
"""
Serializer for filtering library listings.
Base serializer for filtering listings on the content library APIs.
"""
text_search = serializers.CharField(default=None, required=False)
org = serializers.CharField(default=None, required=False)


class ContentLibraryFilterSerializer(BaseFilterSerializer):
"""
Serializer for filtering library listings.
"""
type = serializers.ChoiceField(choices=LIBRARY_TYPES, default=None, required=False)


Expand Down Expand Up @@ -190,3 +200,30 @@ class LibraryXBlockStaticFilesSerializer(serializers.Serializer):
Serializes a LibraryXBlockStaticFile (or a BundleFile)
"""
files = LibraryXBlockStaticFileSerializer(many=True)


class ContentLibraryBlockImportTaskSerializer(serializers.ModelSerializer):
"""
Serializer for a Content Library block import task.
"""

org = serializers.SerializerMethodField()

def get_org(self, obj):
return obj.course_id.org

class Meta:
model = ContentLibraryBlockImportTask
fields = '__all__'


class ContentLibraryBlockImportTaskCreateSerializer(serializers.Serializer):
"""
Serializer to create a new block import task.

The serializer accepts the following parameter:

- The courseware course key to import blocks from.
"""

course_key = CourseKeyField()
40 changes: 40 additions & 0 deletions openedx/core/djangoapps/content_libraries/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""
Celery tasks for Content Libraries.
"""


import logging

from celery import shared_task
from celery_utils.logged_task import LoggedTask

from opaque_keys.edx.keys import CourseKey

from . import api
from .models import ContentLibraryBlockImportTask


logger = logging.getLogger(__name__)


@shared_task(base=LoggedTask)
def import_blocks_from_course(import_task_id, course_key_str):
"""
A Celery task to import blocks from a course through modulestore.
"""

course_key = CourseKey.from_string(course_key_str)

with ContentLibraryBlockImportTask.execute(import_task_id) as import_task:

def on_progress(block_key, block_num, block_count, exception=None):
if exception:
logger.exception('Import block failed: %s', block_key)
else:
logger.info('Import block succesful: %s', block_key)
import_task.save_progress(block_num / block_count)

edx_client = api.EdxModulestoreImportClient(library=import_task.library)
edx_client.import_blocks_from_course(
course_key, on_progress
)
Loading