diff --git a/cms/envs/common.py b/cms/envs/common.py index 3d7683ae841a..bfecbffcaadf 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -111,6 +111,10 @@ # Enterprise service settings ENTERPRISE_CATALOG_INTERNAL_ROOT_URL, + # Blockstore + BLOCKSTORE_USE_BLOCKSTORE_APP_API, + BUNDLE_ASSET_STORAGE_SETTINGS, + # Methods to derive settings _make_mako_template_dirs, _make_locale_paths, @@ -1746,6 +1750,9 @@ # For edx ace template tags 'edx_ace', + + # Blockstore + 'blockstore.apps.bundles', ] @@ -2102,6 +2109,7 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', + 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', ] ############################ Cache Configuration ############################### diff --git a/cms/envs/test.py b/cms/envs/test.py index d8fa4ff8cc70..a42b73336a4b 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -27,6 +27,8 @@ # import settings from LMS for consistent behavior with CMS from lms.envs.test import ( # pylint: disable=wrong-import-order + BLOCKSTORE_USE_BLOCKSTORE_APP_API, + BLOCKSTORE_API_URL, COMPREHENSIVE_THEME_DIRS, # unimport:skip DEFAULT_FILE_STORAGE, ECOMMERCE_API_URL, @@ -40,7 +42,8 @@ REGISTRATION_EXTRA_FIELDS, GRADES_DOWNLOAD, SITE_NAME, - WIKI_ENABLED + WIKI_ENABLED, + XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE, ) @@ -175,6 +178,12 @@ 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, + 'blockstore': { + 'KEY_PREFIX': 'blockstore', + 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', + 'LOCATION': 'edx_loc_mem_cache', + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, } ############################### BLOCKSTORE ##################################### @@ -182,6 +191,13 @@ RUN_BLOCKSTORE_TESTS = os.environ.get('EDXAPP_RUN_BLOCKSTORE_TESTS', 'no').lower() in ('true', 'yes', '1') BLOCKSTORE_API_URL = os.environ.get('EDXAPP_BLOCKSTORE_API_URL', "http://edx.devstack.blockstore-test:18251/api/v1/") BLOCKSTORE_API_AUTH_TOKEN = os.environ.get('EDXAPP_BLOCKSTORE_API_AUTH_TOKEN', 'edxapp-test-key') +BUNDLE_ASSET_STORAGE_SETTINGS = dict( + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), +) ################################# CELERY ###################################### @@ -267,12 +283,6 @@ TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch710') TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200')) -############################# TEMPLATE CONFIGURATION ############################# -# Adds mako template dirs for content_libraries tests -MAKO_TEMPLATE_DIRS_BASE.append( - COMMON_ROOT / 'lib' / 'capa' / 'capa' / 'templates' -) - ########################## AUTHOR PERMISSION ####################### FEATURES['ENABLE_CREATOR_GROUP'] = False diff --git a/cms/urls.py b/cms/urls.py index 541e2aee854e..8a881339e220 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -271,6 +271,8 @@ except ImportError: pass + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + urlpatterns += static( settings.VIDEO_IMAGE_SETTINGS['STORAGE_KWARGS']['base_url'], document_root=settings.VIDEO_IMAGE_SETTINGS['STORAGE_KWARGS']['location'] diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index 45884adb4f9e..76f18d115b42 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -9,7 +9,7 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import AssetKey, CourseKey -from opaque_keys.edx.locator import AssetLocator +from opaque_keys.edx.locator import AssetLocator, LibraryLocatorV2 from PIL import Image from xmodule.assetstore.assetmgr import AssetManager @@ -123,7 +123,7 @@ def get_static_path_from_location(location): @staticmethod def get_base_url_path_for_course_assets(course_key): # lint-amnesty, pylint: disable=missing-function-docstring - if course_key is None: + if (course_key is None) or isinstance(course_key, LibraryLocatorV2): return None assert isinstance(course_key, CourseKey) diff --git a/common/test/problem.html b/common/test/problem.html deleted file mode 100644 index 0c63e205e056..000000000000 --- a/common/test/problem.html +++ /dev/null @@ -1,96 +0,0 @@ - -<%page expression_filter="h"/> -<%! -from django.utils.translation import ngettext, gettext as _ -from openedx.core.djangolib.markup import HTML -%> - -<%namespace name='static' file='static_content.html'/> -

- ${ problem['name'] } -

- -
- -
- ${ HTML(problem['html']) } -
- - -
- % if demand_hint_possible: - - - - % endif - % if save_button: - - - - % endif - % if reset_button: - - - - % endif - % if answer_available: - - - - % endif -
-
- - - % if submit_disabled_cta: - % if submit_disabled_cta.get('event_data'): - - - - - (${submit_disabled_cta['description']}) - % else: -
- - % for form_name, form_value in submit_disabled_cta['form_values'].items(): - - % endfor - - - - - (${submit_disabled_cta['description']}) -
- % endif - % endif -
- ## When attempts are not 0, the CTA above will contain a message about the number of used attempts - % if attempts_allowed and (not submit_disabled_cta or attempts_used == 0): - ${ngettext("You have used {num_used} of {num_total} attempt", "You have used {num_used} of {num_total} attempts", attempts_allowed).format(num_used=attempts_used, num_total=attempts_allowed)} - % endif - ${_("Some problems have options such as save, reset, hints, or show answer. These options follow the Submit button.")} -
-
-
-
- - diff --git a/common/test/problem_ajax.html b/common/test/problem_ajax.html deleted file mode 100644 index bbe365bd9249..000000000000 --- a/common/test/problem_ajax.html +++ /dev/null @@ -1,16 +0,0 @@ - -
- -
diff --git a/common/test/video.html b/common/test/video.html deleted file mode 100644 index ea6a54d99418..000000000000 --- a/common/test/video.html +++ /dev/null @@ -1,122 +0,0 @@ - -<%page expression_filter="h"/> - -<%! -from django.utils.translation import ugettext as _ -from openedx.core.djangolib.js_utils import ( - dump_js_escaped_json, js_escaped_string -) -%> -% if display_name is not UNDEFINED and display_name is not None: -

${display_name}

-% endif - -
-
- -
-
- - -
-
-
- - -
-
-
- -
-
- -
- - % if download_video_link or track or handout or branding_info: -

${_('Downloads and transcripts')}

-
- % if download_video_link: -
-

${_('Video')}

- - ${_('Download video file')} - -
- % endif - % if track: -
-

${_('Transcripts')}

- % if transcript_download_format: -
    - % for item in transcript_download_formats_list: -
  • - <% dname = _("Download {file}").format(file=item['display_name']) %> - ${dname} -
    • - % endfor -
- % else: - ${_('Download transcript')} - % endif -
- % endif - % if handout: -
-

${_('Handouts')}

- ${_('Download Handout')} -
- % endif - % if branding_info: -
- ${branding_info['logo_tag']} - -
- % endif -
- % endif -
-% if cdn_eval: - -% endif; diff --git a/lms/envs/common.py b/lms/envs/common.py index 6c8c62a4446a..acd86c3e90eb 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1071,6 +1071,7 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', + 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] @@ -3224,6 +3225,9 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For save for later 'lms.djangoapps.save_for_later', + + # Blockstore + 'blockstore.apps.bundles', ] ######################### CSRF ######################################### @@ -4951,6 +4955,10 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring BLOCKSTORE_PUBLIC_URL_ROOT = 'http://localhost:18250' BLOCKSTORE_API_URL = 'http://localhost:18250/api/v1/' +# Disable the Blockstore app API by default. +# See openedx.core.lib.blockstore_api.config for details. +BLOCKSTORE_USE_BLOCKSTORE_APP_API = False + # .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE # .. setting_default: default # .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. @@ -4966,6 +4974,40 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # configured to expire after one hour. BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 +# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY +# .. setting_default: None +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is +# set, and `boto3` is installed, this is used as an AWS IAM access key for +# generating signed, read-only URLs for blockstore assets stored in S3. +# Otherwise, URLs are generated based on the default storage configuration. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_URL_STORAGE_KEY = None + +# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET +# .. setting_default: None +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is +# set, and `boto3` is installed, this is used as an AWS IAM secret key for +# generating signed, read-only URLs for blockstore assets stored in S3. +# Otherwise, URLs are generated based on the default storage configuration. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_URL_STORAGE_SECRET = None + +# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS +# .. setting_default: dict, appropriate for file system storage. +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is +# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_STORAGE_SETTINGS = dict( + # Backend storage + # STORAGE_CLASS='storages.backends.s3boto.S3BotoStorage', + # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), +) + ######################### MICROSITE ############################### MICROSITE_ROOT_DIR = '/edx/app/edxapp/edx-microsite' MICROSITE_CONFIGURATION = {} diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index e255327f89d2..133d3cf1e3ba 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -19,7 +19,6 @@ # Don't use S3 in devstack, fall back to filesystem del DEFAULT_FILE_STORAGE -MEDIA_ROOT = "/edx/var/edxapp/uploads" ORA2_FILEUPLOAD_BACKEND = 'django' diff --git a/lms/envs/test.py b/lms/envs/test.py index 406b5156ac98..b50f3e5e8a05 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -223,16 +223,6 @@ }, } -############################### BLOCKSTORE ##################################### -# Blockstore tests -RUN_BLOCKSTORE_TESTS = os.environ.get('EDXAPP_RUN_BLOCKSTORE_TESTS', 'no').lower() in ('true', 'yes', '1') -BLOCKSTORE_API_URL = os.environ.get('EDXAPP_BLOCKSTORE_API_URL', "http://edx.devstack.blockstore-test:18251/api/v1/") -BLOCKSTORE_API_AUTH_TOKEN = os.environ.get('EDXAPP_BLOCKSTORE_API_AUTH_TOKEN', 'edxapp-test-key') -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'blockstore' # This must be set to a working cache for the tests to pass - -# Dummy secret key for dev -SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' - ############################# SECURITY SETTINGS ################################ # Default to advanced security in common.py, so tests can reset here to use # a simpler security model @@ -314,7 +304,7 @@ ############################ STATIC FILES ############################# DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' MEDIA_ROOT = TEST_ROOT / "uploads" -MEDIA_URL = "/static/uploads/" +MEDIA_URL = "/uploads/" STATICFILES_DIRS.append(("uploads", MEDIA_ROOT)) _NEW_STATICFILES_DIRS = [] @@ -553,6 +543,24 @@ derive_settings(__name__) +############################### BLOCKSTORE ##################################### +# Blockstore tests +RUN_BLOCKSTORE_TESTS = os.environ.get('EDXAPP_RUN_BLOCKSTORE_TESTS', 'no').lower() in ('true', 'yes', '1') +BLOCKSTORE_USE_BLOCKSTORE_APP_API = not RUN_BLOCKSTORE_TESTS +BLOCKSTORE_API_URL = os.environ.get('EDXAPP_BLOCKSTORE_API_URL', "http://edx.devstack.blockstore-test:18251/api/v1/") +BLOCKSTORE_API_AUTH_TOKEN = os.environ.get('EDXAPP_BLOCKSTORE_API_AUTH_TOKEN', 'edxapp-test-key') +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'blockstore' # This must be set to a working cache for the tests to pass +BUNDLE_ASSET_STORAGE_SETTINGS = dict( + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), +) + +# Dummy secret key for dev +SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' + ############### Settings for edx-rbac ############### SYSTEM_WIDE_ROLE_CLASSES = os.environ.get("SYSTEM_WIDE_ROLE_CLASSES", []) diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 9e22df4196bd..c1d95c9d6819 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -65,7 +65,7 @@ from django.contrib.auth.models import AbstractUser, Group from django.core.exceptions import PermissionDenied from django.core.validators import validate_unicode_slug -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.utils.translation import gettext as _ from elasticsearch.exceptions import ConnectionError as ElasticConnectionError from lxml import etree @@ -436,15 +436,18 @@ def create_library( ) # Now create the library reference in our database: try: - ref = ContentLibrary.objects.create( - org=org, - slug=slug, - type=library_type, - bundle_uuid=bundle.uuid, - allow_public_learning=allow_public_learning, - allow_public_read=allow_public_read, - license=library_license, - ) + # Atomic transaction required because if this fails, + # we need to delete the bundle in the exception handler. + with transaction.atomic(): + ref = ContentLibrary.objects.create( + org=org, + slug=slug, + type=library_type, + bundle_uuid=bundle.uuid, + allow_public_learning=allow_public_learning, + allow_public_read=allow_public_read, + license=library_license, + ) except IntegrityError: delete_bundle(bundle.uuid) raise LibraryAlreadyExists(slug) # lint-amnesty, pylint: disable=raise-missing-from diff --git a/openedx/core/djangoapps/content_libraries/library_bundle.py b/openedx/core/djangoapps/content_libraries/library_bundle.py index b2d3c34efa15..ed03b1801222 100644 --- a/openedx/core/djangoapps/content_libraries/library_bundle.py +++ b/openedx/core/djangoapps/content_libraries/library_bundle.py @@ -2,7 +2,6 @@ Helper code for working with Blockstore bundles that contain OLX """ -import dateutil.parser import logging # lint-amnesty, pylint: disable=wrong-import-order from functools import lru_cache # lint-amnesty, pylint: disable=wrong-import-order @@ -347,12 +346,17 @@ def get_static_files_for_definition(self, definition_key): problem/quiz1/definition.xml problem/quiz1/static/image1.png Then this will return - [BundleFile(path="image1.png", size, url, hash_digest)] + [BundleFileData(path="image1.png", size, url, hash_digest)] """ path_prefix = self.get_static_prefix_for_definition(definition_key) path_prefix_len = len(path_prefix) return [ - blockstore_api.BundleFile(path=f.path[path_prefix_len:], size=f.size, url=f.url, hash_digest=f.hash_digest) + blockstore_api.BundleFileData( + path=f.path[path_prefix_len:], + size=f.size, + url=f.url, + hash_digest=f.hash_digest, + ) for f in get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if f.path.startswith(path_prefix) ] @@ -369,8 +373,7 @@ def get_last_published_time(self): version = get_bundle_version_number(self.bundle_uuid) if version == 0: return None - created_at_str = blockstore_api.get_bundle_version(self.bundle_uuid, version)['snapshot']['created_at'] - last_published_time = dateutil.parser.parse(created_at_str) + last_published_time = blockstore_api.get_bundle_version(self.bundle_uuid, version).created_at self.cache.set(cache_key, last_published_time) return last_published_time diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 38138bac2e95..34aee552ee66 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -4,10 +4,12 @@ from contextlib import contextmanager from io import BytesIO from urllib.parse import urlencode -import unittest -from unittest.mock import patch +from unittest import mock, skipUnless +from urllib.parse import urlparse from django.conf import settings +from django.test import LiveServerTestCase +from django.test.client import RequestFactory from django.test.utils import override_settings from organizations.models import Organization from rest_framework.test import APITestCase, APIClient @@ -47,8 +49,39 @@ URL_BLOCK_XBLOCK_HANDLER = '/api/xblock/v2/xblocks/{block_key}/handler/{user_id}-{secure_token}/{handler_name}/' -# Decorator for tests that require blockstore -requires_blockstore = unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") +# Decorators for tests that require the blockstore service/app +requires_blockstore = skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") + +requires_blockstore_app = skipUnless(settings.BLOCKSTORE_USE_BLOCKSTORE_APP_API, "Requires blockstore app") + + +class BlockstoreAppTestMixin: + """ + Sets up the environment for tests to be run using the installed Blockstore app. + """ + def setUp(self): + """ + Ensure there's an active request, so that bundle file URLs can be made absolute. + """ + super().setUp() + + # Patch the blockstore get_current_request to use our live_server_url + mock.patch('blockstore.apps.api.methods.get_current_request', + mock.Mock(return_value=self._get_current_request())).start() + self.addCleanup(mock.patch.stopall) + + def _get_current_request(self): + """ + Returns a request object using the live_server_url, if available. + """ + request_args = {} + if hasattr(self, 'live_server_url'): + live_server_url = urlparse(self.live_server_url) + name, port = live_server_url.netloc.split(':') + request_args['SERVER_NAME'] = name + request_args['SERVER_PORT'] = port or '80' + request_args['wsgi.url_scheme'] = live_server_url.scheme + return RequestFactory().request(**request_args) def elasticsearch_test(func): @@ -63,9 +96,11 @@ def elasticsearch_test(func): 'host': settings.TEST_ELASTICSEARCH_HOST, 'port': settings.TEST_ELASTICSEARCH_PORT, }])(func) - func = patch("openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS", new={ - 'refresh': 'wait_for' - })(func) + func = mock.patch( + "openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS", + new={ + 'refresh': 'wait_for' + })(func) return func else: @classmethod @@ -77,20 +112,19 @@ def mock_perform(cls, filter_terms, text_search): size=MAX_SIZE ) - func = patch( + func = mock.patch( "openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS", new={} )(func) - func = patch( + func = mock.patch( "openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase._perform_elastic_search", new=mock_perform )(func) return func -@requires_blockstore @skip_unless_cms # Content Libraries REST API is only available in Studio -class ContentLibrariesRestApiTest(APITestCase): +class _ContentLibrariesRestApiTestMixin: """ Base class for Blockstore-based Content Libraries test that use the REST API @@ -350,3 +384,26 @@ def _get_block_handler_url(self, block_key, handler_name): """ url = URL_BLOCK_GET_HANDLER_URL.format(block_key=block_key, handler_name=handler_name) return self._api('get', url, None, expect_response=200)["handler_url"] + + +@requires_blockstore +class ContentLibrariesRestApiBlockstoreServiceTest(_ContentLibrariesRestApiTestMixin, APITestCase): + """ + Base class for Blockstore-based Content Libraries test that use the REST API + and the standalone Blockstore service. + """ + + +@requires_blockstore_app +class ContentLibrariesRestApiTest( + _ContentLibrariesRestApiTestMixin, + BlockstoreAppTestMixin, + APITestCase, + LiveServerTestCase, +): + """ + Base class for Blockstore-based Content Libraries test that use the REST API + and the installed Blockstore app. + + We run this test with a live server, so that the blockstore asset files can be served. + """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index 7cdbe54eebe7..a39c5371e829 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -19,6 +19,7 @@ from openedx.core.djangoapps.content_libraries.libraries_index import LibraryBlockIndexer, ContentLibraryIndexer from openedx.core.djangoapps.content_libraries.tests.base import ( + ContentLibrariesRestApiBlockstoreServiceTest, ContentLibrariesRestApiTest, elasticsearch_test, URL_BLOCK_METADATA_URL, @@ -33,8 +34,7 @@ @ddt.ddt -@elasticsearch_test -class ContentLibrariesTest(ContentLibrariesRestApiTest): +class ContentLibrariesTestMixin: """ General tests for Blockstore-based Content Libraries @@ -369,6 +369,60 @@ def test_library_blocks(self): # fin + def test_library_blocks_studio_view(self): + """ + Test the happy path of working with an HTML XBlock in a the studio_view of a content library. + """ + lib = self._create_library(slug="testlib2", title="A Test Library", description="Testing XBlocks") + lib_id = lib["id"] + assert lib['has_unpublished_changes'] is False + + # A library starts out empty: + assert self._get_library_blocks(lib_id) == [] + + # Add a 'html' XBlock to the library: + block_data = self._add_block_to_library(lib_id, "html", "html1") + self.assertDictContainsEntries(block_data, { + "id": "lb:CL-TEST:testlib2:html:html1", + "display_name": "Text", + "block_type": "html", + "has_unpublished_changes": True, + }) + block_id = block_data["id"] + # Confirm that the result contains a definition key, but don't check its value, + # which for the purposes of these tests is an implementation detail. + assert 'def_key' in block_data + + # now the library should contain one block and have unpublished changes: + assert self._get_library_blocks(lib_id) == [block_data] + assert self._get_library(lib_id)['has_unpublished_changes'] is True + + # Publish the changes: + self._commit_library_changes(lib_id) + assert self._get_library(lib_id)['has_unpublished_changes'] is False + # And now the block information should also show that block has no unpublished changes: + block_data["has_unpublished_changes"] = False + self.assertDictContainsEntries(self._get_library_block(block_id), block_data) + assert self._get_library_blocks(lib_id) == [block_data] + + # Now update the block's OLX: + orig_olx = self._get_library_block_olx(block_id) + assert ' 1 +@elasticsearch_test +class ContentLibrariesBlockstoreServiceTest( + ContentLibrariesTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + General tests for Blockstore-based Content Libraries, using the standalone Blockstore service. + """ + + +@elasticsearch_test +class ContentLibrariesTest( + ContentLibrariesTestMixin, + ContentLibrariesRestApiTest, +): + """ + General tests for Blockstore-based Content Libraries, using the installed Blockstore app. + """ + + @ddt.ddt class ContentLibraryXBlockValidationTest(APITestCase): """Tests only focused on service validation, no Blockstore needed.""" @@ -941,8 +1015,7 @@ def student_view(self, context=None): @ddt.ddt -@elasticsearch_test -class ContentLibrariesXBlockTypeOverrideTest(ContentLibrariesRestApiTest): +class ContentLibrariesXBlockTypeOverrideTestMixin: """ Tests for Blockstore-based Content Libraries XBlock API, where the expected XBlock type returned is overridden in the request. @@ -1073,3 +1146,23 @@ def test_block_type_handler(self, slug, api_args, expected_type): assert f"lb:CL-TEST:handler-{slug}:video:handler-{slug}" in response['transcripts']['en'] del response['transcripts']['en'] assert response == expected_response + + +@elasticsearch_test +class ContentLibrariesXBlockTypeOverrideBlockstoreServiceTest( + ContentLibrariesXBlockTypeOverrideTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + Tests for the Content Libraries XBlock API type override using the standalone Blockstore service. + """ + + +@elasticsearch_test +class ContentLibrariesXBlockTypeOverrideTest( + ContentLibrariesXBlockTypeOverrideTestMixin, + ContentLibrariesRestApiTest, +): + """ + Tests for the Content Libraries XBlock API type override using the installed Blockstore app. + """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py index 194786db51fe..fb6464c35c06 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py @@ -10,12 +10,14 @@ from search.search_engine_base import SearchEngine from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer -from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test +from openedx.core.djangoapps.content_libraries.tests.base import ( + ContentLibrariesRestApiBlockstoreServiceTest, + ContentLibrariesRestApiTest, + elasticsearch_test, +) -@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) -@elasticsearch_test -class ContentLibraryIndexerTest(ContentLibrariesRestApiTest): +class ContentLibraryIndexerTestMixin: """ Tests the operation of ContentLibraryIndexer """ @@ -181,7 +183,27 @@ def verify_uncommitted_libraries(library_key, has_unpublished_changes, has_unpub @override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) @elasticsearch_test -class LibraryBlockIndexerTest(ContentLibrariesRestApiTest): +class ContentLibraryIndexerBlockstoreServiceTest( + ContentLibraryIndexerTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + Tests the operation of ContentLibraryIndexer using the standalone Blockstore service. + """ + + +@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) +@elasticsearch_test +class ContentLibraryIndexerTest( + ContentLibraryIndexerTestMixin, + ContentLibrariesRestApiTest, +): + """ + Tests the operation of ContentLibraryIndexer using the installed Blockstore app. + """ + + +class LibraryBlockIndexerTestMixin: """ Tests the operation of LibraryBlockIndexer """ @@ -279,3 +301,25 @@ def test_crud_block(self): LibraryBlockIndexer.get_items([block['id']]) self._delete_library(lib['id']) assert LibraryBlockIndexer.get_items([block['id']]) == [] + + +@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) +@elasticsearch_test +class LibraryBlockIndexerBlockstoreServiceTest( + LibraryBlockIndexerTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + Tests the operation of LibraryBlockIndexer using the standalone Blockstore service. + """ + + +@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) +@elasticsearch_test +class LibraryBlockIndexerTest( + LibraryBlockIndexerTestMixin, + ContentLibrariesRestApiTest, +): + """ + Tests the operation of LibraryBlockIndexer using the installed Blockstore app. + """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py index 36e768bce1b4..cf9da3dff33c 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py @@ -6,7 +6,8 @@ from completion.test_utils import CompletionWaffleTestMixin from django.db import connections -from django.test import TestCase, override_settings +from django.test import LiveServerTestCase, TestCase +from django.utils.text import slugify from organizations.models import Organization from rest_framework.test import APIClient from xblock.core import XBlock @@ -14,7 +15,9 @@ from lms.djangoapps.courseware.model_data import get_score from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.content_libraries.tests.base import ( + BlockstoreAppTestMixin, requires_blockstore, + requires_blockstore_app, URL_BLOCK_RENDER_VIEW, URL_BLOCK_GET_HANDLER_URL, URL_BLOCK_METADATA_URL, @@ -33,26 +36,27 @@ class ContentLibraryContentTestMixin: """ Mixin for content library tests that creates two students and a library. """ - @classmethod - def setUpClass(cls): - super().setUpClass() + def setUp(self): + super().setUp() # Create a couple students that the tests can use - cls.student_a = UserFactory.create(username="Alice", email="alice@example.com", password="edx") - cls.student_b = UserFactory.create(username="Bob", email="bob@example.com", password="edx") + self.student_a = UserFactory.create(username="Alice", email="alice@example.com", password="edx") + self.student_b = UserFactory.create(username="Bob", email="bob@example.com", password="edx") + # Create a collection using Blockstore API directly only because there # is not yet any Studio REST API for doing so: - cls.collection = blockstore_api.create_collection("Content Library Test Collection") + self.collection = blockstore_api.create_collection("Content Library Test Collection") # Create an organization - cls.organization = Organization.objects.create( + self.organization = Organization.objects.create( name="Content Libraries Tachyon Exploration & Survey Team", short_name="CL-TEST", ) - cls.library = library_api.create_library( - collection_uuid=cls.collection.uuid, + _, slug = self.id().rsplit('.', 1) + self.library = library_api.create_library( + collection_uuid=self.collection.uuid, library_type=COMPLEX, - org=cls.organization, - slug=cls.__name__, - title=(cls.__name__ + " Test Lib"), + org=self.organization, + slug=slugify(slug), + title=(f"{slug} Test Lib"), description="", allow_public_learning=True, allow_public_read=False, @@ -60,10 +64,7 @@ def setUpClass(cls): ) -@requires_blockstore -# EphemeralKeyValueStore requires a working cache, and the default test cache doesn't work: -@override_settings(XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE='blockstore') -class ContentLibraryRuntimeTest(ContentLibraryContentTestMixin, TestCase): +class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin): """ Basic tests of the Blockstore-based XBlock runtime using XBlocks in a content library. @@ -181,10 +182,25 @@ def test_xblock_metadata(self): @requires_blockstore +class ContentLibraryRuntimeBServiceTest(ContentLibraryRuntimeTestMixin, TestCase): + """ + Tests XBlock runtime using XBlocks in a content library using the standalone Blockstore service. + """ + + +@requires_blockstore_app +class ContentLibraryRuntimeTest(ContentLibraryRuntimeTestMixin, BlockstoreAppTestMixin, LiveServerTestCase): + """ + Tests XBlock runtime using XBlocks in a content library using the installed Blockstore app. + + We run this test with a live server, so that the blockstore asset files can be served. + """ + + # We can remove the line below to enable this in Studio once we implement a session-backed # field data store which we can use for both studio users and anonymous users @skip_unless_lms -class ContentLibraryXBlockUserStateTest(ContentLibraryContentTestMixin, TestCase): +class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin): """ Test that the Blockstore-based XBlock runtime can store and retrieve student state for XBlocks when learners access blocks directly in a library context, @@ -487,8 +503,27 @@ def test_i18n(self): @requires_blockstore +class ContentLibraryXBlockUserStateBServiceTest(ContentLibraryXBlockUserStateTestMixin, TestCase): + """ + Tests XBlock user state for XBlocks in a content library using the standalone Blockstore service. + """ + + +@requires_blockstore_app +class ContentLibraryXBlockUserStateTest( + ContentLibraryXBlockUserStateTestMixin, + BlockstoreAppTestMixin, + LiveServerTestCase, +): + """ + Tests XBlock user state for XBlocks in a content library using the installed Blockstore app. + + We run this test with a live server, so that the blockstore asset files can be served. + """ + + @skip_unless_lms # No completion tracking in Studio -class ContentLibraryXBlockCompletionTest(ContentLibraryContentTestMixin, CompletionWaffleTestMixin, TestCase): +class ContentLibraryXBlockCompletionTestMixin(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): """ Test that the Blockstore-based XBlocks can track their completion status using the completion library. @@ -539,3 +574,30 @@ def get_block_completion_status(): # Now the block is completed assert get_block_completion_status() == 1 + + +@requires_blockstore +class ContentLibraryXBlockCompletionBServiceTest( + ContentLibraryXBlockCompletionTestMixin, + CompletionWaffleTestMixin, + TestCase, +): + """ + Test that the Blockstore-based XBlocks can track their completion status + using the standalone Blockstore service. + """ + + +@requires_blockstore_app +class ContentLibraryXBlockCompletionTest( + ContentLibraryXBlockCompletionTestMixin, + CompletionWaffleTestMixin, + BlockstoreAppTestMixin, + LiveServerTestCase, +): + """ + Test that the Blockstore-based XBlocks can track their completion status + using the installed Blockstore app. + + We run this test with a live server, so that the blockstore asset files can be served. + """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py index 0342e27ebda4..a3ff52c789e9 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py @@ -5,7 +5,10 @@ import requests -from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest +from openedx.core.djangoapps.content_libraries.tests.base import ( + ContentLibrariesRestApiBlockstoreServiceTest, + ContentLibrariesRestApiTest, +) # Binary data representing an SVG image file SVG_DATA = """ @@ -23,7 +26,7 @@ """ -class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): +class ContentLibrariesStaticAssetsTestMixin: """ Tests for static asset files in Blockstore-based Content Libraries @@ -166,3 +169,21 @@ def check_download(): self._commit_library_changes(library["id"]) check_sjson() check_download() + + +class ContentLibrariesStaticAssetsBlockstoreServiceTest( + ContentLibrariesStaticAssetsTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + Tests for static asset files in Blockstore-based Content Libraries, using the standalone Blockstore service. + """ + + +class ContentLibrariesStaticAssetsTest( + ContentLibrariesStaticAssetsTestMixin, + ContentLibrariesRestApiTest, +): + """ + Tests for static asset files in Blockstore-based Content Libraries, using the installed Blockstore app. + """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py index 965a9229984e..58dd7e9b214a 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py @@ -8,6 +8,7 @@ from openedx.core.djangoapps.content_libraries.constants import PROBLEM from .base import ( + ContentLibrariesRestApiBlockstoreServiceTest, ContentLibrariesRestApiTest, URL_LIB_LTI_JWKS, skip_unless_cms, @@ -49,9 +50,7 @@ def test_when_no_keys_then_return_empty(self): self.assertJSONEqual(response.content, '{"keys": []}') -@override_features(ENABLE_CONTENT_LIBRARIES=True, - ENABLE_CONTENT_LIBRARIES_LTI_TOOL=True) -class LibraryBlockLtiUrlViewTest(ContentLibrariesRestApiTest): +class LibraryBlockLtiUrlViewTestMixin: """ Test generating LTI URL for a block in a library. """ @@ -66,12 +65,12 @@ def test_lti_url_generation(self): ) block = self._add_block_to_library(library['id'], PROBLEM, PROBLEM) - usage_key = str(block.usage_key) + usage_key = str(block['id']) url = f'/api/libraries/v2/blocks/{usage_key}/lti/' expected_lti_url = f"/api/libraries/v2/lti/1.3/launch/?id={usage_key}" - response = self._api("GET", url, None, expect_response=200) + response = self._api("get", url, None, expect_response=200) self.assertDictEqual(response, {"lti_url": expected_lti_url}) @@ -79,9 +78,26 @@ def test_block_not_found(self): """ Test the LTI URL cannot be generated as the block not found. """ + self._api("get", '/api/libraries/v2/blocks/lb:CL-TEST:libgg:problem:bad-block/lti/', None, expect_response=404) - self._create_library( - slug="libgg", title="A Test Library", description="Testing library", library_type=PROBLEM, - ) - self._api("GET", '/api/libraries/v2/blocks/not-existing-key/lti/', None, expect_response=404) +@override_features(ENABLE_CONTENT_LIBRARIES=True, + ENABLE_CONTENT_LIBRARIES_LTI_TOOL=True) +class LibraryBlockLtiUrlViewBlockstoreServiceTest( + LibraryBlockLtiUrlViewTestMixin, + ContentLibrariesRestApiBlockstoreServiceTest, +): + """ + Test generating LTI URL for a block in a library, using the standalone Blockstore service. + """ + + +@override_features(ENABLE_CONTENT_LIBRARIES=True, + ENABLE_CONTENT_LIBRARIES_LTI_TOOL=True) +class LibraryBlockLtiUrlViewTest( + LibraryBlockLtiUrlViewTestMixin, + ContentLibrariesRestApiTest, +): + """ + Test generating LTI URL for a block in a library, using the installed Blockstore app. + """ diff --git a/openedx/core/djangoapps/xblock/runtime/runtime.py b/openedx/core/djangoapps/xblock/runtime/runtime.py index 444ba3348878..ba06df6da54b 100644 --- a/openedx/core/djangoapps/xblock/runtime/runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/runtime.py @@ -241,6 +241,8 @@ def service(self, block, service_name): anonymous_user_id=self.anonymous_student_id, ) elif service_name == "mako": + if self.system.student_data_mode == XBlockRuntimeSystem.STUDENT_DATA_EPHEMERAL: + return MakoService(namespace_prefix='lms.') return MakoService() elif service_name == "i18n": return ModuleI18nService(block=block) diff --git a/openedx/core/djangolib/tests/test_blockstore_cache.py b/openedx/core/djangolib/tests/test_blockstore_cache.py index c4535f4b0a36..84d071ab8f95 100644 --- a/openedx/core/djangolib/tests/test_blockstore_cache.py +++ b/openedx/core/djangolib/tests/test_blockstore_cache.py @@ -1,12 +1,15 @@ """ Tests for BundleCache """ - -import unittest from unittest.mock import patch -from django.conf import settings +from django.test import TestCase from openedx.core.djangolib.blockstore_cache import BundleCache +from openedx.core.djangoapps.content_libraries.tests.base import ( + BlockstoreAppTestMixin, + requires_blockstore, + requires_blockstore_app, +) from openedx.core.lib import blockstore_api as api @@ -23,9 +26,8 @@ def setUpClass(cls): cls.draft = api.get_or_create_bundle_draft(cls.bundle.uuid, draft_name="test-draft") -@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") @patch('openedx.core.djangolib.blockstore_cache.MAX_BLOCKSTORE_CACHE_DELAY', 0) -class BundleCacheTest(TestWithBundleMixin, unittest.TestCase): +class BundleCacheTestMixin(TestWithBundleMixin): """ Tests for BundleCache """ @@ -80,8 +82,7 @@ def test_bundle_draft_cache(self): assert cache.get(key2) is None -@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") -class BundleCacheClearTest(TestWithBundleMixin, unittest.TestCase): +class BundleCacheClearTest(TestWithBundleMixin, TestCase): """ Tests for BundleCache's clear() method. Requires MAX_BLOCKSTORE_CACHE_DELAY to be non-zero. This clear() method does @@ -111,3 +112,17 @@ def test_bundle_cache_clear(self): # Now "clear" the cache, forcing the check of the new version: cache.clear() assert cache.get(key1) is None + + +@requires_blockstore +class BundleCacheBlockstoreServiceTest(BundleCacheTestMixin, TestCase): + """ + Tests BundleCache using the standalone Blockstore service. + """ + + +@requires_blockstore_app +class BundleCacheTest(BundleCacheTestMixin, BlockstoreAppTestMixin, TestCase): + """ + Tests BundleCache using the installed Blockstore app. + """ diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py index 483e97c2ebf7..50b352578cdf 100644 --- a/openedx/core/lib/blockstore_api/__init__.py +++ b/openedx/core/lib/blockstore_api/__init__.py @@ -5,15 +5,16 @@ openedx.core.djangolib.blockstore_cache) together with these API methods for improved performance. """ -from .models import ( - Collection, - Bundle, - Draft, - BundleFile, - DraftFile, - LinkReference, - LinkDetails, - DraftLinkDetails, +from blockstore.apps.api.data import ( + BundleFileData, +) +from blockstore.apps.api.exceptions import ( + CollectionNotFound, + BundleNotFound, + DraftNotFound, + BundleVersionNotFound, + BundleFileNotFound, + BundleStorageError, ) from .methods import ( # Collections: @@ -47,11 +48,3 @@ # Misc: force_browser_url, ) -from .exceptions import ( - BlockstoreException, - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleFileNotFound, - BundleStorageError, -) diff --git a/openedx/core/lib/blockstore_api/config/__init__.py b/openedx/core/lib/blockstore_api/config/__init__.py new file mode 100644 index 000000000000..4cd2999d2e32 --- /dev/null +++ b/openedx/core/lib/blockstore_api/config/__init__.py @@ -0,0 +1,13 @@ +""" +Helper method to indicate when the blockstore app API is enabled. +""" +from django.conf import settings +from .waffle import BLOCKSTORE_USE_BLOCKSTORE_APP_API # pylint: disable=invalid-django-waffle-import + + +def use_blockstore_app(): + """ + Use the Blockstore app API if the settings say to (e.g. in test) + or if the waffle switch is enabled. + """ + return settings.BLOCKSTORE_USE_BLOCKSTORE_APP_API or BLOCKSTORE_USE_BLOCKSTORE_APP_API.is_enabled() diff --git a/openedx/core/lib/blockstore_api/config/waffle.py b/openedx/core/lib/blockstore_api/config/waffle.py new file mode 100644 index 000000000000..ebbacb7f599d --- /dev/null +++ b/openedx/core/lib/blockstore_api/config/waffle.py @@ -0,0 +1,20 @@ +""" +Toggles for blockstore. +""" + +from edx_toggles.toggles import WaffleSwitch + +# .. toggle_name: blockstore.use_blockstore_app_api +# .. toggle_implementation: WaffleSwitch +# .. toggle_default: False +# .. toggle_description: Enable to use the installed blockstore app's Python API directly instead of the +# external blockstore service REST API. +# The blockstore REST API is used by default. +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2022-01-13 +# .. toggle_target_removal_date: None +# .. toggle_tickets: TNL-8705, BD-14 +# .. toggle_warnings: This temporary feature toggle does not have a target removal date. +BLOCKSTORE_USE_BLOCKSTORE_APP_API = WaffleSwitch( + 'blockstore.use_blockstore_app_api', __name__ +) diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/openedx/core/lib/blockstore_api/db_routers.py new file mode 100644 index 000000000000..fd0ff50c9510 --- /dev/null +++ b/openedx/core/lib/blockstore_api/db_routers.py @@ -0,0 +1,60 @@ +""" +Blockstore database router. + +Blockstore started life as an IDA, but is now a Django app plugin within edx-platform. +This router exists to smooth blockstore's transition into edxapp. +""" +from django.conf import settings + + +class BlockstoreRouter: + """ + A Database Router that uses the ``blockstore`` database, if it's configured in settings. + """ + ROUTE_APP_LABELS = {'bundles'} + DATABASE_NAME = 'blockstore' + + def _use_blockstore(self, model): + """ + Return True if the given model should use the blockstore database. + + Ensures that a ``blockstore`` database is configured, and checks the ``model``'s app label. + """ + return (self.DATABASE_NAME in settings.DATABASES) and (model._meta.app_label in self.ROUTE_APP_LABELS) + + def db_for_read(self, model, **hints): # pylint: disable=unused-argument + """ + Use the BlockstoreRouter.DATABASE_NAME when reading blockstore app tables. + """ + if self._use_blockstore(model): + return self.DATABASE_NAME + return None + + def db_for_write(self, model, **hints): # pylint: disable=unused-argument + """ + Use the BlockstoreRouter.DATABASE_NAME when writing to blockstore app tables. + """ + if self._use_blockstore(model): + return self.DATABASE_NAME + return None + + def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument + """ + Allow relations if both objects are blockstore app models. + """ + if self._use_blockstore(obj1) and self._use_blockstore(obj2): + return True + return None + + def allow_migrate(self, db, app_label, model_name=None, **hints): # pylint: disable=unused-argument + """ + Ensure the blockstore tables only appear in the blockstore database. + """ + if model_name is not None: + model = hints.get('model') + if model is not None and self._use_blockstore(model): + return db == self.DATABASE_NAME + if db == self.DATABASE_NAME: + return False + + return None diff --git a/openedx/core/lib/blockstore_api/exceptions.py b/openedx/core/lib/blockstore_api/exceptions.py deleted file mode 100644 index b58251d31fd6..000000000000 --- a/openedx/core/lib/blockstore_api/exceptions.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Exceptions that may be raised by the Blockstore API -""" - - -class BlockstoreException(Exception): - pass - - -class NotFound(BlockstoreException): - pass - - -class CollectionNotFound(NotFound): - pass - - -class BundleNotFound(NotFound): - pass - - -class DraftNotFound(NotFound): - pass - - -class BundleFileNotFound(NotFound): - pass - - -class BundleStorageError(BlockstoreException): - pass diff --git a/openedx/core/lib/blockstore_api/methods.py b/openedx/core/lib/blockstore_api/methods.py index 0dacae8a04b3..7d7c65decdc1 100644 --- a/openedx/core/lib/blockstore_api/methods.py +++ b/openedx/core/lib/blockstore_api/methods.py @@ -3,6 +3,7 @@ """ import base64 +from functools import wraps from urllib.parse import urlencode from uuid import UUID @@ -11,23 +12,40 @@ from django.core.exceptions import ImproperlyConfigured import requests -from .models import ( - Bundle, - Collection, - Draft, - BundleFile, - DraftFile, - LinkDetails, - LinkReference, - DraftLinkDetails, +from blockstore.apps.api.data import ( + BundleData, + CollectionData, + DraftData, + BundleVersionData, + BundleFileData, + DraftFileData, + BundleLinkData, + DraftLinkData, + Dependency, ) -from .exceptions import ( +from blockstore.apps.api.exceptions import ( NotFound, CollectionNotFound, BundleNotFound, DraftNotFound, BundleFileNotFound, ) +import blockstore.apps.api.methods as blockstore_api_methods + +from .config import use_blockstore_app + + +def toggle_blockstore_api(func): + """ + Decorator function to toggle usage of the Blockstore service + and the in-built Blockstore app dependency. + """ + @wraps(func) + def wrapper(*args, **kwargs): + if use_blockstore_app(): + return getattr(blockstore_api_methods, func.__name__)(*args, **kwargs) + return func(*args, **kwargs) + return wrapper def api_url(*path_parts): @@ -55,17 +73,17 @@ def api_request(method, url, **kwargs): def _collection_from_response(data): """ Given data about a Collection returned by any blockstore REST API, convert it to - a Collection instance. + a CollectionData instance. """ - return Collection(uuid=UUID(data['uuid']), title=data['title']) + return CollectionData(uuid=UUID(data['uuid']), title=data['title']) def _bundle_from_response(data): """ Given data about a Bundle returned by any blockstore REST API, convert it to - a Bundle instance. + a BundleData instance. """ - return Bundle( + return BundleData( uuid=UUID(data['uuid']), title=data['title'], description=data['description'], @@ -78,25 +96,51 @@ def _bundle_from_response(data): ) +def _bundle_version_from_response(data): + """ + Given data about a BundleVersion returned by any blockstore REST API, convert it to + a BundleVersionData instance. + """ + return BundleVersionData( + bundle_uuid=UUID(data['bundle_uuid']), + version=data.get('version', 0), + change_description=data['change_description'], + created_at=dateutil.parser.parse(data['snapshot']['created_at']), + files={ + path: BundleFileData(path=path, **filedata) + for path, filedata in data['snapshot']['files'].items() + }, + links={ + name: BundleLinkData( + name=name, + direct=Dependency(**link["direct"]), + indirect=[Dependency(**ind) for ind in link["indirect"]], + ) + for name, link in data['snapshot']['links'].items() + } + ) + + def _draft_from_response(data): """ Given data about a Draft returned by any blockstore REST API, convert it to - a Draft instance. + a DraftData instance. """ - return Draft( + return DraftData( uuid=UUID(data['uuid']), bundle_uuid=UUID(data['bundle_uuid']), name=data['name'], + created_at=dateutil.parser.parse(data['staged_draft']['created_at']), updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']), files={ - path: DraftFile(path=path, **file) + path: DraftFileData(path=path, **file) for path, file in data['staged_draft']['files'].items() }, links={ - name: DraftLinkDetails( + name: DraftLinkData( name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], + direct=Dependency(**link["direct"]), + indirect=[Dependency(**ind) for ind in link["indirect"]], modified=link["modified"], ) for name, link in data['staged_draft']['links'].items() @@ -104,6 +148,7 @@ def _draft_from_response(data): ) +@toggle_blockstore_api def get_collection(collection_uuid): """ Retrieve metadata about the specified collection @@ -118,6 +163,7 @@ def get_collection(collection_uuid): return _collection_from_response(data) +@toggle_blockstore_api def create_collection(title): """ Create a new collection. @@ -126,6 +172,7 @@ def create_collection(title): return _collection_from_response(result) +@toggle_blockstore_api def update_collection(collection_uuid, title): """ Update a collection's title @@ -136,6 +183,7 @@ def update_collection(collection_uuid, title): return _collection_from_response(result) +@toggle_blockstore_api def delete_collection(collection_uuid): """ Delete a collection @@ -144,6 +192,7 @@ def delete_collection(collection_uuid): api_request('delete', api_url('collections', str(collection_uuid))) +@toggle_blockstore_api def get_bundles(uuids=None, text_search=None): """ Get the details of all bundles @@ -159,6 +208,7 @@ def get_bundles(uuids=None, text_search=None): return [_bundle_from_response(item) for item in response] +@toggle_blockstore_api def get_bundle(bundle_uuid): """ Retrieve metadata about the specified bundle @@ -173,6 +223,7 @@ def get_bundle(bundle_uuid): return _bundle_from_response(data) +@toggle_blockstore_api def create_bundle(collection_uuid, slug, title="New Bundle", description=""): """ Create a new bundle. @@ -188,6 +239,7 @@ def create_bundle(collection_uuid, slug, title="New Bundle", description=""): return _bundle_from_response(result) +@toggle_blockstore_api def update_bundle(bundle_uuid, **fields): """ Update a bundle's title, description, slug, or collection. @@ -207,6 +259,7 @@ def update_bundle(bundle_uuid, **fields): return _bundle_from_response(result) +@toggle_blockstore_api def delete_bundle(bundle_uuid): """ Delete a bundle @@ -215,6 +268,7 @@ def delete_bundle(bundle_uuid): api_request('delete', api_url('bundles', str(bundle_uuid))) +@toggle_blockstore_api def get_draft(draft_uuid): """ Retrieve metadata about the specified draft. @@ -228,6 +282,7 @@ def get_draft(draft_uuid): return _draft_from_response(data) +@toggle_blockstore_api def get_or_create_bundle_draft(bundle_uuid, draft_name): """ Retrieve metadata about the specified draft. @@ -245,6 +300,7 @@ def get_or_create_bundle_draft(bundle_uuid, draft_name): return get_draft(UUID(response["uuid"])) +@toggle_blockstore_api def commit_draft(draft_uuid): """ Commit all of the pending changes in the draft, creating a new version of @@ -255,6 +311,7 @@ def commit_draft(draft_uuid): api_request('post', api_url('drafts', str(draft_uuid), 'commit')) +@toggle_blockstore_api def delete_draft(draft_uuid): """ Delete the specified draft, removing any staged changes/files/deletes. @@ -264,6 +321,7 @@ def delete_draft(draft_uuid): api_request('delete', api_url('drafts', str(draft_uuid))) +@toggle_blockstore_api def get_bundle_version(bundle_uuid, version_number): """ Get the details of the specified bundle version @@ -271,9 +329,10 @@ def get_bundle_version(bundle_uuid, version_number): if version_number == 0: return None version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number)) - return api_request('get', version_url) + return _bundle_version_from_response(api_request('get', version_url)) +@toggle_blockstore_api def get_bundle_version_files(bundle_uuid, version_number): """ Get a list of the files in the specified bundle version @@ -281,9 +340,10 @@ def get_bundle_version_files(bundle_uuid, version_number): if version_number == 0: return [] version_info = get_bundle_version(bundle_uuid, version_number) - return [BundleFile(path=path, **file_metadata) for path, file_metadata in version_info["snapshot"]["files"].items()] + return list(version_info.files.values()) +@toggle_blockstore_api def get_bundle_version_links(bundle_uuid, version_number): """ Get a dictionary of the links in the specified bundle version @@ -291,22 +351,16 @@ def get_bundle_version_links(bundle_uuid, version_number): if version_number == 0: return {} version_info = get_bundle_version(bundle_uuid, version_number) - return { - name: LinkDetails( - name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], - ) - for name, link in version_info['snapshot']['links'].items() - } + return version_info.links +@toggle_blockstore_api def get_bundle_files_dict(bundle_uuid, use_draft=None): """ Get a dict of all the files in the specified bundle. Returns a dict where the keys are the paths (strings) and the values are - BundleFile or DraftFile tuples. + BundleFileData or DraftFileData tuples. """ bundle = get_bundle(bundle_uuid) if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test @@ -319,6 +373,7 @@ def get_bundle_files_dict(bundle_uuid, use_draft=None): return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)} +@toggle_blockstore_api def get_bundle_files(bundle_uuid, use_draft=None): """ Get an iterator over all the files in the specified bundle or draft. @@ -326,12 +381,13 @@ def get_bundle_files(bundle_uuid, use_draft=None): return get_bundle_files_dict(bundle_uuid, use_draft).values() +@toggle_blockstore_api def get_bundle_links(bundle_uuid, use_draft=None): """ Get a dict of all the links in the specified bundle. Returns a dict where the keys are the link names (strings) and the values - are LinkDetails or DraftLinkDetails tuples. + are BundleLinkData or DraftLinkData tuples. """ bundle = get_bundle(bundle_uuid) if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test @@ -344,6 +400,7 @@ def get_bundle_links(bundle_uuid, use_draft=None): return get_bundle_version_links(bundle_uuid, bundle.latest_version) +@toggle_blockstore_api def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): """ Get the metadata of the specified file. @@ -358,6 +415,7 @@ def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): ) +@toggle_blockstore_api def get_bundle_file_data(bundle_uuid, path, use_draft=None): """ Read all the data in the given bundle file and return it as a @@ -370,6 +428,7 @@ def get_bundle_file_data(bundle_uuid, path, use_draft=None): return r.content +@toggle_blockstore_api def write_draft_file(draft_uuid, path, contents): """ Create or overwrite the file at 'path' in the specified draft with the given @@ -382,11 +441,12 @@ def write_draft_file(draft_uuid, path, contents): """ api_request('patch', api_url('drafts', str(draft_uuid)), json={ 'files': { - path: encode_str_for_draft(contents) if contents is not None else None, + path: _encode_str_for_draft(contents) if contents is not None else None, }, }) +@toggle_blockstore_api def set_draft_link(draft_uuid, link_name, bundle_uuid, version): """ Create or replace the link with the given name in the specified draft so @@ -405,7 +465,7 @@ def set_draft_link(draft_uuid, link_name, bundle_uuid, version): }) -def encode_str_for_draft(input_str): +def _encode_str_for_draft(input_str): """ Given a string, return UTF-8 representation that is then base64 encoded. """ @@ -416,10 +476,10 @@ def encode_str_for_draft(input_str): return base64.b64encode(binary) +@toggle_blockstore_api def force_browser_url(blockstore_file_url): """ - Ensure that the given URL Blockstore is a URL accessible from the end user's - browser. + Ensure that the given devstack URL is a URL accessible from the end user's browser. """ # Hack: on some devstacks, we must necessarily use different URLs for # accessing Blockstore file data from within and outside of docker diff --git a/openedx/core/lib/blockstore_api/models.py b/openedx/core/lib/blockstore_api/models.py deleted file mode 100644 index 8f2127ca9025..000000000000 --- a/openedx/core/lib/blockstore_api/models.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Data models used for Blockstore API Client -""" - -from datetime import datetime -from uuid import UUID - -import attr - - -def _convert_to_uuid(value): - if not isinstance(value, UUID): - return UUID(value) - return value - - -@attr.s(frozen=True) -class Collection: - """ - Metadata about a blockstore collection - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - title = attr.ib(type=str) - - -@attr.s(frozen=True) -class Bundle: - """ - Metadata about a blockstore bundle - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - title = attr.ib(type=str) - description = attr.ib(type=str) - slug = attr.ib(type=str) - drafts = attr.ib(type=dict) # Dict of drafts, where keys are the draft names and values are draft UUIDs - # Note that if latest_version is 0, it means that no versions yet exist - latest_version = attr.ib(type=int, validator=attr.validators.instance_of(int)) - - -@attr.s(frozen=True) -class Draft: - """ - Metadata about a blockstore draft - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - name = attr.ib(type=str) - updated_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) - files = attr.ib(type=dict) - links = attr.ib(type=dict) - - -@attr.s(frozen=True) -class BundleFile: - """ - Metadata about a file in a blockstore bundle or draft. - """ - path = attr.ib(type=str) - size = attr.ib(type=int) - url = attr.ib(type=str) - hash_digest = attr.ib(type=str) - - -@attr.s(frozen=True) -class DraftFile(BundleFile): - """ - Metadata about a file in a blockstore draft. - """ - modified = attr.ib(type=bool) # Was this file modified in the draft? - - -@attr.s(frozen=True) -class LinkReference: - """ - A pointer to a specific BundleVersion - """ - bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - version = attr.ib(type=int) - snapshot_digest = attr.ib(type=str) - - -@attr.s(frozen=True) -class LinkDetails: - """ - Details about a specific link in a BundleVersion or Draft - """ - name = attr.ib(type=str) - direct = attr.ib(type=LinkReference) - indirect = attr.ib(type=list) # List of LinkReference objects - - -@attr.s(frozen=True) -class DraftLinkDetails(LinkDetails): - """ - Details about a specific link in a Draft - """ - modified = attr.ib(type=bool) diff --git a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py index f1496503891e..a1239d5a0068 100644 --- a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py +++ b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py @@ -2,22 +2,25 @@ Tests for xblock_utils.py """ -import unittest from uuid import UUID import pytest -from django.conf import settings +from django.test import TestCase from openedx.core.lib import blockstore_api as api +from openedx.core.djangoapps.content_libraries.tests.base import ( + BlockstoreAppTestMixin, + requires_blockstore, + requires_blockstore_app, +) # A fake UUID that won't represent any real bundle/draft/collection: BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') -@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") -class BlockstoreApiClientTest(unittest.TestCase): +class BlockstoreApiClientTestMixin: """ - Test for the Blockstore API Client. + Tests for the Blockstore API Client. The goal of these tests is not to test that Blockstore works correctly, but that the API client can interact with it and all the API client methods @@ -192,3 +195,17 @@ def test_links(self): # Finally, test deleting a link from course's draft: api.set_draft_link(course_draft.uuid, link2_name, None, None) assert not api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name) + + +@requires_blockstore +class BlockstoreServiceApiClientTest(BlockstoreApiClientTestMixin, TestCase): + """ + Test the Blockstore API Client, using the standalone Blockstore service. + """ + + +@requires_blockstore_app +class BlockstoreAppApiClientTest(BlockstoreApiClientTestMixin, BlockstoreAppTestMixin, TestCase): + """ + Test the Blockstore API Client, using the installed Blockstore app. + """ diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 6e8b1c072370..4b1413cf297d 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -68,6 +68,7 @@ django-user-tasks django-waffle django-webpack-loader # Used to wire webpack bundles into the django asset pipeline djangorestframework +drf-nested-routers # Required by blockstore done-xblock edx-ace edx-api-doc-tools diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 05db7734119d..2203f181f0e8 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/github.in -e common/lib/capa # via # -r requirements/edx/local.in @@ -54,6 +56,7 @@ attrs==21.4.0 # via # -r requirements/edx/base.in # aiohttp + # blockstore # edx-ace # openedx-events babel==2.9.1 @@ -184,6 +187,7 @@ django==3.2.13 # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -275,6 +279,8 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv +django-environ==0.8.1 + # via blockstore django-fernet-fields==0.6 # via # -r requirements/edx/base.in @@ -283,6 +289,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/base.in + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -362,6 +369,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/base.in + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -375,8 +383,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/base.in + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -389,6 +399,8 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via blockstore djangorestframework-xml==2.0.0 # via edx-enterprise docopt==0.6.2 @@ -401,12 +413,16 @@ done-xblock==2.0.4 # via -r requirements/edx/base.in drf-jwt==1.19.2 # via edx-drf-extensions +drf-nested-routers==0.93.4 + # via blockstore drf-yasg==1.20.0 # via edx-api-doc-tools edx-ace==1.5.0 # via -r requirements/edx/base.in edx-api-doc-tools==1.6.0 - # via -r requirements/edx/base.in + # via + # -r requirements/edx/base.in + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/base.in edx-braze-client==0.1.3 @@ -425,13 +441,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/base.in edx-django-release-util==1.2.0 - # via -r requirements/edx/base.in + # via + # -r requirements/edx/base.in + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.in edx-django-utils==4.6.0 # via - # -r requirements/edx/base.in # -r requirements/edx/github.in + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -690,6 +708,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/base.in + # blockstore # edx-django-utils nltk==3.7 # via @@ -762,6 +781,8 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in +pyblake2==1.1.2 + # via blockstore pycountry==22.3.5 # via -r requirements/edx/base.in pycparser==2.21 @@ -853,6 +874,7 @@ pytz==2022.1 # via # -r requirements/edx/base.in # babel + # blockstore # celery # django # django-ses @@ -993,6 +1015,7 @@ soupsieve==2.3.2 sqlparse==0.4.2 # via # -r requirements/edx/base.in + # blockstore # django staff-graded-xblock==2.0.1 # via -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 08da28ab9cf5..eb8b3e284e08 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/testing.txt -e common/lib/capa # via # -r requirements/edx/testing.txt @@ -79,6 +81,7 @@ attrs==21.4.0 # via # -r requirements/edx/testing.txt # aiohttp + # blockstore # edx-ace # jsonschema # openedx-events @@ -270,6 +273,7 @@ django==3.2.13 # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -368,6 +372,8 @@ django-crum==0.7.9 # super-csv django-debug-toolbar==3.2.4 # via -r requirements/edx/development.in +django-environ==0.8.1 + # via blockstore django-fernet-fields==0.6 # via # -r requirements/edx/testing.txt @@ -376,6 +382,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/testing.txt + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -461,6 +468,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/testing.txt + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -474,8 +482,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/testing.txt + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -488,6 +498,8 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via blockstore djangorestframework-xml==2.0.0 # via # -r requirements/edx/testing.txt @@ -509,6 +521,10 @@ drf-jwt==1.19.2 # via # -r requirements/edx/testing.txt # edx-drf-extensions +drf-nested-routers==0.93.4 + # via + # -r requirements/edx/testing.txt + # blockstore drf-yasg==1.20.0 # via # -r requirements/edx/testing.txt @@ -516,11 +532,15 @@ drf-yasg==1.20.0 edx-ace==1.5.0 # via -r requirements/edx/testing.txt edx-api-doc-tools==1.6.0 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/testing.txt edx-braze-client==0.1.3 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-bulk-grades==1.0.0 # via # -r requirements/edx/testing.txt @@ -535,12 +555,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/testing.txt edx-django-release-util==1.2.0 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/testing.txt edx-django-utils==4.6.0 # via # -r requirements/edx/testing.txt + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -931,6 +954,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/testing.txt + # blockstore # edx-django-utils nltk==3.7 # via @@ -1042,6 +1066,8 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt +pyblake2==1.1.2 + # via blockstore pycodestyle==2.8.0 # via -r requirements/edx/testing.txt pycountry==22.3.5 @@ -1210,6 +1236,7 @@ pytz==2022.1 # via # -r requirements/edx/testing.txt # babel + # blockstore # celery # django # django-ses @@ -1417,6 +1444,7 @@ sphinxcontrib-serializinghtml==1.1.5 sqlparse==0.4.2 # via # -r requirements/edx/testing.txt + # blockstore # django # django-debug-toolbar staff-graded-xblock==2.0.1 diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 50bfa1da6c46..6c3327e927be 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -63,6 +63,7 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752 git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776#egg=django-require==1.0.12 # Our libraries: +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 -e git+https://github.com/edx/codejail.git@3.1.3#egg=codejail==3.1.3 -e git+https://github.com/edx/RateXBlock.git@2.0.1#egg=rate-xblock -e git+https://github.com/edx-solutions/xblock-google-drive.git@2d176468e33c0713c911b563f8f65f7cf232f5b6#egg=xblock-google-drive diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 8d6d638cd15e..6fd70e98aca0 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/base.txt -e common/lib/capa # via # -r requirements/edx/base.txt @@ -74,6 +76,7 @@ attrs==21.4.0 # via # -r requirements/edx/base.txt # aiohttp + # blockstore # edx-ace # openedx-events # outcome @@ -257,6 +260,7 @@ distlib==0.3.4 # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -352,6 +356,10 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv +django-environ==0.8.1 + # via + # -r requirements/edx/base.txt + # blockstore django-fernet-fields==0.6 # via # -r requirements/edx/base.txt @@ -360,6 +368,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/base.txt + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -445,6 +454,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/base.txt + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -458,8 +468,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/base.txt + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -472,6 +484,10 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via + # -r requirements/edx/base.txt + # blockstore djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt @@ -491,6 +507,10 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions +drf-nested-routers==0.93.4 + # via + # -r requirements/edx/base.txt + # blockstore drf-yasg==1.20.0 # via # -r requirements/edx/base.txt @@ -498,11 +518,15 @@ drf-yasg==1.20.0 edx-ace==1.5.0 # via -r requirements/edx/base.txt edx-api-doc-tools==1.6.0 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/base.txt edx-braze-client==0.1.3 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-bulk-grades==1.0.0 # via # -r requirements/edx/base.txt @@ -517,12 +541,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/base.txt edx-django-release-util==1.2.0 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.txt edx-django-utils==4.6.0 # via # -r requirements/edx/base.txt + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -877,6 +904,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/base.txt + # blockstore # edx-django-utils nltk==3.7 # via @@ -980,6 +1008,10 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt +pyblake2==1.1.2 + # via + # -r requirements/edx/base.txt + # blockstore pycodestyle==2.8.0 # via -r requirements/edx/testing.in pycountry==22.3.5 @@ -1136,6 +1168,7 @@ pytz==2022.1 # via # -r requirements/edx/base.txt # babel + # blockstore # celery # django # django-ses @@ -1311,6 +1344,7 @@ soupsieve==2.3.2 sqlparse==0.4.2 # via # -r requirements/edx/base.txt + # blockstore # django staff-graded-xblock==2.0.1 # via -r requirements/edx/base.txt