diff --git a/.gitignore b/.gitignore index f642f3b2..9cabbe5d 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,5 @@ dev.db .vscode +# Media files (for uploads) +media/ diff --git a/olx_importer/apps.py b/olx_importer/apps.py index 416dfe97..68fb21b8 100644 --- a/olx_importer/apps.py +++ b/olx_importer/apps.py @@ -9,5 +9,6 @@ class OLXImporterConfig(AppConfig): """ Configuration for the OLX Importer Django application. """ + name = "olx_importer" verbose_name = "OLX Importer" diff --git a/olx_importer/management/commands/load_components.py b/olx_importer/management/commands/load_components.py index 1e06df71..08fc47cc 100644 --- a/olx_importer/management/commands/load_components.py +++ b/olx_importer/management/commands/load_components.py @@ -11,9 +11,9 @@ Open Question: If the data model is extensible, how do we know whether a change has really happened between what's currently stored/published for a particular -item and the new value we want to set? For Content that's easy, because we have -actual hashes of the data. But it's not clear how that would work for something -like an ComponentVersion. We'd have to have some kind of mechanism where every +item and the new value we want to set? For RawContent that's easy, because we +have actual hashes of the data. But it's not clear how that would work for +something like an ComponentVersion. We'd have to have some kind of mechanism where every pp that wants to attach data gets to answer the question of "has anything changed?" in order to decide if we really make a new ComponentVersion or not. """ @@ -25,22 +25,28 @@ import re import xml.etree.ElementTree as ET +from django.core.files.base import ContentFile from django.core.management.base import BaseCommand from django.db import transaction from openedx_learning.core.publishing.models import LearningPackage, PublishLogEntry from openedx_learning.core.components.models import ( - Content, Component, ComponentVersion, ComponentVersionContent, - ComponentPublishLogEntry, PublishedComponent, + Component, + ComponentVersion, + ComponentVersionRawContent, + ComponentPublishLogEntry, + PublishedComponent, + RawContent, + TextContent, ) from openedx_learning.lib.fields import create_hash_digest -SUPPORTED_TYPES = ['problem', 'video', 'html'] +SUPPORTED_TYPES = ["problem", "video", "html"] logger = logging.getLogger(__name__) class Command(BaseCommand): - help = 'Load sample Component data from course export' + help = "Load sample Component data from course export" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -61,11 +67,10 @@ def init_known_types(self): # officially "text/javascript" mimetypes.add_type("text/javascript", ".js") mimetypes.add_type("text/javascript", ".mjs") - def add_arguments(self, parser): - parser.add_argument('course_data_path', type=pathlib.Path) - parser.add_argument('learning_package_identifier', type=str) + parser.add_argument("course_data_path", type=pathlib.Path) + parser.add_argument("learning_package_identifier", type=str) def handle(self, course_data_path, learning_package_identifier, **options): self.course_data_path = course_data_path @@ -73,8 +78,8 @@ def handle(self, course_data_path, learning_package_identifier, **options): self.load_course_data(learning_package_identifier) def get_course_title(self): - course_type_dir = self.course_data_path / 'course' - course_xml_file = next(course_type_dir.glob('*.xml')) + course_type_dir = self.course_data_path / "course" + course_xml_file = next(course_type_dir.glob("*.xml")) course_root = ET.parse(course_xml_file).getroot() return course_root.attrib.get("display_name", "Unknown Course") @@ -87,9 +92,9 @@ def load_course_data(self, learning_package_identifier): learning_package, _created = LearningPackage.objects.get_or_create( identifier=learning_package_identifier, defaults={ - 'title': title, - 'created': now, - 'updated': now, + "title": title, + "created": now, + "updated": now, }, ) self.learning_package = learning_package @@ -105,11 +110,13 @@ def load_course_data(self, learning_package_identifier): self.import_block_type(block_type, now, publish_log_entry) def create_content(self, static_local_path, now, component_version): - identifier = pathlib.Path('static') / static_local_path + identifier = pathlib.Path("static") / static_local_path real_path = self.course_data_path / identifier mime_type, _encoding = mimetypes.guess_type(identifier) if mime_type is None: - logger.error(f" no mimetype found for {real_path}, defaulting to application/binary") + logger.error( + f" no mimetype found for {real_path}, defaulting to application/binary" + ) mime_type = "application/binary" try: @@ -120,20 +127,26 @@ def create_content(self, static_local_path, now, component_version): hash_digest = create_hash_digest(data_bytes) - content, _created = Content.objects.get_or_create( + raw_content, created = RawContent.objects.get_or_create( learning_package=self.learning_package, mime_type=mime_type, hash_digest=hash_digest, - defaults = dict( - data=data_bytes, + defaults=dict( size=len(data_bytes), created=now, - ) + ), ) - ComponentVersionContent.objects.get_or_create( + if created: + raw_content.file.save( + f"{raw_content.learning_package.uuid}/{hash_digest}", + ContentFile(data_bytes), + ) + + ComponentVersionRawContent.objects.get_or_create( component_version=component_version, - content=content, + raw_content=raw_content, identifier=identifier, + learner_downloadable=True, ) def import_block_type(self, block_type, now, publish_log_entry): @@ -146,35 +159,49 @@ def import_block_type(self, block_type, now, publish_log_entry): static_files_regex = re.compile(r"""['"]\/static\/(.+?)["'\?]""") block_data_path = self.course_data_path / block_type - for xml_file_path in block_data_path.glob('*.xml'): + for xml_file_path in block_data_path.glob("*.xml"): components_found += 1 identifier = xml_file_path.stem # Find or create the Component itself component, _created = Component.objects.get_or_create( learning_package=self.learning_package, - namespace='xblock.v1', + namespace="xblock.v1", type=block_type, identifier=identifier, - defaults = { - 'created': now, - } + defaults={ + "created": now, + }, ) - # Create the Content entry for the raw data... + # Create the RawContent entry for the raw data... data_bytes = xml_file_path.read_bytes() hash_digest = create_hash_digest(data_bytes) - data_str = codecs.decode(data_bytes, 'utf-8') - content, _created = Content.objects.get_or_create( + + raw_content, created = RawContent.objects.get_or_create( learning_package=self.learning_package, - mime_type=f'application/vnd.openedx.xblock.v1.{block_type}+xml', + mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", hash_digest=hash_digest, - defaults = dict( - data=data_bytes, + defaults=dict( + # text=data_str, size=len(data_bytes), created=now, - ) + ), ) + if created: + raw_content.file.save( + f"{raw_content.learning_package.uuid}/{hash_digest}", + ContentFile(data_bytes), + ) + + # Decode as utf-8-sig in order to strip any BOM from the data. + data_str = codecs.decode(data_bytes, "utf-8-sig") + TextContent.objects.create( + raw_content=raw_content, + text=data_str, + length=len(data_str), + ) + # TODO: Get associated file contents, both with the static regex, as # well as with XBlock-specific code that examines attributes in # video and HTML tag definitions. @@ -185,7 +212,7 @@ def import_block_type(self, block_type, now, publish_log_entry): logger.error(f"Parse error for {xml_file_path}: {err}") continue - display_name = block_root.attrib.get('display_name', "") + display_name = block_root.attrib.get("display_name", "") # Create the ComponentVersion component_version = ComponentVersion.objects.create( @@ -195,10 +222,11 @@ def import_block_type(self, block_type, now, publish_log_entry): created=now, created_by=None, ) - ComponentVersionContent.objects.create( + ComponentVersionRawContent.objects.create( component_version=component_version, - content=content, - identifier='source.xml', + raw_content=raw_content, + identifier="source.xml", + learner_downloadable=False, ) static_files_found = static_files_regex.findall(data_str) for static_local_path in static_files_found: diff --git a/openedx_learning/contrib/media_server/__init__.py b/openedx_learning/contrib/media_server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openedx_learning/contrib/media_server/apps.py b/openedx_learning/contrib/media_server/apps.py new file mode 100644 index 00000000..2612be02 --- /dev/null +++ b/openedx_learning/contrib/media_server/apps.py @@ -0,0 +1,22 @@ +from django.apps import AppConfig +from django.core.exceptions import ImproperlyConfigured +from django.conf import settings + + +class MediaServerConfig(AppConfig): + """ + Configuration for the Media Server application. + """ + + name = "openedx_learning.contrib.media_server" + verbose_name = "Learning Core: Media Server" + default_auto_field = "django.db.models.BigAutoField" + + def ready(self): + if not settings.DEBUG: + # Until we get proper security and support for running this app + # under a separate domain, just don't allow it to be run in + # production environments. + raise ImproperlyConfigured( + "The media_server app should only be run in DEBUG mode!" + ) diff --git a/openedx_learning/contrib/media_server/readme.rst b/openedx_learning/contrib/media_server/readme.rst new file mode 100644 index 00000000..aef39566 --- /dev/null +++ b/openedx_learning/contrib/media_server/readme.rst @@ -0,0 +1,16 @@ +Media Server App +================ + +The ``media_server`` app exists to serve media files that are ultimately backed by the RawContent model, *for development purposes and for sites with light-to-moderate traffic*. It also provides an API that can be used by CDNs for high traffic sites. + +Motivation +---------- + +The ``components`` app stores large binary file data by calculating the hash and creating a django-storages backed file named after that hash. This is efficient from a storage point of view, because we don't store redundant copies for every version of a Component. There are at least two drawbacks: + +* We have unintelligibly named files that are confusing for clients. +* Intra-file links between media files break. For instance, if we have a piece of HTML that makes a reference to a VTT file, that filename will have changed. + +This app tries to bridge that gap by serving URLs that preserve the original file names and give the illusion that there is a seprate set of media files for every version of a Component, but does a lookup behind the scenes to serve the correct hash-based-file. + +The big caveat on this is that Django is not really optimized to do this sort of asset serving. The most scalable approach is to have a CDN-backed solution where ``media_server`` serves the locations of files that are converted by worker code to serving the actual assets. (More details to follow when that part gets built out.) diff --git a/openedx_learning/contrib/media_server/urls.py b/openedx_learning/contrib/media_server/urls.py new file mode 100644 index 00000000..554966ac --- /dev/null +++ b/openedx_learning/contrib/media_server/urls.py @@ -0,0 +1,16 @@ +from django.urls import path + +from .views import component_asset + +urlpatterns = [ + path( + ( + "component_asset/" + "/" + "/" + "/" + "" + ), + component_asset, + ) +] diff --git a/openedx_learning/contrib/media_server/views.py b/openedx_learning/contrib/media_server/views.py new file mode 100644 index 00000000..86e92d13 --- /dev/null +++ b/openedx_learning/contrib/media_server/views.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from django.http import FileResponse +from django.http import Http404 +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied + +from openedx_learning.core.components.api import get_component_version_content + + +def component_asset( + request, learning_package_identifier, component_identifier, version_num, asset_path +): + """ + Serve the ComponentVersion asset data. + + This function maps from a logical URL with Component and verison data like: + media_server/component_asset/course101/finalexam-problem14/1/static/images/fig3.png + To the actual data file as stored in file/object storage, which looks like: + media/055499fd-f670-451a-9727-501ea9dfbf5b/a9528d66739a297aa0cd17106b0bc0f7515b8e78 + + TODO: + * ETag support + * Range queries + * Serving from a different domain than the rest of the service + """ + try: + cvc = get_component_version_content( + learning_package_identifier, component_identifier, version_num, asset_path + ) + except ObjectDoesNotExist: + raise Http404("File not found") + + if not cvc.learner_downloadable and not ( + request.user and request.user.is_superuser + ): + raise PermissionDenied("This file is not publicly downloadable.") + + response = FileResponse(cvc.raw_content.file, filename=Path(asset_path).name) + response["Content-Type"] = cvc.raw_content.mime_type + + return response diff --git a/openedx_learning/contrib/readme.rst b/openedx_learning/contrib/readme.rst index e69de29b..908ed621 100644 --- a/openedx_learning/contrib/readme.rst +++ b/openedx_learning/contrib/readme.rst @@ -0,0 +1,9 @@ +Contrib Package +=============== + +The ``contrib`` package holds Django apps that *could* be implemented in separate repos, but are bundled here because it's more convenient to do so. + +Guidelines +---------- + +Nothing from ``lib`` or ``core`` should *ever* import from ``contrib``. diff --git a/openedx_learning/core/components/admin.py b/openedx_learning/core/components/admin.py index 36e4a799..10712d6a 100644 --- a/openedx_learning/core/components/admin.py +++ b/openedx_learning/core/components/admin.py @@ -1,5 +1,3 @@ -import base64 - from django.contrib import admin from django.db.models.aggregates import Count, Sum from django.template.defaultfilters import filesizeformat @@ -9,20 +7,11 @@ from .models import ( Component, ComponentVersion, - Content, + ComponentVersionRawContent, PublishedComponent, + RawContent, ) - - -class ReadOnlyModelAdmin(admin.ModelAdmin): - def has_add_permission(self, request): - return False - - def has_change_permission(self, request, obj=None): - return False - - def has_delete_permission(self, request, obj=None): - return False +from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin class ComponentVersionInline(admin.TabularInline): @@ -70,8 +59,8 @@ def get_queryset(self, request): "component_version", "component_publish_log_entry__publish_log_entry", ) - .annotate(size=Sum("component_version__contents__size")) - .annotate(content_count=Count("component_version__contents")) + .annotate(size=Sum("component_version__raw_contents__size")) + .annotate(content_count=Count("component_version__raw_contents")) ) readonly_fields = ["component", "component_version", "component_publish_log_entry"] @@ -111,12 +100,16 @@ def identifier(self, pc): """ return format_html( '{}', - reverse("admin:components_componentversion_change", args=(pc.component_version_id,)), + reverse( + "admin:components_componentversion_change", + args=(pc.component_version_id,), + ), pc.component.identifier, ) def content_count(self, pc): return pc.content_count + content_count.short_description = "#" def size(self, pc): @@ -135,24 +128,36 @@ def title(self, pc): return pc.component_version.title -class ContentInline(admin.TabularInline): - model = ComponentVersion.contents.through - fields = ["format_identifier", "format_size", "rendered_data"] - readonly_fields = ["content", "format_identifier", "format_size", "rendered_data"] +class RawContentInline(admin.TabularInline): + model = ComponentVersion.raw_contents.through + fields = [ + "format_identifier", + "format_size", + "learner_downloadable", + "rendered_data", + ] + readonly_fields = [ + "raw_content", + "format_identifier", + "format_size", + "rendered_data", + ] extra = 0 - def rendered_data(self, cv_obj): - return content_preview(cv_obj.content, 100_000) + def rendered_data(self, cvc_obj): + return content_preview(cvc_obj) + + def format_size(self, cvc_obj): + return filesizeformat(cvc_obj.raw_content.size) - def format_size(self, cv_obj): - return filesizeformat(cv_obj.content.size) format_size.short_description = "Size" - def format_identifier(self, cv_obj): + def format_identifier(self, cvc_obj): return format_html( '{}', - reverse("admin:components_content_change", args=(cv_obj.content_id,)), - cv_obj.identifier, + link_for_cvc(cvc_obj), + # reverse("admin:components_content_change", args=(cvc_obj.content_id,)), + cvc_obj.identifier, ) format_identifier.short_description = "Identifier" @@ -166,7 +171,7 @@ class ComponentVersionAdmin(ReadOnlyModelAdmin): "title", "version_num", "created", - "contents", + "raw_contents", ] fields = [ "component", @@ -175,48 +180,46 @@ class ComponentVersionAdmin(ReadOnlyModelAdmin): "version_num", "created", ] - inlines = [ContentInline] + inlines = [RawContentInline] -@admin.register(Content) -class ContentAdmin(ReadOnlyModelAdmin): +@admin.register(RawContent) +class RawContentAdmin(ReadOnlyModelAdmin): list_display = [ "hash_digest", "learning_package", "mime_type", - "format_size", + "size", "created", ] fields = [ "learning_package", "hash_digest", "mime_type", - "format_size", + "size", "created", - "rendered_data", + "text_preview", ] readonly_fields = [ "learning_package", "hash_digest", "mime_type", - "format_size", + "size", "created", - "rendered_data", + "text_preview", ] list_filter = ("mime_type", "learning_package") - search_fields = ("hash_digest", "size") + search_fields = ("hash_digest",) - def format_size(self, content_obj): - return filesizeformat(content_obj.size) - format_size.short_description = "Size" - - def rendered_data(self, content_obj): - return content_preview(content_obj, 10_000_000) + def text_preview(self, raw_content_obj): + if hasattr(raw_content_obj, "text_content"): + return format_text_for_admin_display(raw_content_obj.text_content.text) + return "" def is_displayable_text(mime_type): - # Our usual text files, includiing things like text/markdown, text/html - media_type, media_subtype = mime_type.split('/') + # Our usual text files, including things like text/markdown, text/html + media_type, media_subtype = mime_type.split("/") if media_type == "text": return True @@ -228,7 +231,7 @@ def is_displayable_text(mime_type): # Other application/* types that we know we can display. if media_subtype in ["json", "x-subrip"]: return True - + # Other formats that are really specific types of JSON if media_subtype.endswith("+json"): return True @@ -236,21 +239,40 @@ def is_displayable_text(mime_type): return False -def content_preview(content_obj, size_limit): - if content_obj.size > size_limit: - return f"Too large to preview." +def link_for_cvc(cvc_obj: ComponentVersionRawContent): + return "/media_server/component_asset/{}/{}/{}/{}".format( + cvc_obj.raw_content.learning_package.identifier, + cvc_obj.component_version.component.identifier, + cvc_obj.component_version.version_num, + cvc_obj.identifier, + ) + + +def format_text_for_admin_display(text): + return format_html( + '
\n{}\n
', + text, + ) - # image before text check, since SVGs can be either, but we probably want to - # see the image version in the admin. - if content_obj.mime_type.startswith("image/"): - b64_str = base64.b64encode(content_obj.data).decode("ascii") - encoded_img_src = f"data:{content_obj.mime_type};base64,{b64_str}" - return format_html('', encoded_img_src) - if is_displayable_text(content_obj.mime_type): +def content_preview(cvc_obj: ComponentVersionRawContent): + raw_content_obj = cvc_obj.raw_content + + if raw_content_obj.mime_type.startswith("image/"): return format_html( - '
\n{}\n
', - content_obj.data.decode("utf-8"), + '', + # TODO: configure with settings value: + "/media_server/component_asset/{}/{}/{}/{}".format( + cvc_obj.raw_content.learning_package.identifier, + cvc_obj.component_version.component.identifier, + cvc_obj.component_version.version_num, + cvc_obj.identifier, + ), + ) + + if hasattr(raw_content_obj, "text_content"): + return format_text_for_admin_display( + raw_content_obj.text_content.text, ) return format_html("This content type cannot be displayed.") diff --git a/openedx_learning/core/components/api.py b/openedx_learning/core/components/api.py new file mode 100644 index 00000000..41d92f46 --- /dev/null +++ b/openedx_learning/core/components/api.py @@ -0,0 +1,43 @@ +""" +Public API for querying and manipulating Components. + +This API is still under construction and should not be considered "stable" until +this repo hits a 1.0 release. +""" +from django.db.models import Q +from pathlib import Path + +from .models import ComponentVersionRawContent + + +def get_component_version_content( + learning_package_identifier: str, + component_identifier: str, + version_num: int, + identifier: Path, +) -> ComponentVersionRawContent: + """ + Look up ComponentVersionRawContent by human readable identifiers. + + Notes: + + 1. This function is returning a model, which we generally frown upon. + 2. I'd like to experiment with different lookup methods + (see https://github.com/openedx/openedx-learning/issues/34) + + Can raise a django.core.exceptions.ObjectDoesNotExist error if there is no + matching ComponentVersionRawContent. + """ + return ComponentVersionRawContent.objects.select_related( + "raw_content", + "component_version", + "component_version__component", + "component_version__component__learning_package", + ).get( + Q( + component_version__component__learning_package__identifier=learning_package_identifier + ) + & Q(component_version__component__identifier=component_identifier) + & Q(component_version__version_num=version_num) + & Q(identifier=identifier) + ) diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index 273a54ff..f4a2c284 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,14 +1,14 @@ -# Generated by Django 4.1 on 2023-02-10 18:58 +# Generated by Django 4.1.6 on 2023-04-14 00:12 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): - initial = True dependencies = [ @@ -41,7 +41,14 @@ class Migration(migrations.Migration): ("namespace", models.CharField(max_length=100)), ("type", models.CharField(blank=True, max_length=100)), ("identifier", models.CharField(max_length=255)), - ("created", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "learning_package", models.ForeignKey( @@ -83,7 +90,14 @@ class Migration(migrations.Migration): validators=[django.core.validators.MinValueValidator(1)] ), ), - ("created", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "component", models.ForeignKey( @@ -91,6 +105,14 @@ class Migration(migrations.Migration): to="components.component", ), ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ "verbose_name": "Component Version", @@ -98,7 +120,7 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name="Content", + name="RawContent", fields=[ ( "id", @@ -114,11 +136,18 @@ class Migration(migrations.Migration): ( "size", models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(10000000)] + validators=[django.core.validators.MaxValueValidator(50000000)] ), ), - ("created", models.DateTimeField()), - ("data", models.BinaryField(max_length=10000000)), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ("file", models.FileField(null=True, upload_to="")), ( "learning_package", models.ForeignKey( @@ -127,9 +156,48 @@ class Migration(migrations.Migration): ), ), ], + options={ + "verbose_name": "Raw Content", + "verbose_name_plural": "Raw Contents", + }, + ), + migrations.CreateModel( + name="PublishedComponent", + fields=[ + ( + "component", + models.OneToOneField( + on_delete=django.db.models.deletion.RESTRICT, + primary_key=True, + serialize=False, + to="components.component", + ), + ), + ], + options={ + "verbose_name": "Published Component", + "verbose_name_plural": "Published Components", + }, ), migrations.CreateModel( - name="ComponentVersionContent", + name="TextContent", + fields=[ + ( + "raw_content", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="text_content", + serialize=False, + to="components.rawcontent", + ), + ), + ("text", models.TextField(blank=True, max_length=100000)), + ("length", models.PositiveIntegerField()), + ], + ), + migrations.CreateModel( + name="ComponentVersionRawContent", fields=[ ( "id", @@ -140,7 +208,17 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), ("identifier", models.CharField(max_length=255)), + ("learner_downloadable", models.BooleanField(default=False)), ( "component_version", models.ForeignKey( @@ -149,30 +227,21 @@ class Migration(migrations.Migration): ), ), ( - "content", + "raw_content", models.ForeignKey( on_delete=django.db.models.deletion.RESTRICT, - to="components.content", + to="components.rawcontent", ), ), ], ), migrations.AddField( model_name="componentversion", - name="contents", + name="raw_contents", field=models.ManyToManyField( related_name="component_versions", - through="components.ComponentVersionContent", - to="components.content", - ), - ), - migrations.AddField( - model_name="componentversion", - name="created_by", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, + through="components.ComponentVersionRawContent", + to="components.rawcontent", ), ), migrations.CreateModel( @@ -211,84 +280,66 @@ class Migration(migrations.Migration): ), ], ), - migrations.CreateModel( - name="PublishedComponent", - fields=[ - ( - "component", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="components.component", - ), - ), - ( - "component_publish_log_entry", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentpublishlogentry", - ), - ), - ( - "component_version", - models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentversion", - ), - ), - ], - options={ - "verbose_name": "Published Component", - "verbose_name_plural": "Published Components", - }, - ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "mime_type"], name="content_idx_lp_mime_type", ), ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "-size"], name="content_idx_lp_rsize" ), ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "-created"], name="content_idx_lp_rcreated" ), ), migrations.AddConstraint( - model_name="content", + model_name="rawcontent", constraint=models.UniqueConstraint( fields=("learning_package", "mime_type", "hash_digest"), name="content_uniq_lc_mime_type_hash_digest", ), ), + migrations.AddField( + model_name="publishedcomponent", + name="component_publish_log_entry", + field=models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentpublishlogentry", + ), + ), + migrations.AddField( + model_name="publishedcomponent", + name="component_version", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentversion", + ), + ), migrations.AddIndex( - model_name="componentversioncontent", + model_name="componentversionrawcontent", index=models.Index( - fields=["content", "component_version"], - name="componentversioncontent_c_cv", + fields=["raw_content", "component_version"], name="cvrawcontent_c_cv" ), ), migrations.AddIndex( - model_name="componentversioncontent", + model_name="componentversionrawcontent", index=models.Index( - fields=["component_version", "content"], - name="componentversioncontent_cv_d", + fields=["component_version", "raw_content"], name="cvrawcontent_cv_d" ), ), migrations.AddConstraint( - model_name="componentversioncontent", + model_name="componentversionrawcontent", constraint=models.UniqueConstraint( fields=("component_version", "identifier"), - name="componentversioncontent_uniq_cv_id", + name="cvrawcontent_uniq_cv_id", ), ), migrations.AddIndex( diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index fcd284d3..b1d68339 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -1,5 +1,5 @@ """ -The model hierarchy is Component -> ComponentVersion -> Content. +The model hierarchy is Component -> ComponentVersion -> RawContent. A Component is an entity like a Problem or Video. It has enough information to identify the Component and determine what the handler should be (e.g. XBlock @@ -11,13 +11,13 @@ publish status is tracked in PublishedComponent, with historical publish data in ComponentPublishLogEntry. -Content is a simple model holding unversioned, raw data, along with some simple -metadata like size and MIME type. +RawContent is a simple model holding unversioned, raw data, along with some +simple metadata like size and MIME type. -Multiple pieces of Content may be associated with a ComponentVersion, through -the ComponentVersionContent model. ComponentVersionContent allows to specify a -Component-local identifier. We're using this like a file path by convention, but -it's possible we might want to have special identifiers later. +Multiple pieces of RawContent may be associated with a ComponentVersion, through +the ComponentVersionRawContent model. ComponentVersionRawContent allows to +specify a Component-local identifier. We're using this like a file path by +convention, but it's possible we might want to have special identifiers later. """ from django.db import models from django.conf import settings @@ -34,22 +34,40 @@ class Component(models.Model): """ - This represents any content that has ever existed in a LearningPackage. + This represents any Component that has ever existed in a LearningPackage. + + What is a Component + ------------------- + + A Component is an entity like a Problem or Video. It has enough information + to identify itself and determine what the handler should be (e.g. XBlock + Problem), but little beyond that. A Component will have many ComponentVersions over time, and most metadata is - associated with the ComponentVersion model. Make a foreign key to this model - when you need a stable reference that will exist for as long as the - LearningPackage itself exists. It is possible for an Component to have no - published ComponentVersion, either because it was never published or because - it's been "deleted" (made unavailable). + associated with the ComponentVersion model and the RawContent that + ComponentVersions are associated with. A Component belongs to one and only one LearningPackage. - The UUID should be treated as immutable. The identifier field *is* mutable, - but changing it will affect all ComponentVersions. If you are referencing - this model from within the same process, use a foreign key to the id. If you - are referencing this Component from an external system, use the UUID. Do NOT - use the identifier if you can help it, since this can be changed. + How to use this model + --------------------- + + Make a foreign key to the Component model when you need a stable reference + that will exist for as long as the LearningPackage itself exists. It is + possible for an Component to have no published ComponentVersion, either + because it was never published or because it's been "deleted" (made + unavailable) at some point, but the Component will continue to exist. + + The UUID should be treated as immutable. + + The identifier field *is* mutable, but changing it will affect all + ComponentVersions. + + If you are referencing this model from within the same process, use a + foreign key to the id. If you are referencing this Component from an + external system/service, use the UUID. The identifier is the part that is + most likely to be human-readable, and may be exported/copied, but try not to + rely on it, since this value may change. Note: When we actually implement the ability to change identifiers, we should make a history table and a modified attribute on this model. @@ -82,7 +100,7 @@ class Meta: # a given LearningPackage. Note that this means it is possible to # have two Components that have the exact same identifier. An XBlock # would be modeled as namespace="xblock.v1" with the type as the - # block_type, so the identifier would only be the block_id (the + # block_type, so the identifier would only be the block_id (the # very last part of the UsageKey). models.UniqueConstraint( fields=[ @@ -102,7 +120,6 @@ class Meta: fields=["learning_package", "identifier"], name="component_idx_lp_identifier", ), - # Global Identifier Index: # * Search by identifier across all Components on the site. This # would be a support-oriented tool from Django Admin. @@ -110,7 +127,6 @@ class Meta: fields=["identifier"], name="component_idx_identifier", ), - # LearningPackage (reverse) Created Index: # * Search for most recently *created* Components for a given # LearningPackage, since they're the most likely to be actively @@ -134,11 +150,11 @@ class ComponentVersion(models.Model): A particular version of a Component. This holds the title (because that's versioned information) and the contents - via a M:M relationship with Content via ComponentVersionContent. + via a M:M relationship with RawContent via ComponentVersionRawContent. * Each ComponentVersion belongs to one and only one Component. * ComponentVersions have a version_num that should increment by one with - each new version. + each new version. """ uuid = immutable_uuid_field() @@ -165,16 +181,16 @@ class ComponentVersion(models.Model): # removed. Open edX in general doesn't let you remove users, but we should # try to model it so that this is possible eventually. created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.SET_NULL, - null=True, + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, ) # The contents hold the actual interesting data associated with this # ComponentVersion. - contents = models.ManyToManyField( - "Content", - through="ComponentVersionContent", + raw_contents = models.ManyToManyField( + "RawContent", + through="ComponentVersionRawContent", related_name="component_versions", ) @@ -206,11 +222,12 @@ class Meta: fields=["component", "-created"], name="cv_idx_component_rcreated", ), - # Title Index: # * Search by title. models.Index( - fields=["title",], + fields=[ + "title", + ], name="cv_idx_title", ), ] @@ -268,38 +285,47 @@ class Meta: verbose_name_plural = "Published Components" -class Content(models.Model): +class RawContent(models.Model): """ This is the most basic piece of raw content data, with no version metadata. - Content stores data in an immutable Binary BLOB `data` field. This data is - not auto-normalized in any way, meaning that pieces of content that are - semantically equivalent (e.g. differently spaced/sorted JSON) will result in + RawContent stores data using the "file" field. This data is not + auto-normalized in any way, meaning that pieces of content that are + semantically equivalent (e.g. differently spaced/sorted JSON) may result in new entries. This model is intentionally ignorant of what these things mean, because it expects supplemental data models to build on top of it. - Two Content instances _can_ have the same hash_digest if they are of + Two RawContent instances _can_ have the same hash_digest if they are of different MIME types. For instance, an empty text file and an empty SRT file will both hash the same way, but be considered different entities. - The other fields on Content are for data that is intrinsic to the file data - itself (e.g. the size). Any smart parsing of the contents into more - structured metadata should happen in other models that hang off of Content. + The other fields on RawContent are for data that is intrinsic to the file + data itself (e.g. the size). Any smart parsing of the contents into more + structured metadata should happen in other models that hang off of + RawContent. + + RawContent models are not versioned in any way. The concept of versioning + only exists at a higher level. - Content models are not versioned in any way. The concept of versioning only - exists at a higher level. + RawContent is optimized for cheap storage, not low latency. It stores + content in a FileField. If you need faster text access across multiple rows, + add a TextContent entry that corresponds to the relevant RawContent. - Since this model uses a BinaryField to hold its data, we have to be careful - about scalability issues. For instance, video files should not be stored - here directly. There is a 10 MB limit set for the moment, to accomodate - things like PDF files and images, but the itention is for the vast majority - of rows to be much smaller than that. + If you need to transform this RawContent into more structured data for your + application, create a model with a OneToOneField(primary_key=True) + relationship to RawContent. Just remember that *you should always create the + RawContent entry* first, to ensure content is always exportable, even if + your app goes away in the future. """ - # Cap item size at 10 MB for now. - MAX_SIZE = 10_000_000 + # 50 MB is our current limit, based on the current Open edX Studio file + # upload size limit. + MAX_FILE_SIZE = 50_000_000 learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + + # This hash value may be calculated using create_hash_digest from the + # openedx.lib.fields module. hash_digest = hash_field() # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type @@ -310,16 +336,28 @@ class Content(models.Model): # that becomes necessary. mime_type = models.CharField(max_length=255, blank=False, null=False) + # This is the size of the raw data file in bytes. This can be different than + # the character length, since UTF-8 encoding can use anywhere between 1-4 + # bytes to represent any given character. size = models.PositiveBigIntegerField( - validators=[MaxValueValidator(MAX_SIZE)], + validators=[MaxValueValidator(MAX_FILE_SIZE)], ) - # This should be manually set so that multiple Content rows being set in the - # same transaction are created with the same timestamp. The timestamp should - # be UTC. + # This should be manually set so that multiple RawContent rows being set in + # the same transaction are created with the same timestamp. The timestamp + # should be UTC. created = manual_date_time_field() - data = models.BinaryField(null=False, max_length=MAX_SIZE) + # All content for the LearningPackage should be stored in files. See model + # docstring for more details on how to store this data in supplementary data + # models that offer better latency guarantees. + file = models.FileField( + null=True, + storage=settings.OPENEDX_LEARNING.get( + "STORAGE", + settings.DEFAULT_FILE_STORAGE, + ), + ) class Meta: constraints = [ @@ -336,7 +374,7 @@ class Meta: ] indexes = [ # LearningPackage MIME type Index: - # * Break down Content counts by type/subtype within a + # * Break down Content counts by type/subtype with in a # LearningPackage. # * Find all the Content in a LearningPackage that matches a # certain MIME type (e.g. "image/png", "application/pdf". @@ -356,31 +394,102 @@ class Meta: models.Index( fields=["learning_package", "-created"], name="content_idx_lp_rcreated", - ) + ), ] + verbose_name = "Raw Content" + verbose_name_plural = "Raw Contents" + + +class TextContent(models.Model): + """ + TextContent supplements RawContent to give an in-table text copy. + + This model exists so that we can have lower-latency access to this data, + particularly if we're pulling back multiple rows at once. + Apps are encouraged to create their own data models that further extend this + one with a more intelligent, parsed data model. For example, individual + XBlocks might parse the OLX in this model into separate data models for + VideoBlock, ProblemBlock, etc. -class ComponentVersionContent(models.Model): + The reason this is built directly into the Learning Core data model is + because we want to be able to easily access and browse this data even if the + app-extended models get deleted (e.g. if they are deprecated and removed). """ - Determines the Content for a given ComponentVersion. + + # 100K is our limit for text data, like OLX. This means 100K *characters*, + # not bytes. Since UTF-8 encodes characters using as many as 4 bytes, this + # couled be as much as 400K of data if we had nothing but emojis. + MAX_TEXT_LENGTH = 100_000 + + raw_content = models.OneToOneField( + RawContent, + on_delete=models.CASCADE, + primary_key=True, + related_name="text_content", + ) + text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + length = models.PositiveIntegerField(null=False) + + +class ComponentVersionRawContent(models.Model): + """ + Determines the RawContent for a given ComponentVersion. An ComponentVersion may be associated with multiple pieces of binary data. For instance, a Video ComponentVersion might be associated with multiple transcripts in different languages. - When Content is associated with an ComponentVersion, it has some local + When RawContent is associated with an ComponentVersion, it has some local identifier that is unique within the the context of that ComponentVersion. This allows the ComponentVersion to do things like store an image file and reference it by a "path" identifier. - Content is immutable and sharable across multiple ComponentVersions and even - across LearningPackages. + RawContent is immutable and sharable across multiple ComponentVersions and + even across LearningPackages. """ + raw_content = models.ForeignKey(RawContent, on_delete=models.RESTRICT) component_version = models.ForeignKey(ComponentVersion, on_delete=models.CASCADE) - content = models.ForeignKey(Content, on_delete=models.RESTRICT) + + uuid = immutable_uuid_field() identifier = identifier_field() + # Is this RawContent downloadable during the learning experience? This is + # NOT about public vs. private permissions on course assets, as that will be + # a policy that can be changed independently of new versions of the content. + # For instance, a course team could decide to flip their course assets from + # private to public for CDN caching reasons, and that should not require + # new ComponentVersions to be created. + # + # What the ``learner_downloadable`` field refers to is whether this asset is + # supposed to *ever* be directly downloadable by browsers during the + # learning experience. This will be True for things like images, PDFs, and + # video transcript files. This field will be False for things like: + # + # * Problem Block OLX will contain the answers to the problem. The XBlock + # runtime and ProblemBlock will use this information to generate HTML and + # grade responses, but the the user's browser is never permitted to + # actually download the raw OLX itself. + # * Many courses include a python_lib.zip file holding custom Python code + # to be used by codejail to assess student answers. This code will also + # potentially reveal answers, and is never intended to be downloadable by + # the student's browser. + # * Some course teams will upload other file formats that their OLX is + # derived from (e.g. specially formatted LaTeX files). These files will + # likewise contain answers and should never be downloadable by the + # student. + # * Other custom metadata may be attached as files in the import, such as + # custom identifiers, author information, etc. + # + # Even if ``learner_downloadble`` is True, the LMS may decide that this + # particular student isn't allowed to see this particular piece of content + # yet–e.g. because they are not enrolled, or because the exam this Component + # is a part of hasn't started yet. That's a matter of LMS permissions and + # policy that is not intrinsic to the content itself, and exists at a layer + # above this. + learner_downloadable = models.BooleanField(default=False) + class Meta: constraints = [ # Uniqueness is only by ComponentVersion and identifier. If for some @@ -388,16 +497,16 @@ class Meta: # content with two different identifiers, that is permitted. models.UniqueConstraint( fields=["component_version", "identifier"], - name="componentversioncontent_uniq_cv_id", + name="cvrawcontent_uniq_cv_id", ), ] indexes = [ models.Index( - fields=["content", "component_version"], - name="componentversioncontent_c_cv", + fields=["raw_content", "component_version"], + name="cvrawcontent_c_cv", ), models.Index( - fields=["component_version", "content"], - name="componentversioncontent_cv_d", + fields=["component_version", "raw_content"], + name="cvrawcontent_cv_d", ), ] diff --git a/openedx_learning/core/components/readme.rst b/openedx_learning/core/components/readme.rst index e2752f63..70cf5a5f 100644 --- a/openedx_learning/core/components/readme.rst +++ b/openedx_learning/core/components/readme.rst @@ -18,5 +18,6 @@ Architecture Guidelines * We're keeping nearly unlimited history, so per-version metadata (i.e. the space/time cost of making a new version) must be kept low. * Do not assume that all Components will be XBlocks. -* Encourage other apps to make models that join to (and add their own metadata to) Component, ComponentVersion, Content, etc. But it should be done in such a way that this app is not aware of them. +* Encourage other apps to make models that join to (and add their own metadata to) Component, ComponentVersion, RawContent, TextContent etc. But it should be done in such a way that this app is not aware of them. * Always preserve the most raw version of the data possible, e.g. OLX, even if XBlocks then extend that with more sophisticated data models. At some point those XBlocks will get deprecated/removed, and we will still want to be able to export the raw data. +* Exports should be fast and *not* require the invocation of plugin code. \ No newline at end of file diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 16f01183..350a1849 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,13 +1,13 @@ -# Generated by Django 4.1 on 2023-02-10 18:56 +# Generated by Django 4.1.6 on 2023-04-14 00:12 from django.conf import settings from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): - initial = True dependencies = [ @@ -38,8 +38,22 @@ class Migration(migrations.Migration): ), ("identifier", models.CharField(max_length=255)), ("title", models.CharField(max_length=1000)), - ("created", models.DateTimeField()), - ("updated", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ( + "updated", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ], options={ "verbose_name": "Learning Package", @@ -68,7 +82,14 @@ class Migration(migrations.Migration): ), ), ("message", models.CharField(blank=True, default="", max_length=1000)), - ("published_at", models.DateTimeField()), + ( + "published_at", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "learning_package", models.ForeignKey( diff --git a/openedx_learning/lib/admin_utils.py b/openedx_learning/lib/admin_utils.py new file mode 100644 index 00000000..f7509726 --- /dev/null +++ b/openedx_learning/lib/admin_utils.py @@ -0,0 +1,28 @@ +""" +Convenience utilities for the Django Admin. +""" +from django.contrib import admin + + +class ReadOnlyModelAdmin(admin.ModelAdmin): + """ + ModelAdmin subclass that removes any editing ability. + + The Django Admin is really useful for quickly examining model data. At the + same time, model creation and updates follow specific rules that are meant + to be enforced above the model layer (in api.py files), so making edits in + the Django Admin is potentially dangerous. + + In general, if you're providing Django Admin interfaces for your + openedx-learning related app data models, you should subclass this class + instead of subclassing admin.ModelAdmin directly. + """ + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index b18137c0..4955ad11 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -6,6 +6,11 @@ * Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html +* The UUID fields are intended to be globally unique identifiers that other + services can store and rely on staying the same. +* The "identifier" fields can be more human-friendly strings, but these may only + be unique within a given context. These values should be treated as mutable, + even if they rarely change in practice. TODO: * Try making a CaseSensitiveCharField and CaseInsensitiveCharField @@ -22,6 +27,8 @@ from django.db import models +from .validators import validate_utc_datetime + def identifier_field(): """ @@ -86,6 +93,9 @@ def manual_date_time_field(): """ DateTimeField that does not auto-generate values. + The datetimes entered for this field *must be UTC* or it will raise a + ValidationError. + The reason for this convention is that we are often creating many rows of data in the same transaction. They are semantically being created or modified "at the same time", even if each individual row is milliseconds @@ -103,4 +113,7 @@ def manual_date_time_field(): auto_now=False, auto_now_add=False, null=False, + validators=[ + validate_utc_datetime, + ], ) diff --git a/openedx_learning/lib/validators.py b/openedx_learning/lib/validators.py new file mode 100644 index 00000000..035a3554 --- /dev/null +++ b/openedx_learning/lib/validators.py @@ -0,0 +1,12 @@ +from datetime import datetime, timezone + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + + +def validate_utc_datetime(dt: datetime): + if dt.tzinfo != timezone.utc: + raise ValidationError( + _("The timezone for %(datetime)s is not UTC."), + params={"datetime": dt}, + ) diff --git a/openedx_learning/rest_api/urls.py b/openedx_learning/rest_api/urls.py index f6e1c2e8..edc533ff 100644 --- a/openedx_learning/rest_api/urls.py +++ b/openedx_learning/rest_api/urls.py @@ -1,4 +1,3 @@ -from django.contrib import admin from django.urls import include, path urlpatterns = [path("v1/", include("openedx_learning.rest_api.v1.urls"))] diff --git a/projects/dev.py b/projects/dev.py index 13cb86c6..13c351d0 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -4,91 +4,94 @@ from pathlib import Path # Build paths inside the project like this: BASE_DIR / {dir_name} / -BASE_DIR = Path(__file__).resolve().parents[2] +BASE_DIR = Path(__file__).resolve().parents[1] DEBUG = True DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'dev.db', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "dev.db", + "USER": "", + "PASSWORD": "", + "HOST": "", + "PORT": "", } } INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.messages', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.sessions", + "django.contrib.staticfiles", # Admin - 'django.contrib.admin', - 'django.contrib.admindocs', - + "django.contrib.admin", + "django.contrib.admindocs", # Learning Core Apps - 'openedx_learning.core.components.apps.ComponentsConfig', - 'openedx_learning.core.publishing.apps.PublishingConfig', - + "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.publishing.apps.PublishingConfig", # Learning Contrib Apps - + "openedx_learning.contrib.media_server.apps.MediaServerConfig", # Apps that don't belong in this repo in the long term, but are here to make # testing/iteration easier until the APIs stabilize. - 'olx_importer.apps.OLXImporterConfig', - + "olx_importer.apps.OLXImporterConfig", # REST API - 'rest_framework', - 'openedx_learning.rest_api.apps.RESTAPIConfig', + "rest_framework", + "openedx_learning.rest_api.apps.RESTAPIConfig", ) MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", # Admin-specific - 'django.contrib.admindocs.middleware.XViewMiddleware', + "django.contrib.admindocs.middleware.XViewMiddleware", ] TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ] - } + }, }, ] LOCALE_PATHS = [ - BASE_DIR / 'openedx_learning' / 'conf' / 'locale', + BASE_DIR / "conf" / "locale", ] -ROOT_URLCONF = 'projects.urls' +ROOT_URLCONF = "projects.urls" -SECRET_KEY = 'insecure-secret-key' +SECRET_KEY = "insecure-secret-key" -STATIC_URL = '/static/' +STATIC_URL = "/static/" STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] STATICFILES_DIRS = [ -# BASE_DIR / 'projects' / 'static' + # BASE_DIR / 'projects' / 'static' ] -MEDIA_URL = '/media/' +MEDIA_URL = "/media/" +MEDIA_ROOT = BASE_DIR / "media" USE_TZ = True + +# openedx-learning required configuration +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +} diff --git a/projects/urls.py b/projects/urls.py index 92928a09..67364cfc 100644 --- a/projects/urls.py +++ b/projects/urls.py @@ -2,8 +2,8 @@ from django.urls import include, path urlpatterns = [ - path('admin/doc/', include('django.contrib.admindocs.urls')), - path('admin/', admin.site.urls), - - path('rest_api/', include('openedx_learning.rest_api.urls')) + path("admin/doc/", include("django.contrib.admindocs.urls")), + path("admin/", admin.site.urls), + path("media_server/", include("openedx_learning.contrib.media_server.urls")), + path("rest_api/", include("openedx_learning.rest_api.urls")), ] diff --git a/requirements/base.in b/requirements/base.in index c0122e9d..f5ebd940 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,15 +1,6 @@ # Core requirements for using this application -c constraints.txt -Django # Web application framework +Django<4.0 # Web application framework -# For the Python API layer (eventually) -attrs - -# Serialization -pyyaml - -# Django Rest Framework + extras for the openedx_lor project -djangorestframework -markdown -django-filter +djangorestframework<4.0 # REST API diff --git a/requirements/base.txt b/requirements/base.txt index ff9722db..7c80551d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,33 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via django -attrs==22.1.0 - # via -r requirements/base.in -backports-zoneinfo==0.2.1 - # via django -django==4.1 +django==3.2.18 # via # -r requirements/base.in - # django-filter # djangorestframework -django-filter==22.1 - # via -r requirements/base.in -djangorestframework==3.13.1 - # via -r requirements/base.in -importlib-metadata==4.12.0 - # via markdown -markdown==3.4.1 +djangorestframework==3.14.0 # via -r requirements/base.in -pytz==2022.2.1 - # via djangorestframework -pyyaml==6.0 - # via -r requirements/base.in -sqlparse==0.4.2 +pytz==2023.3 + # via + # django + # djangorestframework +sqlparse==0.4.3 # via django -zipp==3.8.1 - # via importlib-metadata diff --git a/requirements/ci.in b/requirements/ci.in index ebc12352..8776af10 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -1,6 +1,6 @@ # Requirements for running tests on CI -c constraints.txt -codecov # Code coverage reporting +coverage # Code coverage reporting tox # Virtualenv management for tests import-linter # Track our internal dependencies diff --git a/requirements/ci.txt b/requirements/ci.txt index 912932df..e6ca7b96 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,54 +1,51 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -certifi==2022.6.15 - # via requests -charset-normalizer==2.1.1 - # via requests +cachetools==5.3.0 + # via tox +chardet==5.1.0 + # via tox click==8.1.3 # via import-linter -codecov==2.1.12 +colorama==0.4.6 + # via tox +coverage==7.2.3 # via -r requirements/ci.in -coverage==6.4.4 - # via codecov distlib==0.3.6 # via virtualenv -filelock==3.8.0 +filelock==3.11.0 # via # tox # virtualenv -grimp==1.3 +grimp==2.3 # via import-linter -idna==3.3 - # via requests -import-linter==1.3.0 +import-linter==1.8.0 # via -r requirements/ci.in -networkx==2.8.6 - # via grimp -packaging==21.3 - # via tox -platformdirs==2.5.2 - # via virtualenv +packaging==23.1 + # via + # pyproject-api + # tox +platformdirs==3.2.0 + # via + # tox + # virtualenv pluggy==1.0.0 # via tox -py==1.11.0 - # via tox -pyparsing==3.0.9 - # via packaging -requests==2.28.1 - # via codecov -six==1.16.0 - # via tox -toml==0.10.2 +pyproject-api==1.5.1 # via tox -tox==3.25.1 +tomli==2.0.1 + # via + # import-linter + # pyproject-api + # tox +tox==4.4.12 # via -r requirements/ci.in -typing-extensions==4.3.0 - # via import-linter -urllib3==1.26.12 - # via requests -virtualenv==20.16.4 +typing-extensions==4.5.0 + # via + # grimp + # import-linter +virtualenv==20.21.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 71421be7..2fbb634c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,44 +1,41 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/quality.txt # django -astroid==2.12.5 +astroid==2.15.2 # via # -r requirements/quality.txt # pylint # pylint-celery -attrs==22.1.0 - # via - # -r requirements/quality.txt - # pytest -backports-zoneinfo==0.2.1 - # via - # -r requirements/quality.txt - # django -bleach==5.0.1 +bleach==6.0.0 # via # -r requirements/quality.txt # readme-renderer -build==0.8.0 +build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2022.6.15 +cachetools==5.3.0 # via # -r requirements/ci.txt + # tox +certifi==2022.12.7 + # via # -r requirements/quality.txt # requests -chardet==5.0.0 - # via diff-cover -charset-normalizer==2.1.1 +chardet==5.1.0 # via # -r requirements/ci.txt + # diff-cover + # tox +charset-normalizer==3.1.0 + # via # -r requirements/quality.txt # requests click==8.1.3 @@ -59,21 +56,18 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -codecov==2.1.12 - # via -r requirements/ci.txt -commonmark==0.9.1 +colorama==0.4.6 # via - # -r requirements/quality.txt - # rich -coverage[toml]==6.4.4 + # -r requirements/ci.txt + # tox +coverage[toml]==7.2.3 # via # -r requirements/ci.txt # -r requirements/quality.txt - # codecov # pytest-cov -diff-cover==6.5.1 +diff-cover==7.5.0 # via -r requirements/dev.in -dill==0.3.5.1 +dill==0.3.6 # via # -r requirements/quality.txt # pylint @@ -81,55 +75,58 @@ distlib==0.3.6 # via # -r requirements/ci.txt # virtualenv -django==4.1 +django==3.2.18 # via # -r requirements/quality.txt - # django-filter # djangorestframework # edx-i18n-tools -django-filter==22.1 - # via -r requirements/quality.txt -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via -r requirements/quality.txt docutils==0.19 # via # -r requirements/quality.txt # readme-renderer -edx-i18n-tools==0.9.1 +edx-i18n-tools==0.9.2 # via -r requirements/dev.in -edx-lint==5.2.4 +edx-lint==5.3.4 # via -r requirements/quality.txt -filelock==3.8.0 +exceptiongroup==1.1.1 + # via + # -r requirements/quality.txt + # pytest +filelock==3.11.0 # via # -r requirements/ci.txt # tox # virtualenv -grimp==1.3 +grimp==2.3 # via # -r requirements/ci.txt # import-linter -idna==3.3 +idna==3.4 # via - # -r requirements/ci.txt # -r requirements/quality.txt # requests -import-linter==1.3.0 +import-linter==1.8.0 # via -r requirements/ci.txt -importlib-metadata==4.12.0 +importlib-metadata==6.3.0 # via # -r requirements/quality.txt # keyring - # markdown # twine -iniconfig==1.1.1 +importlib-resources==5.12.0 + # via + # -r requirements/quality.txt + # keyring +iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.10.1 +isort==5.12.0 # via # -r requirements/quality.txt # pylint -jaraco-classes==3.2.2 +jaraco-classes==3.2.3 # via # -r requirements/quality.txt # keyring @@ -138,17 +135,19 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.9.0 +keyring==23.13.1 # via # -r requirements/quality.txt # twine -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markdown==3.4.1 - # via -r requirements/quality.txt -markupsafe==2.1.1 +markdown-it-py==2.2.0 + # via + # -r requirements/quality.txt + # rich +markupsafe==2.1.2 # via # -r requirements/quality.txt # jinja2 @@ -156,43 +155,41 @@ mccabe==0.7.0 # via # -r requirements/quality.txt # pylint -more-itertools==8.14.0 +mdurl==0.1.2 # via # -r requirements/quality.txt - # jaraco-classes -networkx==2.8.6 + # markdown-it-py +more-itertools==9.1.0 # via - # -r requirements/ci.txt - # grimp -packaging==21.3 + # -r requirements/quality.txt + # jaraco-classes +packaging==23.1 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # build + # pyproject-api # pytest # tox -path==16.4.0 +path==16.6.0 # via edx-i18n-tools -pbr==5.10.0 +pbr==5.11.1 # via # -r requirements/quality.txt # stevedore -pep517==0.13.0 - # via - # -r requirements/pip-tools.txt - # build -pip-tools==6.8.0 +pip-tools==6.13.0 # via -r requirements/pip-tools.txt -pkginfo==1.8.3 +pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==2.5.2 +platformdirs==3.2.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint + # tox # virtualenv pluggy==1.0.0 # via @@ -201,25 +198,19 @@ pluggy==1.0.0 # diff-cover # pytest # tox -polib==1.1.1 +polib==1.2.0 # via edx-i18n-tools -py==1.11.0 - # via - # -r requirements/ci.txt - # -r requirements/quality.txt - # pytest - # tox -pycodestyle==2.9.1 +pycodestyle==2.10.0 # via -r requirements/quality.txt -pydocstyle==6.1.1 +pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.13.0 +pygments==2.15.0 # via # -r requirements/quality.txt # diff-cover # readme-renderer # rich -pylint==2.15.0 +pylint==2.17.2 # via # -r requirements/quality.txt # edx-lint @@ -239,46 +230,47 @@ pylint-plugin-utils==0.7 # -r requirements/quality.txt # pylint-celery # pylint-django -pyparsing==3.0.9 +pyproject-api==1.5.1 # via # -r requirements/ci.txt + # tox +pyproject-hooks==1.0.0 + # via # -r requirements/pip-tools.txt - # -r requirements/quality.txt - # packaging -pytest==7.1.3 + # build +pytest==7.3.0 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/quality.txt # code-annotations -pytz==2022.2.1 +pytz==2023.3 # via # -r requirements/quality.txt + # django # djangorestframework pyyaml==6.0 # via # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.0 +readme-renderer==37.3 # via # -r requirements/quality.txt # twine -requests==2.28.1 +requests==2.28.2 # via - # -r requirements/ci.txt # -r requirements/quality.txt - # codecov # requests-toolbelt # twine -requests-toolbelt==0.9.1 +requests-toolbelt==0.10.1 # via # -r requirements/quality.txt # twine @@ -286,26 +278,24 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==12.5.1 +rich==13.3.4 # via # -r requirements/quality.txt # twine six==1.16.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # bleach # edx-lint - # tox snowballstemmer==2.2.0 # via # -r requirements/quality.txt # pydocstyle -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/quality.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/quality.txt # code-annotations @@ -313,46 +303,46 @@ text-unidecode==1.3 # via # -r requirements/quality.txt # python-slugify -toml==0.10.2 - # via - # -r requirements/ci.txt - # tox tomli==2.0.1 # via + # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # build # coverage - # pep517 + # import-linter # pylint + # pyproject-api + # pyproject-hooks # pytest -tomlkit==0.11.4 + # tox +tomlkit==0.11.7 # via # -r requirements/quality.txt # pylint -tox==3.25.1 +tox==4.4.12 # via # -r requirements/ci.txt # tox-battery tox-battery==0.6.1 # via -r requirements/dev.in -twine==4.0.1 +twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.3.0 +typing-extensions==4.5.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # astroid + # grimp # import-linter # pylint # rich -urllib3==1.26.12 +urllib3==1.26.15 # via - # -r requirements/ci.txt # -r requirements/quality.txt # requests # twine -virtualenv==20.16.4 +virtualenv==20.21.0 # via # -r requirements/ci.txt # tox @@ -360,18 +350,19 @@ webencodings==0.5.1 # via # -r requirements/quality.txt # bleach -wheel==0.37.1 +wheel==0.40.0 # via # -r requirements/pip-tools.txt # pip-tools -wrapt==1.14.1 +wrapt==1.15.0 # via # -r requirements/quality.txt # astroid -zipp==3.8.1 +zipp==3.15.0 # via # -r requirements/quality.txt # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index 4418ed34..bd988df3 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.12 # via sphinx -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/test.txt # django @@ -28,9 +28,9 @@ beautifulsoup4==4.12.2 # via pydata-sphinx-theme bleach==5.0.1 # via readme-renderer -certifi==2022.6.15 +certifi==2022.12.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via @@ -38,20 +38,18 @@ click==8.1.3 # code-annotations code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==6.4.4 +coverage[toml]==7.2.3 # via # -r requirements/test.txt # pytest-cov -django==4.1 +django==3.2.18 # via # -r requirements/test.txt - # django-filter # djangorestframework -django-filter==22.1 + # sphinxcontrib-django +djangorestframework==3.14.0 # via -r requirements/test.txt -djangorestframework==3.13.1 - # via -r requirements/test.txt -doc8==1.0.0 +doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 # via @@ -64,12 +62,9 @@ idna==3.3 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==4.12.0 - # via - # -r requirements/test.txt - # markdown - # sphinx -iniconfig==1.1.1 +importlib-metadata==6.3.0 + # via sphinx +iniconfig==2.0.0 # via # -r requirements/test.txt # pytest @@ -78,19 +73,17 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markdown==3.4.1 - # via -r requirements/test.txt -markupsafe==2.1.1 +markupsafe==2.1.2 # via # -r requirements/test.txt # jinja2 -packaging==21.3 +packaging==23.1 # via # -r requirements/test.txt # pydata-sphinx-theme # pytest # sphinx -pbr==5.10.0 +pbr==5.11.1 # via # -r requirements/test.txt # stevedore @@ -111,35 +104,32 @@ pygments==2.13.0 # pydata-sphinx-theme # readme-renderer # sphinx -pyparsing==3.0.9 - # via - # -r requirements/test.txt - # packaging -pytest==7.1.3 +pytest==7.3.0 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2022.2.1 +pytz==2023.3 # via # -r requirements/test.txt # babel + # django # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.0 +readme-renderer==37.3 # via -r requirements/doc.in -requests==2.28.1 +requests==2.28.2 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -160,9 +150,9 @@ sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-django==0.5.1 +sphinxcontrib-django==2.3 # via -r requirements/doc.in -sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx @@ -170,11 +160,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/test.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/test.txt # code-annotations @@ -195,7 +185,5 @@ urllib3==1.26.12 # via requests webencodings==0.5.1 # via bleach -zipp==3.8.1 - # via - # -r requirements/test.txt - # importlib-metadata +zipp==3.15.0 + # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ebb8aa6a..fd0cc1c7 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,26 +1,22 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -build==0.8.0 +build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools -packaging==21.3 +packaging==23.1 # via build -pep517==0.13.0 - # via build -pip-tools==6.8.0 +pip-tools==6.13.0 # via -r requirements/pip-tools.in -pyparsing==3.0.9 - # via packaging +pyproject-hooks==1.0.0 + # via build tomli==2.0.1 - # via - # build - # pep517 -wheel==0.37.1 + # via build +wheel==0.40.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/quality.txt b/requirements/quality.txt index 615387d0..134d429a 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,30 +1,22 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/test.txt # django -astroid==2.12.5 +astroid==2.15.2 # via # pylint # pylint-celery -attrs==22.1.0 - # via - # -r requirements/test.txt - # pytest -backports-zoneinfo==0.2.1 - # via - # -r requirements/test.txt - # django -bleach==5.0.1 +bleach==6.0.0 # via readme-renderer -certifi==2022.6.15 +certifi==2022.12.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via @@ -38,92 +30,89 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -commonmark==0.9.1 - # via rich -coverage[toml]==6.4.4 +coverage[toml]==7.2.3 # via # -r requirements/test.txt # pytest-cov -dill==0.3.5.1 +dill==0.3.6 # via pylint -django==4.1 +django==3.2.18 # via # -r requirements/test.txt - # django-filter # djangorestframework -django-filter==22.1 - # via -r requirements/test.txt -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via -r requirements/test.txt docutils==0.19 # via readme-renderer -edx-lint==5.2.4 +edx-lint==5.3.4 # via -r requirements/quality.in -idna==3.3 - # via requests -importlib-metadata==4.12.0 +exceptiongroup==1.1.1 # via # -r requirements/test.txt + # pytest +idna==3.4 + # via requests +importlib-metadata==6.3.0 + # via # keyring - # markdown # twine -iniconfig==1.1.1 +importlib-resources==5.12.0 + # via keyring +iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.10.1 +isort==5.12.0 # via # -r requirements/quality.in # pylint -jaraco-classes==3.2.2 +jaraco-classes==3.2.3 # via keyring jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.9.0 +keyring==23.13.1 # via twine -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via astroid -markdown==3.4.1 - # via -r requirements/test.txt -markupsafe==2.1.1 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.2 # via # -r requirements/test.txt # jinja2 mccabe==0.7.0 # via pylint -more-itertools==8.14.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes -packaging==21.3 +packaging==23.1 # via # -r requirements/test.txt # pytest -pbr==5.10.0 +pbr==5.11.1 # via # -r requirements/test.txt # stevedore -pkginfo==1.8.3 +pkginfo==1.9.6 # via twine -platformdirs==2.5.2 +platformdirs==3.2.0 # via pylint pluggy==1.0.0 # via # -r requirements/test.txt # pytest -py==1.11.0 - # via - # -r requirements/test.txt - # pytest -pycodestyle==2.9.1 +pycodestyle==2.10.0 # via -r requirements/quality.in -pydocstyle==6.1.1 +pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.13.0 +pygments==2.15.0 # via # readme-renderer # rich -pylint==2.15.0 +pylint==2.17.2 # via # edx-lint # pylint-celery @@ -137,42 +126,39 @@ pylint-plugin-utils==0.7 # via # pylint-celery # pylint-django -pyparsing==3.0.9 - # via - # -r requirements/test.txt - # packaging -pytest==7.1.3 +pytest==7.3.0 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2022.2.1 +pytz==2023.3 # via # -r requirements/test.txt + # django # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.0 +readme-renderer==37.3 # via twine -requests==2.28.1 +requests==2.28.2 # via # requests-toolbelt # twine -requests-toolbelt==0.9.1 +requests-toolbelt==0.10.1 # via twine rfc3986==2.0.0 # via twine -rich==12.5.1 +rich==13.3.4 # via twine six==1.16.0 # via @@ -180,11 +166,11 @@ six==1.16.0 # edx-lint snowballstemmer==2.2.0 # via pydocstyle -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/test.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/test.txt # code-annotations @@ -198,24 +184,24 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.11.4 +tomlkit==0.11.7 # via pylint -twine==4.0.1 +twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.3.0 +typing-extensions==4.5.0 # via # astroid # pylint # rich -urllib3==1.26.12 +urllib3==1.26.15 # via # requests # twine webencodings==0.5.1 # via bleach -wrapt==1.14.1 +wrapt==1.15.0 # via astroid -zipp==3.8.1 +zipp==3.15.0 # via - # -r requirements/test.txt # importlib-metadata + # importlib-resources diff --git a/requirements/test.txt b/requirements/test.txt index a055df20..9410f389 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,18 +1,10 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 - # via - # -r requirements/base.txt - # django -attrs==22.1.0 - # via - # -r requirements/base.txt - # pytest -backports-zoneinfo==0.2.1 +asgiref==3.6.0 # via # -r requirements/base.txt # django @@ -20,61 +12,49 @@ click==8.1.3 # via code-annotations code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==6.4.4 +coverage[toml]==7.2.3 # via pytest-cov # via # -r requirements/base.txt - # django-filter # djangorestframework -django-filter==22.1 - # via -r requirements/base.txt -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via -r requirements/base.txt -importlib-metadata==4.12.0 - # via - # -r requirements/base.txt - # markdown -iniconfig==1.1.1 +exceptiongroup==1.1.1 + # via pytest +iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markdown==3.4.1 - # via -r requirements/base.txt -markupsafe==2.1.1 +markupsafe==2.1.2 # via jinja2 -packaging==21.3 +packaging==23.1 # via pytest -pbr==5.10.0 +pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -py==1.11.0 - # via pytest -pyparsing==3.0.9 - # via packaging -pytest==7.1.3 +pytest==7.3.0 # via # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in -python-slugify==6.1.2 +python-slugify==8.0.1 # via code-annotations -pytz==2022.2.1 +pytz==2023.3 # via # -r requirements/base.txt + # django # djangorestframework pyyaml==6.0 - # via - # -r requirements/base.txt - # code-annotations -sqlparse==0.4.2 + # via code-annotations +sqlparse==0.4.3 # via # -r requirements/base.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via code-annotations text-unidecode==1.3 # via python-slugify @@ -82,7 +62,3 @@ tomli==2.0.1 # via # coverage # pytest -zipp==3.8.1 - # via - # -r requirements/base.txt - # importlib-metadata diff --git a/test_settings.py b/test_settings.py index e41be8f3..d2d0820f 100644 --- a/test_settings.py +++ b/test_settings.py @@ -16,38 +16,43 @@ def root(*args): DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'default.db', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "default.db", + "USER": "", + "PASSWORD": "", + "HOST": "", + "PORT": "", } } INSTALLED_APPS = [ - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.messages', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.sessions", + "django.contrib.staticfiles", # Admin -# 'django.contrib.admin', -# 'django.contrib.admindocs', - + # 'django.contrib.admin', + # 'django.contrib.admindocs', # Our own apps - 'openedx_learning.core.publishing.apps.PublishingConfig', - 'openedx_learning.core.components.apps.ComponentsConfig', + "openedx_learning.core.publishing.apps.PublishingConfig", + "openedx_learning.core.components.apps.ComponentsConfig", ] LOCALE_PATHS = [ - root('openedx_learning', 'conf', 'locale'), + root("openedx_learning", "conf", "locale"), ] -ROOT_URLCONF = 'projects.urls' +ROOT_URLCONF = "projects.urls" -SECRET_KEY = 'insecure-secret-key' +SECRET_KEY = "insecure-secret-key" USE_TZ = True + +# openedx-learning required configuration +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +}