File uploads + Experimental Media Server

.gitignore

@@ -65,3 +65,5 @@ dev.db
 
 .vscode
 
+# Media files (for uploads)
+media/

Makefile

@@ -42,6 +42,7 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy
 	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
 	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
 	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
+	$(PIP_COMPILE) -o requirements/filepony.txt requirements/filepony.in
 	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
 	# Let tox control the Django version for tests
 	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp

filepony/asset_api.py

@@ -0,0 +1,23 @@
+"""
+All Django access should be encapsulated here.
+"""
+from django.db.models import Q
+
+from openedx_learning.core.components.models import ComponentVersionContent
+
+
+def get_content(package_identifier, component_identifier, version_num, asset_path):
+    """
+    """
+    cv = ComponentVersionContent.objects.select_related(
+        "content",
+        "component_version",
+        "component_version__component",
+        "component_version__component__learning_package",
+    ).get(
+        Q(component_version__component__learning_package__identifier=package_identifier)
+        & Q(component_version__component__identifier=component_identifier)
+        & Q(component_version__version_num=version_num)
+        & Q(identifier=asset_path)
+    )
+    return cv.content

filepony/django_init.py

@@ -0,0 +1,8 @@
+import django
+import os
+
+
+def setup():
+    # Django initialization (must precede any Django-related imports):
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "projects.dev")
+    django.setup()

filepony/main.py

@@ -0,0 +1,33 @@
+"""
+
+"""
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from starlette.responses import StreamingResponse
+
+from . import django_init
+from asgiref.sync import sync_to_async
+
+django_init.setup()
+
+from .asset_api import get_content
+
+# FastAPI init:
+app = FastAPI()
+
+
+# Example URL: /components/lp129/finalexam-problem14/1/static/images/finalexam-problem14_fig1.png
+@app.get(
+    "/components/{package_identifier}/{component_identifier}/{version_num}/{asset_path:path}"
+)
+async def component_asset(
+    package_identifier: str,
+    component_identifier: str,
+    version_num: int,
+    asset_path: str,
+) -> StreamingResponse:
+    content = await sync_to_async(get_content, thread_sensitive=True)(
+        package_identifier, component_identifier, version_num, asset_path
+    )
+    path = content.file.path
+    return FileResponse(path, media_type=content.mime_type)

olx_importer/management/commands/load_components.py

@@ -25,6 +25,7 @@
 import re
 import xml.etree.ElementTree as ET
 
+from django.core.files.base import ContentFile
 from django.core.management.base import BaseCommand
 from django.db import transaction
 
@@ -120,16 +121,21 @@ def create_content(self, static_local_path, now, component_version):
 
         hash_digest = create_hash_digest(data_bytes)
 
-        content, _created = Content.objects.get_or_create(
+        content, created = Content.objects.get_or_create(
             learning_package=self.learning_package,
             mime_type=mime_type,
             hash_digest=hash_digest,
             defaults = dict(
-                data=data_bytes,
                 size=len(data_bytes),
                 created=now,
             )
         )
+        if created:
+            content.file.save(
+                f"{content.learning_package.uuid}/{hash_digest}",
+                ContentFile(data_bytes),
+            )
+
         ComponentVersionContent.objects.get_or_create(
             component_version=component_version,
             content=content,

openedx_learning/core/components/admin.py

@@ -1,6 +1,5 @@
-import base64
-
 from django.contrib import admin
+from django.conf import settings
 from django.db.models.aggregates import Count, Sum
 from django.template.defaultfilters import filesizeformat
 from django.urls import reverse
@@ -9,6 +8,7 @@
 from .models import (
     Component,
     ComponentVersion,
+    ComponentVersionContent,
     Content,
     PublishedComponent,
 )
@@ -135,14 +135,32 @@ def title(self, pc):
         return pc.component_version.title
 
 
+def content_path(cvc_obj: ComponentVersionContent):
+    """
+    NOTE: This is probably really inefficient with sql lookups at the moment.
+    """
+    learning_package_identifier = cvc_obj.content.learning_package.identifier
+    component_identifier = cvc_obj.component_version.component.identifier
+    version_num = cvc_obj.component_version.version_num
+    asset_path = cvc_obj.identifier
+
+    return "{}/components/{}/{}/{}/{}".format(
+        settings.FILEPONY['HOST'],
+        learning_package_identifier,
+        component_identifier,
+        version_num,
+        asset_path,
+    )
+
+
 class ContentInline(admin.TabularInline):
     model = ComponentVersion.contents.through
     fields = ["format_identifier", "format_size", "rendered_data"]
     readonly_fields = ["content", "format_identifier", "format_size", "rendered_data"]
     extra = 0
 
-    def rendered_data(self, cv_obj):
-        return content_preview(cv_obj.content, 100_000)
+    def rendered_data(self, cvc_obj):
+        return content_preview(cvc_obj, 100_000)
 
     def format_size(self, cv_obj):
         return filesizeformat(cv_obj.content.size)
@@ -193,6 +211,7 @@ class ContentAdmin(ReadOnlyModelAdmin):
         "mime_type",
         "format_size",
         "created",
+        "file",
         "rendered_data",
     ]
     readonly_fields = [
@@ -201,6 +220,7 @@ class ContentAdmin(ReadOnlyModelAdmin):
         "mime_type",
         "format_size",
         "created",
+        "file",
         "rendered_data",
     ]
     list_filter = ("mime_type", "learning_package")
@@ -236,16 +256,40 @@ def is_displayable_text(mime_type):
     return False
 
 
-def content_preview(content_obj, size_limit):
+def content_preview(cvc_obj, size_limit):
+    content_obj = cvc_obj.content
+
     if content_obj.size > size_limit:
         return f"Too large to preview."
 
     # image before text check, since SVGs can be either, but we probably want to
     # see the image version in the admin.
     if content_obj.mime_type.startswith("image/"):
-        b64_str = base64.b64encode(content_obj.data).decode("ascii")
-        encoded_img_src = f"data:{content_obj.mime_type};base64,{b64_str}"
-        return format_html('<img src="{}" style="max-width: 100%;" />', encoded_img_src)
+        return format_html(
+            '<img src="{}" style="max-width: 100%;" />',
+            content_path(cvc_obj),
+        )
+
+    if is_displayable_text(content_obj.mime_type):
+        return format_html(
+            '<pre style="white-space: pre-wrap;">\n{}\n</pre>',
+            content_obj.data.decode("utf-8"),
+        )
+
+    return format_html("This content type cannot be displayed.")
+
+
+def content_preview_old(content_obj, size_limit):
+    if content_obj.size > size_limit:
+        return f"Too large to preview."
+
+    # image before text check, since SVGs can be either, but we probably want to
+    # see the image version in the admin.
+    #if content_obj.mime_type.startswith("image/"):
+    #    return format_html(
+    #        '<img src="{}" style="max-width: 100%;" />',
+    #        f"{settings.FILEPONY['HOST']}/components/{content_obj.learning_package.identifier}/"
+    #    )
 
     if is_displayable_text(content_obj.mime_type):
         return format_html(

openedx_learning/core/components/migrations/0002_add_content_file.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1 on 2023-02-11 21:18
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("components", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="content",
+            name="file",
+            field=models.FileField(null=True, upload_to=""),
+        ),
+        migrations.AlterField(
+            model_name="content",
+            name="data",
+            field=models.BinaryField(max_length=100000, null=True),
+        ),
+        migrations.AlterField(
+            model_name="content",
+            name="size",
+            field=models.PositiveBigIntegerField(
+                validators=[django.core.validators.MaxValueValidator(100000)]
+            ),
+        ),
+    ]

openedx_learning/core/components/models.py

@@ -297,7 +297,7 @@ class Content(models.Model):
     """
 
     # Cap item size at 10 MB for now.
-    MAX_SIZE = 10_000_000
+    MAX_SIZE = 100_000
 
     learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE)
     hash_digest = hash_field()
@@ -319,7 +319,8 @@ class Content(models.Model):
     # be UTC.
     created = manual_date_time_field()
 
-    data = models.BinaryField(null=False, max_length=MAX_SIZE)
+    data = models.BinaryField(null=True, max_length=MAX_SIZE)
+    file = models.FileField(null=True)
 
     class Meta:
         constraints = [

projects/dev.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / {dir_name} /
-BASE_DIR = Path(__file__).resolve().parents[2]
+BASE_DIR = Path(__file__).resolve().parents[1]
 
 
 DEBUG = True
@@ -74,7 +74,7 @@
 ]
 
 LOCALE_PATHS = [
-    BASE_DIR / 'openedx_learning' / 'conf' / 'locale',
+    BASE_DIR / 'conf' / 'locale',
 ]
 
 ROOT_URLCONF = 'projects.urls'
@@ -90,5 +90,10 @@
 #     BASE_DIR / 'projects' / 'static'
 ]
 MEDIA_URL = '/media/'
+MEDIA_ROOT = BASE_DIR / 'media'
 
 USE_TZ = True
+
+FILEPONY = {
+    'HOST': 'http://localhost:8001',
+}

requirements/base.txt

@@ -1,33 +1,33 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-asgiref==3.5.2
+asgiref==3.6.0
     # via django
-attrs==22.1.0
+attrs==22.2.0
     # via -r requirements/base.in
 backports-zoneinfo==0.2.1
     # via django
-django==4.1
+django==4.1.6
     # via
     #   -r requirements/base.in
     #   django-filter
     #   djangorestframework
 django-filter==22.1
     # via -r requirements/base.in
-djangorestframework==3.13.1
+djangorestframework==3.14.0
     # via -r requirements/base.in
-importlib-metadata==4.12.0
+importlib-metadata==6.0.0
     # via markdown
 markdown==3.4.1
     # via -r requirements/base.in
-pytz==2022.2.1
+pytz==2022.7.1
     # via djangorestframework
 pyyaml==6.0
     # via -r requirements/base.in
-sqlparse==0.4.2
+sqlparse==0.4.3
     # via django
-zipp==3.8.1
+zipp==3.13.0
     # via importlib-metadata