Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions cms/djangoapps/contentstore/tests/test_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from django.conf import settings
from django.test.client import Client
from django.test.utils import override_settings
from django.core.files.storage import storages

from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
Expand Down Expand Up @@ -281,19 +282,24 @@ def test_video_components_present_while_import(self):

@override_settings(
COURSE_IMPORT_EXPORT_STORAGE="cms.djangoapps.contentstore.storage.ImportExportS3Storage",
DEFAULT_FILE_STORAGE="django.core.files.storage.FileSystemStorage"
STORAGES={
'default': {
'BACKEND': "django.core.files.storage.FileSystemStorage"
}
}
)
def test_resolve_default_storage(self):
def test_default_storage(self):
""" Ensure the default storage is invoked, even if course export storage is configured """
storage = resolve_storage_backend(
storage_key="default",
legacy_setting_key="DEFAULT_FILE_STORAGE"
)
storage = storages["default"]
self.assertEqual(storage.__class__.__name__, "FileSystemStorage")

@override_settings(
COURSE_IMPORT_EXPORT_STORAGE="cms.djangoapps.contentstore.storage.ImportExportS3Storage",
DEFAULT_FILE_STORAGE="django.core.files.storage.FileSystemStorage",
STORAGES={
'default': {
'BACKEND': "django.core.files.storage.FileSystemStorage"
}
},
COURSE_IMPORT_EXPORT_BUCKET="bucket_name_test"
)
def test_resolve_happy_path_storage(self):
Expand Down
15 changes: 12 additions & 3 deletions cms/djangoapps/export_course_metadata/test_signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory
from common.djangoapps.util.storage import resolve_storage_backend
from storages.backends.s3boto3 import S3Boto3Storage
from django.core.files.storage import storages

from .signals import export_course_metadata
from .toggles import EXPORT_COURSE_METADATA_FLAG
Expand Down Expand Up @@ -60,16 +61,24 @@ def test_happy_path(self, patched_content, patched_storage):

@override_settings(
COURSE_METADATA_EXPORT_STORAGE="cms.djangoapps.export_course_metadata.storage.CourseMetadataExportS3Storage",
DEFAULT_FILE_STORAGE="django.core.files.storage.FileSystemStorage"
STORAGES={
'default': {
'BACKEND': "django.core.files.storage.FileSystemStorage"
}
}
)
def test_resolve_default_storage(self):
""" Ensure the default storage is invoked, even if course export storage is configured """
storage = resolve_storage_backend(storage_key="default", legacy_setting_key="default")
storage = storages["default"]
self.assertEqual(storage.__class__.__name__, "FileSystemStorage")

@override_settings(
COURSE_METADATA_EXPORT_STORAGE="cms.djangoapps.export_course_metadata.storage.CourseMetadataExportS3Storage",
DEFAULT_FILE_STORAGE="django.core.files.storage.FileSystemStorage",
STORAGES={
"default": {
"BACKEND": "django.core.files.storage.FileSystemStorage"
}
},
COURSE_METADATA_EXPORT_BUCKET="bucket_name_test"
)
def test_resolve_happy_path_storage(self):
Expand Down
11 changes: 9 additions & 2 deletions cms/envs/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1157,7 +1157,6 @@
'YUI_BINARY': 'yui-compressor',
}

STATICFILES_STORAGE = 'openedx.core.storage.ProductionStorage'
STATICFILES_STORAGE_KWARGS = {}

# List of finder classes that know how to find static files in various locations.
Expand Down Expand Up @@ -2386,7 +2385,15 @@
BULK_EMAIL_LOG_SENT_EMAILS = False

############### Settings for django file storage ##################
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
STORAGES = {
'default': {
'BACKEND': 'django.core.files.storage.FileSystemStorage'
},
'staticfiles': {
'BACKEND': 'openedx.core.storage.ProductionStorage'
}
}


###################### Grade Downloads ######################
# These keys are used for all of our asynchronous downloadable files, including
Expand Down
4 changes: 2 additions & 2 deletions cms/envs/devstack.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from .production import * # pylint: disable=wildcard-import, unused-wildcard-import

# Don't use S3 in devstack, fall back to filesystem
del DEFAULT_FILE_STORAGE
STORAGES['default']['BACKEND'] = 'django.core.files.storage.FileSystemStorage'
COURSE_IMPORT_EXPORT_STORAGE = 'django.core.files.storage.FileSystemStorage'
USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE

Expand Down Expand Up @@ -56,7 +56,7 @@

# Skip packaging and optimization in development
PIPELINE['PIPELINE_ENABLED'] = False
STATICFILES_STORAGE = 'openedx.core.storage.DevelopmentStorage'
STORAGES['staticfiles']['BACKEND'] = 'openedx.core.storage.DevelopmentStorage'

# Revert to the default set of finders as we don't want the production pipeline
STATICFILES_FINDERS = [
Expand Down
6 changes: 5 additions & 1 deletion cms/envs/mock.yml
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,11 @@ DATABASES:
USER: user
DATA_DIR: /edx/var/edxapp
DEFAULT_FEEDBACK_EMAIL: feedback@example.com
DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage
STORAGES:
default:
BACKEND: storages.backends.s3boto3.S3Boto3Storage
staticfiles:
BACKEND: openedx.core.storage.ProductionStorage
DEFAULT_FROM_EMAIL: no-reply@registration.localhost
DEFAULT_HASHING_ALGORITHM: sha256
DEFAULT_JWT_ISSUER:
Expand Down
35 changes: 24 additions & 11 deletions cms/envs/production.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,11 @@ def get_env_setting(setting):
'MKTG_URL_LINK_MAP',
'REST_FRAMEWORK',
'EVENT_BUS_PRODUCER_CONFIG',
'DEFAULT_FILE_STORAGE',
'STATICFILES_STORAGE',
]
})


#######################################################################################################################
#### LOAD THE EDX-PLATFORM GIT REVISION
####
Expand Down Expand Up @@ -150,11 +151,6 @@ def get_env_setting(setting):
if 'staticfiles' in CACHES:
CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION

# In order to transition from local disk asset storage to S3 backed asset storage,
# we need to run asset collection twice, once for local disk and once for S3.
# Once we have migrated to service assets off S3, then we can convert this back to
# managed by the yaml file contents
STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', STATICFILES_STORAGE)

MKTG_URL_LINK_MAP.update(_YAML_TOKENS.get('MKTG_URL_LINK_MAP', {}))

Expand Down Expand Up @@ -194,21 +190,38 @@ def get_env_setting(setting):
# The number of seconds that a generated URL is valid for.
AWS_QUERYSTRING_EXPIRE = 7 * 24 * 60 * 60 # 7 days

# Change to S3Boto3 if we haven't specified another default storage AND we have specified AWS creds.
if (not _YAML_TOKENS.get('DEFAULT_FILE_STORAGE')) and AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
_yaml_storages = _YAML_TOKENS.get('STORAGES', {})

_storages_default_backend_is_missing = not _yaml_storages.get('default', {}).get('BACKEND')

# For backward compatibility, if YAML provides legacy keys (DEFAULT_FILE_STORAGE, STATICFILES_STORAGE)
# and STORAGES doesn’t explicitly define the corresponding backend, migrate the legacy value into STORAGES.
# If YAML doesn't provide lagacy keys, no backend is defined in STORAGES['default'] and AWS creds are present,
# fall back to S3Boto3Storage.
#
# This ensures YAML-provided values take precedence over defaults from common.py,
# without overwriting user-defined STORAGES and AWS creds are treated only as a fallback.
if _storages_default_backend_is_missing:
if 'DEFAULT_FILE_STORAGE' in _YAML_TOKENS:
STORAGES['default']['BACKEND'] = _YAML_TOKENS['DEFAULT_FILE_STORAGE']
elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
STORAGES['default']['BACKEND'] = 'storages.backends.s3boto3.S3Boto3Storage'

# Apply legacy STATICFILES_STORAGE if no backend is defined for "staticfiles"
if 'STATICFILES_STORAGE' in _YAML_TOKENS and not _yaml_storages.get('staticfiles', {}).get('BACKEND'):
STORAGES['staticfiles']['BACKEND'] = _YAML_TOKENS['STATICFILES_STORAGE']

if COURSE_IMPORT_EXPORT_BUCKET:
COURSE_IMPORT_EXPORT_STORAGE = 'cms.djangoapps.contentstore.storage.ImportExportS3Storage'
else:
COURSE_IMPORT_EXPORT_STORAGE = DEFAULT_FILE_STORAGE
COURSE_IMPORT_EXPORT_STORAGE = STORAGES['default']['BACKEND']

USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE

if COURSE_METADATA_EXPORT_BUCKET:
COURSE_METADATA_EXPORT_STORAGE = 'cms.djangoapps.export_course_metadata.storage.CourseMetadataExportS3Storage'
else:
COURSE_METADATA_EXPORT_STORAGE = DEFAULT_FILE_STORAGE
COURSE_METADATA_EXPORT_STORAGE = STORAGES['default']['BACKEND']

# The normal database user does not have enough permissions to run migrations.
# Migrations are run with separate credentials, given as DB_MIGRATION_*
Expand Down
3 changes: 1 addition & 2 deletions cms/envs/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
from lms.envs.test import ( # pylint: disable=wrong-import-order, disable=unused-import
ACCOUNT_MICROFRONTEND_URL,
COMPREHENSIVE_THEME_DIRS, # unimport:skip
DEFAULT_FILE_STORAGE,
ECOMMERCE_API_URL,
ENABLE_COMPREHENSIVE_THEMING,
JWT_AUTH,
Expand Down Expand Up @@ -91,7 +90,7 @@
# If we don't add these settings, then Django templates that can't
# find pipelined assets will raise a ValueError.
# http://stackoverflow.com/questions/12816941/unit-testing-with-django-pipeline
STATICFILES_STORAGE = "pipeline.storage.NonPackagingPipelineStorage"
STORAGES['staticfiles']['BACKEND'] = "pipeline.storage.NonPackagingPipelineStorage"
STATIC_URL = "/static/"

# Update module store settings per defaults for tests
Expand Down
2 changes: 1 addition & 1 deletion common/djangoapps/util/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def store_uploaded_file(
file_storage = DefaultStorage()
# If a file already exists with the supplied name, file_storage will make the filename unique.
stored_file_name = file_storage.save(stored_file_name, uploaded_file)
if is_private and settings.DEFAULT_FILE_STORAGE == 'storages.backends.s3boto3.S3Boto3Storage':
if is_private and settings.STORAGES['default']['BACKEND'] == 'storages.backends.s3boto3.S3Boto3Storage':
S3Boto3Storage().connection.meta.client.put_object_acl(
ACL='private',
Bucket=settings.AWS_STORAGE_BUCKET_NAME,
Expand Down
6 changes: 2 additions & 4 deletions common/djangoapps/util/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ def resolve_storage_backend(

storage_path = getattr(settings, legacy_setting_key, None)
storages_config = getattr(settings, 'STORAGES', {})

if options is None:
options = {}
options = options or {}

if storage_key in storages_config:
# Use case 1: STORAGES is defined
Expand Down Expand Up @@ -70,5 +68,5 @@ def resolve_storage_backend(
break
storage_path = storage_path.get(deep_setting_key)

StorageClass = import_string(storage_path or settings.DEFAULT_FILE_STORAGE)
StorageClass = import_string(storage_path or storages_config["default"]["BACKEND"])
return StorageClass(**options)
76 changes: 76 additions & 0 deletions common/djangoapps/util/tests/test_resolve_storage_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from django.test import TestCase
from django.test.utils import override_settings
from unittest.mock import patch, MagicMock

from common.djangoapps.util.storage import resolve_storage_backend

Expand All @@ -20,6 +21,7 @@ class ResolveStorageTest(TestCase):
BLOCK_STRUCTURES_SETTINGS="cms.djangoapps.contentstore.storage.ImportExportS3Storage"
)
def test_legacy_settings(self):
"""Test legacy string-based storage settings."""
storage = resolve_storage_backend(
storage_key="block_structures_settings",
legacy_setting_key="BLOCK_STRUCTURES_SETTINGS",
Expand All @@ -33,6 +35,7 @@ def test_legacy_settings(self):
}
)
def test_nested_legacy_settings(self):
"""Test legacy nested dictionary."""
storage = resolve_storage_backend(
storage_key="block_structures_settings",
legacy_setting_key="BLOCK_STRUCTURES_SETTINGS",
Expand All @@ -47,10 +50,83 @@ def test_nested_legacy_settings(self):
}
)
def test_nested_legacy_settings_failed(self):
"""Test legacy nested dictionary settings with missing key falls back to default."""
storage = resolve_storage_backend(
storage_key="block_structures_settings",
legacy_setting_key="BLOCK_STRUCTURES_SETTINGS",
legacy_sec_setting_keys=["STORAGE_CLASS"],
options={}
)
assert storage.__class__.__name__ == DEFAULT_STORAGE_CLASS_NAME

@override_settings(
STORAGES={
"default": {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {}
}
},
LEGACY_SETTING="cms.djangoapps.contentstore.storage.ImportExportS3Storage"
)
def test_missing_storage_key_fallback_to_legacy(self):
"""Test fallback to legacy settings when storage key not found in STORAGES."""
storage = resolve_storage_backend(
storage_key="nonexistent_storage",
legacy_setting_key="LEGACY_SETTING",
options={}
)
assert storage.__class__.__name__ == "ImportExportS3Storage"

def test_no_storages_no_legacy_setting(self):
"""Test fallback to default storage when neither STORAGES nor legacy setting exists."""
storage = resolve_storage_backend(
storage_key="nonexistent_storage",
legacy_setting_key="NONEXISTENT_LEGACY_SETTING",
options={}
)
assert storage.__class__.__name__ == DEFAULT_STORAGE_CLASS_NAME

@override_settings(
STORAGES={
"default": {
"BACKEND": "cms.djangoapps.contentstore.storage.ImportExportS3Storage",
"OPTIONS": {}
}
}
)
def test_fallback_to_custom_default_backend(self):
"""Test fallback uses custom default backend from STORAGES config."""
storage = resolve_storage_backend(
storage_key="nonexistent_storage",
legacy_setting_key="NONEXISTENT_LEGACY_SETTING",
options={}
)
assert storage.__class__.__name__ == "ImportExportS3Storage"

@override_settings(
STORAGES={
"default": {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {}
},
"custom_storage_key": {
"BACKEND": "cms.djangoapps.contentstore.storage.ImportExportS3Storage",
"OPTIONS": {}
}
}
)
@patch('common.djangoapps.util.storage.storages')
def test_modern_storages_config(self, mock_storages):
"""Test modern Django STORAGES configuration that takes precedence."""
mock_storage_instance = MagicMock()
mock_storage_instance.__class__.__name__ = "ImportExportS3Storage"
mock_storages.__getitem__.return_value = mock_storage_instance

storage = resolve_storage_backend(
storage_key="custom_storage_key",
legacy_setting_key="SOME_LEGACY_SETTING",
options={}
)

mock_storages.__getitem__.assert_called_once_with("custom_storage_key")
assert storage.__class__.__name__ == "ImportExportS3Storage"
4 changes: 2 additions & 2 deletions lms/djangoapps/instructor_task/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,12 +272,12 @@ def __init__(self, storage_class=None, storage_kwargs=None):
@classmethod
def from_config(cls, config_name):
"""
By default, the default file storage specified by the `DEFAULT_FILE_STORAGE`
By default, the default file storage specified by the `STORAGES['default']`
setting will be used. To configure the storage used, add a dict in
settings with the following fields::

STORAGE_CLASS : The import path of the storage class to use. If
not set, the DEFAULT_FILE_STORAGE setting will be used.
not set, the STORAGES['default']['BACKEND'] setting will be used.
STORAGE_KWARGS : An optional dict of kwargs to pass to the storage
constructor. This can be used to specify a
different S3 bucket or root path, for example.
Expand Down
10 changes: 8 additions & 2 deletions lms/envs/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -2182,7 +2182,6 @@
'UGLIFYJS_BINARY': 'node_modules/.bin/uglifyjs',
}

STATICFILES_STORAGE = 'openedx.core.storage.ProductionStorage'
STATICFILES_STORAGE_KWARGS = {}

# List of finder classes that know how to find static files in various locations.
Expand Down Expand Up @@ -4609,7 +4608,14 @@
}

############### Settings for django file storage ##################
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
STORAGES = {
'default': {
'BACKEND': 'django.core.files.storage.FileSystemStorage'
},
'staticfiles': {
'BACKEND': 'openedx.core.storage.ProductionStorage'
}
}

### Proctoring configuration (redirct URLs and keys shared between systems) ####
PROCTORING_BACKENDS = {
Expand Down
Loading
Loading