From a7ed21d104a854a9a4e3273006118161689cae0c Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Tue, 25 Feb 2025 17:20:01 -0500 Subject: [PATCH 1/3] refactor: Clean up CMS production settings cruft --- cms/envs/production.py | 596 ++++++++++++++--------------------------- lms/envs/production.py | 7 +- 2 files changed, 201 insertions(+), 402 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index da5642b53c6c..121383845110 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -1,5 +1,8 @@ """ -This is the default template for our main set of AWS servers. +Override common.py with key-value pairs from YAML (plus some extra defaults & post-processing). + +This file is in the process of being restructured. Please see: +https://github.com/openedx/edx-platform/blob/master/docs/decisions/0022-settings-simplification.rst """ # We intentionally define lots of variables that aren't used, and @@ -8,12 +11,10 @@ import codecs -import copy import os import warnings import yaml -import django from django.core.exceptions import ImproperlyConfigured from django.urls import reverse_lazy from edx_django_utils.plugins import add_plugins @@ -37,7 +38,14 @@ def get_env_setting(setting): error_msg = "Set the %s env variable" % setting raise ImproperlyConfigured(error_msg) # lint-amnesty, pylint: disable=raise-missing-from -############### ALWAYS THE SAME ################################ + +####################################################################################################################### +#### PRODUCTION DEFAULTS +#### +#### Configure some defaults (beyond what has already been configured in common.py) before loading the YAML file. +#### DO NOT ADD NEW DEFAULTS HERE! Put any new setting defaults in common.py instead, along with a setting annotation. +#### TODO: Move all these defaults into common.py. +#### DEBUG = False @@ -50,9 +58,60 @@ def get_env_setting(setting): # https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header # for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -############### END ALWAYS THE SAME ################################ -# A file path to a YAML file from which to load all the configuration for the edx platform +CELERY_RESULT_BACKEND = 'django-cache' +BROKER_HEARTBEAT = 60.0 +BROKER_HEARTBEAT_CHECKRATE = 2 +STATIC_URL_BASE = None +OPENAPI_CACHE_TIMEOUT = 60 * 60 +STATIC_ROOT_BASE = None +CMS_BASE = None +LMS_BASE = None +LMS_ROOT_URL = None +CMS_ROOT_URL = None +AUTHORING_API_URL = '' +CHAT_COMPLETION_API = '' +CHAT_COMPLETION_API_KEY = '' +LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT = '' +LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT = '' +LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT = '' +LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT = '' +IDA_LOGOUT_URI_LIST = [] +SESSION_COOKIE_DOMAIN = None +SESSION_COOKIE_HTTPONLY = True +REGISTRATION_EMAIL_PATTERNS_ALLOWED = None +AWS_STORAGE_BUCKET_NAME = 'edxuploads' +AWS_QUERYSTRING_AUTH = True +AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' +CELERY_ALWAYS_EAGER = False +CELERY_BROKER_TRANSPORT = "" +CELERY_BROKER_HOSTNAME = "" +CELERY_BROKER_USER = "" +CELERY_BROKER_PASSWORD = "" +CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION = True +PARSE_KEYS = {} +VIDEO_CDN_URL = {} +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None + +EMAIL_FILE_PATH = Derived(lambda settings: settings.DATA_DIR / "emails" / "studio") +LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) +ENTERPRISE_API_URL = Derived(lambda settings: settings.LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/') +ENTERPRISE_CONSENT_API_URL = Derived(lambda settings: settings.LMS_INTERNAL_ROOT_URL + '/consent/api/v1/') +POLICY_CHANGE_GRADES_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +SHARED_COOKIE_DOMAIN = Derived(lambda settings: settings.SESSION_COOKIE_DOMAIN) +SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) +VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +UPDATE_SEARCH_INDEX_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +BROKER_USE_SSL = Derived(lambda settings: settings.CELERY_BROKER_USE_SSL) + + +####################################################################################################################### +#### YAML LOADING +#### + +# A file path to a YAML file from which to load configuration overrides for LMS. try: CONFIG_FILE = get_env_setting('CMS_CFG') except ImproperlyConfigured: @@ -64,34 +123,41 @@ def get_env_setting(setting): ) with codecs.open(CONFIG_FILE, encoding='utf-8') as f: - __config__ = yaml.safe_load(f) - - # ENV_TOKENS and AUTH_TOKENS are included for reverse compatability. - # Removing them may break plugins that rely on them. - ENV_TOKENS = __config__ - AUTH_TOKENS = __config__ - - # Add the key/values from config into the global namespace of this module. - # But don't override the FEATURES dict because we do that in an additive way. - __config_copy__ = copy.deepcopy(__config__) - - KEYS_WITH_MERGED_VALUES = [ - 'FEATURES', - 'TRACKING_BACKENDS', - 'EVENT_TRACKING_BACKENDS', - 'JWT_AUTH', - 'CELERY_QUEUES', - 'MKTG_URL_LINK_MAP', - 'MKTG_URL_OVERRIDES', - 'REST_FRAMEWORK', - 'EVENT_BUS_PRODUCER_CONFIG', - ] - for key in KEYS_WITH_MERGED_VALUES: - if key in __config_copy__: - del __config_copy__[key] - - vars().update(__config_copy__) + # _YAML_TOKENS starts out with the exact contents of the CMS_CFG YAML file. + # Please avoid adding new references to _YAML_TOKENS. Such references make our settings logic more complex. + # Instead, just reference the Django settings, which we define in the next step... + _YAML_TOKENS = yaml.safe_load(f) + + # Update the global namespace of this module with the key-value pairs from _YAML_TOKENS. + # In other words: For (almost) every YAML key-value pair, define/update a Django setting with that name and value. + vars().update({ + + # Note: If `value` is a mutable object (e.g., a dict), then it will be aliased between the global namespace and + # _YAML_TOKENS. In other words, updates to `value` will manifest in _YAML_TOKENS as well. This is intentional, + # in order to maintain backwards compatibility with old Django plugins which use _YAML_TOKENS. + key: value + for key, value in _YAML_TOKENS.items() + + # Do NOT define/update Django settings for these particular special keys. + # We handle each of these with its special logic (below, in this same module). + # For example, we need to *update* the default FEATURES dict rather than wholesale *override* it. + if key not in [ + 'FEATURES', + 'TRACKING_BACKENDS', + 'EVENT_TRACKING_BACKENDS', + 'JWT_AUTH', + 'CELERY_QUEUES', + 'MKTG_URL_LINK_MAP', + 'REST_FRAMEWORK', + 'EVENT_BUS_PRODUCER_CONFIG', + ] + }) + + +####################################################################################################################### +#### LOAD THE EDX-PLATFORM GIT REVISION +#### try: # A file path to a YAML file from which to load all the code revisions currently deployed @@ -105,86 +171,36 @@ def get_env_setting(setting): # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -###################################### CELERY ################################ + +####################################################################################################################### +#### POST-PROCESSING OF YAML +#### +#### This is where we do a bunch of logic to post-process the results of the YAML, including: conditionally setting +#### updates, merging dicts+lists which we did not override, and in some cases simply ignoring the YAML value in favor +#### of a specific production value. +#### # Don't use a connection pool, since connections are dropped by ELB. BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 -# Allow env to configure celery result backend with default set to django-cache -CELERY_RESULT_BACKEND = ENV_TOKENS.get('CELERY_RESULT_BACKEND', 'django-cache') - -# When the broker is behind an ELB, use a heartbeat to refresh the -# connection and to detect if it has been dropped. -BROKER_HEARTBEAT = ENV_TOKENS.get('BROKER_HEARTBEAT', 60.0) -BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) - # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" -# STATIC_URL_BASE specifies the base url to use for static files -STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None) if STATIC_URL_BASE: STATIC_URL = STATIC_URL_BASE if not STATIC_URL.endswith("/"): STATIC_URL += "/" STATIC_URL += 'studio/' -DEFAULT_COURSE_VISIBILITY_IN_CATALOG = ENV_TOKENS.get( - 'DEFAULT_COURSE_VISIBILITY_IN_CATALOG', - DEFAULT_COURSE_VISIBILITY_IN_CATALOG -) - -# DEFAULT_MOBILE_AVAILABLE specifies if the course is available for mobile by default -DEFAULT_MOBILE_AVAILABLE = ENV_TOKENS.get( - 'DEFAULT_MOBILE_AVAILABLE', - DEFAULT_MOBILE_AVAILABLE -) - -# How long to cache OpenAPI schemas and UI, in seconds. -OPENAPI_CACHE_TIMEOUT = ENV_TOKENS.get('OPENAPI_CACHE_TIMEOUT', 60 * 60) - -# STATIC_ROOT specifies the directory where static files are -# collected - -STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None) if STATIC_ROOT_BASE: STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio' WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" WEBPACK_LOADER['WORKERS']['STATS_FILE'] = STATIC_ROOT / "webpack-worker-stats.json" -DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) -EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', DATA_DIR / "emails" / "studio") - - -# CMS_BASE: Public domain name of Studio (should be resolvable from the end-user's browser) -CMS_BASE = ENV_TOKENS.get('CMS_BASE') -LMS_BASE = ENV_TOKENS.get('LMS_BASE') -LMS_ROOT_URL = ENV_TOKENS.get('LMS_ROOT_URL') -CMS_ROOT_URL = ENV_TOKENS.get('CMS_ROOT_URL') -LMS_INTERNAL_ROOT_URL = ENV_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) -ENTERPRISE_API_URL = ENV_TOKENS.get('ENTERPRISE_API_URL', LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/') -ENTERPRISE_CONSENT_API_URL = ENV_TOKENS.get('ENTERPRISE_CONSENT_API_URL', LMS_INTERNAL_ROOT_URL + '/consent/api/v1/') -AUTHORING_API_URL = ENV_TOKENS.get('AUTHORING_API_URL', '') -# Note that FEATURES['PREVIEW_LMS_BASE'] gets read in from the environment file. - -CHAT_COMPLETION_API = ENV_TOKENS.get('CHAT_COMPLETION_API', '') -CHAT_COMPLETION_API_KEY = ENV_TOKENS.get('CHAT_COMPLETION_API_KEY', '') -LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT', '') -LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT = ENV_TOKENS.get( - 'LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT', - '' -) -LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT', '') -LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT', '') - -# List of logout URIs for each IDA that the learner should be logged out of when they logout of -# Studio. Only applies to IDA for which the social auth flow uses DOT (Django OAuth Toolkit). -IDA_LOGOUT_URI_LIST = ENV_TOKENS.get('IDA_LOGOUT_URI_LIST', []) - -SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) +DATA_DIR = path(DATA_DIR) ALLOWED_HOSTS = [ # TODO: bbeggs remove this before prod, temp fix to get load testing running @@ -192,9 +208,6 @@ def get_env_setting(setting): CMS_BASE, ] -LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) - -CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -210,157 +223,51 @@ def get_env_setting(setting): # we need to run asset collection twice, once for local disk and once for S3. # Once we have migrated to service assets off S3, then we can convert this back to # managed by the yaml file contents -STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', ENV_TOKENS.get('STATICFILES_STORAGE', STATICFILES_STORAGE)) - -# Load all AWS_ prefixed variables to allow an S3Boto3Storage to be configured -_locals = locals() -for key, value in ENV_TOKENS.items(): - if key.startswith('AWS_'): - _locals[key] = value - -SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') -SESSION_COOKIE_HTTPONLY = ENV_TOKENS.get('SESSION_COOKIE_HTTPONLY', True) - -REGISTRATION_EMAIL_PATTERNS_ALLOWED = ENV_TOKENS.get('REGISTRATION_EMAIL_PATTERNS_ALLOWED') - -# allow for environments to specify what cookie name our login subsystem should use -# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can -# happen with some browsers (e.g. Firefox) -if ENV_TOKENS.get('SESSION_COOKIE_NAME', None): - # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() - SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME')) - -# This is the domain that is used to set shared cookies between various sub-domains. -# By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. -SHARED_COOKIE_DOMAIN = ENV_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) - -# Determines whether the CSRF token can be transported on -# unencrypted channels. It is set to False here for backward compatibility, -# but it is highly recommended that this is True for environments accessed -# by end users. -CSRF_COOKIE_SECURE = ENV_TOKENS.get('CSRF_COOKIE_SECURE', False) - -# values are already updated above with default CSRF_TRUSTED_ORIGINS values but in -# case of new django version these values will override. -if django.VERSION[0] >= 4: # for greater than django 3.2 use schemes. - CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) - -#Email overrides -MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {})) -MKTG_URL_OVERRIDES.update(ENV_TOKENS.get('MKTG_URL_OVERRIDES', MKTG_URL_OVERRIDES)) - -for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): - oldvalue = CODE_JAIL.get(name) - if isinstance(oldvalue, dict): - for subname, subvalue in value.items(): - oldvalue[subname] = subvalue - else: - CODE_JAIL[name] = value - -COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) - -# COMPREHENSIVE_THEME_LOCALE_PATHS contain the paths to themes locale directories e.g. -# "COMPREHENSIVE_THEME_LOCALE_PATHS" : [ -# "/edx/src/edx-themes/conf/locale" -# ], -COMPREHENSIVE_THEME_LOCALE_PATHS = ENV_TOKENS.get('COMPREHENSIVE_THEME_LOCALE_PATHS', []) - -# PREPEND_LOCALE_PATHS contain the paths to locale directories to load first e.g. -# "PREPEND_LOCALE_PATHS" : [ -# "/edx/my-locale/" -# ], -PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', []) +STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', STATICFILES_STORAGE) -#Timezone overrides -TIME_ZONE = ENV_TOKENS.get('CELERY_TIMEZONE', CELERY_TIMEZONE) +CSRF_TRUSTED_ORIGINS = _YAML_TOKENS.get("CSRF_TRUSTED_ORIGINS_WITH_SCHEME", []) -##### REGISTRATION RATE LIMIT SETTINGS ##### -REGISTRATION_VALIDATION_RATELIMIT = ENV_TOKENS.get( - 'REGISTRATION_VALIDATION_RATELIMIT', REGISTRATION_VALIDATION_RATELIMIT -) -REGISTRATION_RATELIMIT = ENV_TOKENS.get('REGISTRATION_RATELIMIT', REGISTRATION_RATELIMIT) +MKTG_URL_LINK_MAP.update(_YAML_TOKENS.get('MKTG_URL_LINK_MAP', {})) -# Push to LMS overrides -GIT_REPO_EXPORT_DIR = ENV_TOKENS.get('GIT_REPO_EXPORT_DIR', '/edx/var/edxapp/export_course_repos') +#Timezone overrides +TIME_ZONE = CELERY_TIMEZONE -ENV_FEATURES = ENV_TOKENS.get('FEATURES', {}) -for feature, value in ENV_FEATURES.items(): +for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value # Additional installed apps -for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): +for app in _YAML_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) -LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), - service_variant=SERVICE_VARIANT) - -# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text -# values when the varibale is an empty string. Therefore, setting these variable as empty text in related -# json files will make the system reads thier values from django translation files -PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME -PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION -STUDIO_NAME = ENV_TOKENS.get('STUDIO_NAME') or STUDIO_NAME -STUDIO_SHORT_NAME = ENV_TOKENS.get('STUDIO_SHORT_NAME') or STUDIO_SHORT_NAME - -# Event Tracking -if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS: - TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS") - -# Heartbeat -HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HEARTBEAT_CELERY_ROUTING_KEY) - -# Sometimes, OAuth2 clients want the user to redirect back to their site after logout. But to determine if the given -# redirect URL/path is safe for redirection, the following variable is used by edX. -LOGIN_REDIRECT_WHITELIST = ENV_TOKENS.get( - 'LOGIN_REDIRECT_WHITELIST', - LOGIN_REDIRECT_WHITELIST +LOGGING = get_logger_config( + LOG_DIR, + logging_env=LOGGING_ENV, + service_variant=SERVICE_VARIANT, ) + LOGIN_REDIRECT_WHITELIST.extend([reverse_lazy('home')]) ############### XBlock filesystem field config ########## -if 'DJFS' in AUTH_TOKENS and AUTH_TOKENS['DJFS'] is not None: - DJFS = AUTH_TOKENS['DJFS'] - if 'url_root' in DJFS: - DJFS['url_root'] = DJFS['url_root'].format(platform_revision=EDX_PLATFORM_REVISION) - - -AWS_SES_REGION_NAME = ENV_TOKENS.get('AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = ENV_TOKENS.get('AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') +if 'url_root' in DJFS: + DJFS['url_root'] = DJFS['url_root'].format(platform_revision=EDX_PLATFORM_REVISION) # Note that this is the Studio key for Segment. There is a separate key for the LMS. -CMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') - -SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] +CMS_SEGMENT_KEY = _YAML_TOKENS.get('SEGMENT_KEY') -AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None -AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads') - -# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it -# normally appends to every returned URL. -AWS_QUERYSTRING_AUTH = AUTH_TOKENS.get('AWS_QUERYSTRING_AUTH', True) - AWS_DEFAULT_ACL = 'private' AWS_BUCKET_ACL = AWS_DEFAULT_ACL # The number of seconds that a generated URL is valid for. AWS_QUERYSTRING_EXPIRE = 7 * 24 * 60 * 60 # 7 days -AWS_S3_CUSTOM_DOMAIN = AUTH_TOKENS.get('AWS_S3_CUSTOM_DOMAIN', 'edxuploads.s3.amazonaws.com') -if AUTH_TOKENS.get('DEFAULT_FILE_STORAGE'): - DEFAULT_FILE_STORAGE = AUTH_TOKENS.get('DEFAULT_FILE_STORAGE') -elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: +# Change to S3Boto3 if we haven't specified another default storage AND we have specified AWS creds. +if (not _YAML_TOKENS.get('DEFAULT_FILE_STORAGE')) and AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -else: - DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' - -COURSE_IMPORT_EXPORT_BUCKET = ENV_TOKENS.get('COURSE_IMPORT_EXPORT_BUCKET', '') if COURSE_IMPORT_EXPORT_BUCKET: COURSE_IMPORT_EXPORT_STORAGE = 'cms.djangoapps.contentstore.storage.ImportExportS3Storage' @@ -369,15 +276,11 @@ def get_env_setting(setting): USER_TASKS_ARTIFACT_STORAGE = COURSE_IMPORT_EXPORT_STORAGE -COURSE_METADATA_EXPORT_BUCKET = ENV_TOKENS.get('COURSE_METADATA_EXPORT_BUCKET', '') - if COURSE_METADATA_EXPORT_BUCKET: COURSE_METADATA_EXPORT_STORAGE = 'cms.djangoapps.export_course_metadata.storage.CourseMetadataExportS3Storage' else: COURSE_METADATA_EXPORT_STORAGE = DEFAULT_FILE_STORAGE -DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) - # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* # environment variables @@ -392,59 +295,35 @@ def get_env_setting(setting): 'PORT': os.environ.get('DB_MIGRATION_PORT', database['PORT']), }) -MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE)) - -MODULESTORE_FIELD_OVERRIDE_PROVIDERS = ENV_TOKENS.get( - 'MODULESTORE_FIELD_OVERRIDE_PROVIDERS', - MODULESTORE_FIELD_OVERRIDE_PROVIDERS -) - -XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( - 'XBLOCK_FIELD_DATA_WRAPPERS', - XBLOCK_FIELD_DATA_WRAPPERS -) - -CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) -DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG) +MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE) # Celery Broker -CELERY_ALWAYS_EAGER = ENV_TOKENS.get("CELERY_ALWAYS_EAGER", False) -CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") -CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "") -CELERY_BROKER_VHOST = ENV_TOKENS.get("CELERY_BROKER_VHOST", "") -CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "") -CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "") -CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION = ENV_TOKENS.get("CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION", True) BROKER_URL = "{}://{}:{}@{}/{}".format(CELERY_BROKER_TRANSPORT, CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOSTNAME, CELERY_BROKER_VHOST) -BROKER_USE_SSL = ENV_TOKENS.get('CELERY_BROKER_USE_SSL', False) try: BROKER_TRANSPORT_OPTIONS = { 'fanout_patterns': True, 'fanout_prefix': True, - **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) + **_YAML_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) } except TypeError as exc: raise ImproperlyConfigured('CELERY_BROKER_TRANSPORT_OPTIONS must be a dict') from exc -# Message expiry time in seconds -CELERY_EVENT_QUEUE_TTL = ENV_TOKENS.get('CELERY_EVENT_QUEUE_TTL', None) - -# Allow CELERY_QUEUES to be overwritten by ENV_TOKENS, -ENV_CELERY_QUEUES = ENV_TOKENS.get('CELERY_QUEUES', None) -if ENV_CELERY_QUEUES: - CELERY_QUEUES = {queue: {} for queue in ENV_CELERY_QUEUES} +# Build a CELERY_QUEUES dict the way that celery expects, based on a couple lists of queue names from the YAML. +_YAML_CELERY_QUEUES = _YAML_TOKENS.get('CELERY_QUEUES', None) +if _YAML_CELERY_QUEUES: + CELERY_QUEUES = {queue: {} for queue in _YAML_CELERY_QUEUES} # Then add alternate environment queues -ALTERNATE_QUEUE_ENVS = ENV_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() +_YAML_ALTERNATE_WORKER_QUEUES = _YAML_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() ALTERNATE_QUEUES = [ DEFAULT_PRIORITY_QUEUE.replace(QUEUE_VARIANT, alternate + '.') - for alternate in ALTERNATE_QUEUE_ENVS + for alternate in _YAML_ALTERNATE_WORKER_QUEUES ] CELERY_QUEUES.update( @@ -455,148 +334,82 @@ def get_env_setting(setting): } ) -# Queue to use for updating grades due to grading policy change -POLICY_CHANGE_GRADES_ROUTING_KEY = ENV_TOKENS.get('POLICY_CHANGE_GRADES_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) - -# Queue to use for individual learner course regrades -SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY = ENV_TOKENS.get( - 'SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY', - DEFAULT_PRIORITY_QUEUE -) - -SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = ENV_TOKENS.get( - 'SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY', - HIGH_PRIORITY_QUEUE -) - # Event tracking -TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( - AUTH_TOKENS.get("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", [])) - -##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### -MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED +TRACKING_BACKENDS.update(_YAML_TOKENS.get("TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update( + _YAML_TOKENS.get("EVENT_TRACKING_BACKENDS", {}) ) - -MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS +EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( + EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST ) -#### PASSWORD POLICY SETTINGS ##### -AUTH_PASSWORD_VALIDATORS = ENV_TOKENS.get("AUTH_PASSWORD_VALIDATORS", AUTH_PASSWORD_VALIDATORS) - -### INACTIVITY SETTINGS #### -SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIMEOUT_IN_SECONDS") - -################ PUSH NOTIFICATIONS ############### -PARSE_KEYS = AUTH_TOKENS.get("PARSE_KEYS", {}) - - -# Video Caching. Pairing country codes with CDN URLs. -# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='} -VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {}) if FEATURES['ENABLE_COURSEWARE_INDEX'] or FEATURES['ENABLE_LIBRARY_INDEX']: # Use ElasticSearch for the search engine SEARCH_ENGINE = "search.elastic.ElasticSearchEngine" # TODO: Once we have successfully upgraded to ES7, switch this back to ELASTIC_SEARCH_CONFIG. -ELASTIC_SEARCH_CONFIG = ENV_TOKENS.get('ELASTIC_SEARCH_CONFIG_ES7', [{}]) +ELASTIC_SEARCH_CONFIG = _YAML_TOKENS.get('ELASTIC_SEARCH_CONFIG_ES7', [{}]) -XBLOCK_SETTINGS = ENV_TOKENS.get('XBLOCK_SETTINGS', {}) -XBLOCK_SETTINGS.setdefault("VideoBlock", {})["licensing_enabled"] = FEATURES.get("LICENSING", False) -XBLOCK_SETTINGS.setdefault("VideoBlock", {})['YOUTUBE_API_KEY'] = AUTH_TOKENS.get('YOUTUBE_API_KEY', YOUTUBE_API_KEY) +XBLOCK_SETTINGS.setdefault("VideoBlock", {})["licensing_enabled"] = FEATURES["LICENSING"] +XBLOCK_SETTINGS.setdefault("VideoBlock", {})['YOUTUBE_API_KEY'] = YOUTUBE_API_KEY ############################ OAUTH2 Provider ################################### #### JWT configuration #### -JWT_AUTH.update(ENV_TOKENS.get('JWT_AUTH', {})) -JWT_AUTH.update(AUTH_TOKENS.get('JWT_AUTH', {})) +JWT_AUTH.update(_YAML_TOKENS.get('JWT_AUTH', {})) ######################## CUSTOM COURSES for EDX CONNECTOR ###################### -if FEATURES.get('CUSTOM_COURSES_EDX'): +if FEATURES['CUSTOM_COURSES_EDX']: INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig') -########## Settings for video transcript migration tasks ############ -VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = ENV_TOKENS.get('VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE) - -########## Settings youtube thumbnails scraper tasks ############ -SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = ENV_TOKENS.get('SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE) - -########## Settings update search index task ############ -UPDATE_SEARCH_INDEX_JOB_QUEUE = ENV_TOKENS.get('UPDATE_SEARCH_INDEX_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE) - ########################## Extra middleware classes ####################### # Allow extra middleware classes to be added to the app through configuration. -MIDDLEWARE.extend(ENV_TOKENS.get('EXTRA_MIDDLEWARE_CLASSES', [])) +MIDDLEWARE.extend(EXTRA_MIDDLEWARE_CLASSES) -########################## Settings for Completion API ##################### -# Once a user has watched this percentage of a video, mark it as complete: -# (0.0 = 0%, 1.0 = 100%) -COMPLETION_VIDEO_COMPLETE_PERCENTAGE = ENV_TOKENS.get( - 'COMPLETION_VIDEO_COMPLETE_PERCENTAGE', - COMPLETION_VIDEO_COMPLETE_PERCENTAGE, -) - -####################### Enterprise Settings ###################### - -# A default dictionary to be used for filtering out enterprise customer catalog. -ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER = ENV_TOKENS.get( - 'ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER', - ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER -) -ENTERPRISE_CATALOG_INTERNAL_ROOT_URL = ENV_TOKENS.get( - 'ENTERPRISE_CATALOG_INTERNAL_ROOT_URL', - ENTERPRISE_CATALOG_INTERNAL_ROOT_URL -) -INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT = ENV_TOKENS.get( - 'INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT', - INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT -) - -############### Settings for Retirement ##################### -RETIREMENT_SERVICE_WORKER_USERNAME = ENV_TOKENS.get( - 'RETIREMENT_SERVICE_WORKER_USERNAME', - RETIREMENT_SERVICE_WORKER_USERNAME -) - -############### Settings for Exams #################### -EXAMS_SERVICE_URL = ENV_TOKENS.get('EXAMS_SERVICE_URL', EXAMS_SERVICE_URL) -EXAMS_SERVICE_USERNAME = ENV_TOKENS.get('EXAMS_SERVICE_USERNAME', EXAMS_SERVICE_USERNAME) - -############### Settings for edx-rbac ############### -SYSTEM_WIDE_ROLE_CLASSES = ENV_TOKENS.get('SYSTEM_WIDE_ROLE_CLASSES') or SYSTEM_WIDE_ROLE_CLASSES - -######################## Setting for content libraries ######################## -MAX_BLOCKS_PER_CONTENT_LIBRARY = ENV_TOKENS.get('MAX_BLOCKS_PER_CONTENT_LIBRARY', MAX_BLOCKS_PER_CONTENT_LIBRARY) - -########################## Derive Any Derived Settings ####################### +####################################################################################################################### +#### DERIVE ANY DERIVED SETTINGS +#### derive_settings(__name__) -####################### Plugin Settings ########################## - -# This is at the bottom because it is going to load more settings after base settings are loaded +####################################################################################################################### +#### LOAD SETTINGS FROM DJANGO PLUGINS +#### +#### This is at the bottom because it is going to load more settings after base settings are loaded +#### + +# These dicts are defined solely for BACKWARDS COMPATIBILITY with existing plugins which may theoretically +# rely upon them. Please do not add new references to these dicts! +# - If you need to access the YAML values in this module, use _YAML_TOKENS. +# - If you need to access to these values elsewhere, use the corresponding rendered `settings.*` +# value rathering than diving into these dicts. +ENV_TOKENS = _YAML_TOKENS +AUTH_TOKENS = _YAML_TOKENS +ENV_FEATURES = _YAML_TOKENS.get("FEATURES", {}) +ENV_CELERY_QUEUES = _YAML_CELERY_QUEUES +ALTERNATE_QUEUE_ENVS = _YAML_ALTERNATE_WORKER_QUEUES + +# Load production.py in plugins add_plugins(__name__, ProjectType.CMS, SettingsType.PRODUCTION) -############# CORS headers for cross-domain requests ################# -if FEATURES.get('ENABLE_CORS_HEADERS'): - CORS_ALLOW_CREDENTIALS = True - CORS_ORIGIN_WHITELIST = ENV_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) - CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) - CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) +####################################################################################################################### +#### MORE YAML POST-PROCESSING +#### +#### More post-processing, but these will not be available Django plugins. +#### Unclear whether or not these are down here intentionally. +#### -################# Settings for brand logos. ################# -LOGO_URL = ENV_TOKENS.get('LOGO_URL', LOGO_URL) -LOGO_URL_PNG = ENV_TOKENS.get('LOGO_URL_PNG', LOGO_URL_PNG) -LOGO_TRADEMARK_URL = ENV_TOKENS.get('LOGO_TRADEMARK_URL', LOGO_TRADEMARK_URL) -FAVICON_URL = ENV_TOKENS.get('FAVICON_URL', FAVICON_URL) +############# CORS headers for cross-domain requests ################# +if FEATURES['ENABLE_CORS_HEADERS']: + CORS_ALLOW_CREDENTIALS = True + CORS_ORIGIN_WHITELIST = _YAML_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) + CORS_ORIGIN_ALLOW_ALL = _YAML_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) + CORS_ALLOW_INSECURE = _YAML_TOKENS.get('CORS_ALLOW_INSECURE', False) ######################## CELERY ROTUING ######################## @@ -617,48 +430,33 @@ def get_env_setting(setting): 'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE}, } -LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '') - ############## XBlock extra mixins ############################ XBLOCK_MIXINS += tuple(XBLOCK_EXTRA_MIXINS) -############## Settings for course import olx validation ############################ -COURSE_OLX_VALIDATION_STAGE = ENV_TOKENS.get('COURSE_OLX_VALIDATION_STAGE', COURSE_OLX_VALIDATION_STAGE) -COURSE_OLX_VALIDATION_IGNORE_LIST = ENV_TOKENS.get( - 'COURSE_OLX_VALIDATION_IGNORE_LIST', - COURSE_OLX_VALIDATION_IGNORE_LIST -) - -################# show account activate cta after register ######################## -SHOW_ACCOUNT_ACTIVATION_CTA = ENV_TOKENS.get('SHOW_ACCOUNT_ACTIVATION_CTA', SHOW_ACCOUNT_ACTIVATION_CTA) - -LANGUAGE_COOKIE_NAME = ENV_TOKENS.get('LANGUAGE_COOKIE', None) or ENV_TOKENS.get( - 'LANGUAGE_COOKIE_NAME', LANGUAGE_COOKIE_NAME) - -################# Discussions micro frontend URL ######################## -DISCUSSIONS_MICROFRONTEND_URL = ENV_TOKENS.get('DISCUSSIONS_MICROFRONTEND_URL', DISCUSSIONS_MICROFRONTEND_URL) - -################### Discussions micro frontend Feedback URL################### -DISCUSSIONS_MFE_FEEDBACK_URL = ENV_TOKENS.get('DISCUSSIONS_MFE_FEEDBACK_URL', DISCUSSIONS_MFE_FEEDBACK_URL) - -############################ AI_TRANSLATIONS URL ################################## -AI_TRANSLATIONS_API_URL = ENV_TOKENS.get('AI_TRANSLATIONS_API_URL', AI_TRANSLATIONS_API_URL) +# Translation overrides +LANGUAGE_COOKIE_NAME = _YAML_TOKENS.get('LANGUAGE_COOKIE') or LANGUAGE_COOKIE_NAME ############## DRF overrides ############## -REST_FRAMEWORK.update(ENV_TOKENS.get('REST_FRAMEWORK', {})) +REST_FRAMEWORK.update(_YAML_TOKENS.get('REST_FRAMEWORK', {})) # keys for big blue button live provider +# TODO: This should not be in the core platform. If it has to stay for now, though, then we should move these +# defaults into common.py COURSE_LIVE_GLOBAL_CREDENTIALS["BIG_BLUE_BUTTON"] = { - "KEY": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_KEY', None), - "SECRET": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_SECRET', None), - "URL": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_URL', None), + "KEY": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_KEY'), + "SECRET": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_SECRET'), + "URL": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_URL'), } +# TODO: We believe that this could be more simply defined as CMS_ROOT_URL. We are not making the change now, +# but please don't follow this pattern in other defaults... INACTIVE_USER_URL = f'http{"s" if HTTPS == "on" else ""}://{CMS_BASE}' ############## Event bus producer ############## -EVENT_BUS_PRODUCER_CONFIG = merge_producer_configs(EVENT_BUS_PRODUCER_CONFIG, - ENV_TOKENS.get('EVENT_BUS_PRODUCER_CONFIG', {})) +EVENT_BUS_PRODUCER_CONFIG = merge_producer_configs( + EVENT_BUS_PRODUCER_CONFIG, + _YAML_TOKENS.get('EVENT_BUS_PRODUCER_CONFIG', {}) +) ############## Authoring API drf-spectacular openapi settings ############## # These fields override the spectacular settings default values. @@ -689,11 +487,9 @@ def get_env_setting(setting): ], } -BEAMER_PRODUCT_ID = ENV_TOKENS.get('BEAMER_PRODUCT_ID', BEAMER_PRODUCT_ID) -# .. setting_name: DISABLED_COUNTRIES -# .. setting_default: [] -# .. setting_description: List of country codes that should be disabled -# .. for now it wil impact country listing in auth flow and user profile. -# .. eg ['US', 'CA'] -DISABLED_COUNTRIES = ENV_TOKENS.get('DISABLED_COUNTRIES', []) +####################################################################################################################### +# HEY! Don't add anything to the end of this file. +# Add your defaults to common.py instead! +# If you really need to add post-YAML logic, add it above the "DERIVE ANY DERIVED SETTINGS" section. +####################################################################################################################### diff --git a/lms/envs/production.py b/lms/envs/production.py index 4887804e7e0a..712200f8f1bb 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -129,6 +129,8 @@ def get_env_setting(setting): EMAIL_FILE_PATH = Derived(lambda settings: settings.DATA_DIR / "emails" / "lms") LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) +CSRF_TRUSTED_ORIGINS = Derived(lambda settings: settings.CSRF_TRUSTED_ORIGINS_WITH_SCHEME) + # This is the domain that is used to set shared cookies between various sub-domains. # By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN SHARED_COOKIE_DOMAIN = Derived(lambda settings: settings.SESSION_COOKIE_DOMAIN) @@ -341,7 +343,6 @@ def get_env_setting(setting): service_variant=SERVICE_VARIANT, ) -# Determines which origins are trusted for unsafe requests eg. POST requests. CSRF_TRUSTED_ORIGINS = _YAML_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) if FEATURES['ENABLE_CORS_HEADERS'] or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'): @@ -393,7 +394,7 @@ def get_env_setting(setting): # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py -MODULESTORE = convert_module_store_setting_if_needed(_YAML_TOKENS.get('MODULESTORE', MODULESTORE)) +MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE) BROKER_URL = "{}://{}:{}@{}/{}".format(CELERY_BROKER_TRANSPORT, CELERY_BROKER_USER, @@ -532,6 +533,8 @@ def get_env_setting(setting): ########################## Extra middleware classes ####################### # Allow extra middleware classes to be added to the app through configuration. +# TODO: Declare `EXTRA_MIDDLEWARE_CLASSES = []` in lms/envs/common.py so that we can simplify this +# next line. See CMS settings for the example of what we want. MIDDLEWARE.extend(_YAML_TOKENS.get('EXTRA_MIDDLEWARE_CLASSES', [])) ########################## Derive Any Derived Settings ####################### From afd8abec62bed52204d3c2c67c0dbdff6ebd49a2 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 28 Feb 2025 15:28:17 -0500 Subject: [PATCH 2/3] docs: Make LMS's production.py comments match CMS's --- lms/envs/production.py | 79 +++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/lms/envs/production.py b/lms/envs/production.py index 712200f8f1bb..5fb41f9f47fb 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -1,9 +1,8 @@ """ -This is the default template for our main set of AWS servers. +Override common.py with key-value pairs from YAML (plus some extra defaults & post-processing). -Common traits: -* Use memcached, and cache-backed sessions -* Use a MySQL 5.1 database +This file is in the process of being restructured. Please see: +https://github.com/openedx/edx-platform/blob/master/docs/decisions/0022-settings-simplification.rst """ # We intentionally define lots of variables that aren't used, and @@ -43,10 +42,13 @@ def get_env_setting(setting): raise ImproperlyConfigured(error_msg) # lint-amnesty, pylint: disable=raise-missing-from -################################################# PRODUCTION DEFAULTS ################################################ -# We configure some defaults (beyond what has already been configured in common.py) before loading the YAML file below. -# DO NOT ADD NEW DEFAULTS HERE! Put any new setting defaults in common.py instead, along with a setting annotation. -# TODO: Move all these defaults into common.py. +####################################################################################################################### +#### PRODUCTION DEFAULTS +#### +#### Configure some defaults (beyond what has already been configured in common.py) before loading the YAML file. +#### DO NOT ADD NEW DEFAULTS HERE! Put any new setting defaults in common.py instead, along with a setting annotation. +#### TODO: Move all these defaults into common.py. +#### DEBUG = False DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False @@ -87,8 +89,6 @@ def get_env_setting(setting): VIDEO_CDN_URL = {} HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = {} AWS_STORAGE_BUCKET_NAME = 'edxuploads' -# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it -# normally appends to every returned URL. AWS_QUERYSTRING_AUTH = True AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' MONGODB_LOG = {} @@ -147,7 +147,6 @@ def get_env_setting(setting): # Queue to use for expiring old entitlements ENTITLEMENTS_EXPIRATION_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) -# Intentional defaults. ID_VERIFICATION_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) PASSWORD_RESET_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) ACTIVATION_EMAIL_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) @@ -159,28 +158,12 @@ def get_env_setting(setting): PROGRAM_CERTIFICATES_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) -############## OPEN EDX ENTERPRISE SERVICE CONFIGURATION ###################### -# The Open edX Enterprise service is currently hosted via the LMS container/process. -# However, for all intents and purposes this service is treated as a standalone IDA. -# These configuration settings are specific to the Enterprise service and you should -# not find references to them within the edx-platform project. - -# Publicly-accessible enrollment URL, for use on the client side. ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = Derived( lambda settings: (settings.LMS_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH ) - -# Enrollment URL used on the server-side. ENTERPRISE_ENROLLMENT_API_URL = Derived( lambda settings: (settings.LMS_INTERNAL_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH ) - -############## ENTERPRISE SERVICE API CLIENT CONFIGURATION ###################### -# The LMS communicates with the Enterprise service via the requests.Session() client -# The below environmental settings are utilized by the LMS when interacting with -# the service, and override the default parameters which are defined in common.py - - DEFAULT_ENTERPRISE_API_URL = Derived( lambda settings: ( None if settings.LMS_INTERNAL_ROOT_URL is None @@ -188,7 +171,6 @@ def get_env_setting(setting): ) ) ENTERPRISE_API_URL = DEFAULT_ENTERPRISE_API_URL - DEFAULT_ENTERPRISE_CONSENT_API_URL = Derived( lambda settings: ( None if settings.LMS_INTERNAL_ROOT_URL is None @@ -199,6 +181,8 @@ def get_env_setting(setting): ####################################################################################################################### +#### YAML LOADING +#### # A file path to a YAML file from which to load configuration overrides for LMS. CONFIG_FILE = get_env_setting('LMS_CFG') @@ -235,6 +219,11 @@ def get_env_setting(setting): ] }) + +####################################################################################################################### +#### LOAD THE EDX-PLATFORM GIT REVISION +#### + try: # A file path to a YAML file from which to load all the code revisions currently deployed REVISION_CONFIG_FILE = get_env_setting('REVISION_CFG') @@ -247,7 +236,13 @@ def get_env_setting(setting): # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -###################################### CELERY ################################ + +####################################################################################################################### +#### POST-PROCESSING OF YAML +#### +#### This is where we do a bunch of logic to post-process the results of the YAML, including: conditionally setting +#### updates, merging dicts+lists which we did not override, and in some cases simply ignoring the YAML value in favor +#### of a specific production value. # Don't use a connection pool, since connections are dropped by ELB. BROKER_POOL_LIMIT = 0 @@ -301,7 +296,6 @@ def get_env_setting(setting): # Once we have migrated to service assets off S3, then we can convert this back to # managed by the yaml file contents - # Build a CELERY_QUEUES dict the way that celery expects, based on a couple lists of queue names from the YAML. _YAML_CELERY_QUEUES = _YAML_TOKENS.get('CELERY_QUEUES', None) if _YAML_CELERY_QUEUES: @@ -537,11 +531,19 @@ def get_env_setting(setting): # next line. See CMS settings for the example of what we want. MIDDLEWARE.extend(_YAML_TOKENS.get('EXTRA_MIDDLEWARE_CLASSES', [])) -########################## Derive Any Derived Settings ####################### + +####################################################################################################################### +#### DERIVE ANY DERIVED SETTINGS +#### derive_settings(__name__) -############################### Plugin Settings ############################### + +####################################################################################################################### +#### LOAD SETTINGS FROM DJANGO PLUGINS +#### +#### This is at the bottom because it is going to load more settings after base settings are loaded +#### # This is at the bottom because it is going to load more settings after base settings are loaded @@ -560,6 +562,13 @@ def get_env_setting(setting): add_plugins(__name__, ProjectType.LMS, SettingsType.PRODUCTION) +####################################################################################################################### +#### MORE YAML POST-PROCESSING +#### +#### More post-processing, but these will not be available Django plugins. +#### Unclear whether or not these are down here intentionally. +#### + ######################## CELERY ROUTING ######################## # Defines alternate environment tasks, as a dict of form { task_name: alternate_queue } @@ -642,8 +651,8 @@ def get_env_setting(setting): _YAML_TOKENS.get('EVENT_BUS_PRODUCER_CONFIG', {}) ) -##################################################################################################### +####################################################################################################################### # HEY! Don't add anything to the end of this file. # Add your defaults to common.py instead! -# If you really need to add post-YAML logic, add it above the "Derive Any Derived Settings" section. -###################################################################################################### +# If you really need to add post-YAML logic, add it above the "DERIVE ANY DERIVED SETTINGS" section. +####################################################################################################################### From 16d41776c9f3007f2e2a8a4a75f94d4ef664f0a7 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 28 Feb 2025 17:51:24 -0500 Subject: [PATCH 3/3] style: Alphabetize defaults in production.py We separate out the handful of settings which have useful comments. The rest of the settings' comments were not helpful--they were either just stating the obvious, or they were duplicative of what's documented in common.py. --- cms/envs/production.py | 67 +++++++++++++++++---------------- lms/envs/production.py | 84 ++++++++++++++++-------------------------- 2 files changed, 65 insertions(+), 86 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index 121383845110..582130ff7ba2 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -49,62 +49,61 @@ def get_env_setting(setting): DEBUG = False -SESSION_ENGINE = 'django.contrib.sessions.backends.cache' - -# IMPORTANT: With this enabled, the server must always be behind a proxy that -# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise, -# a user can fool our server into thinking it was an https connection. -# See -# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header -# for other warnings. +# IMPORTANT: With this enabled, the server must always be behind a proxy that strips the header HTTP_X_FORWARDED_PROTO +# from client requests. Otherwise, a user can fool our server into thinking it was an https connection. See +# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -CELERY_RESULT_BACKEND = 'django-cache' +# Constant defaults (alphabetical) +AUTHORING_API_URL = '' +AWS_QUERYSTRING_AUTH = True +AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' +AWS_STORAGE_BUCKET_NAME = 'edxuploads' BROKER_HEARTBEAT = 60.0 BROKER_HEARTBEAT_CHECKRATE = 2 -STATIC_URL_BASE = None -OPENAPI_CACHE_TIMEOUT = 60 * 60 -STATIC_ROOT_BASE = None -CMS_BASE = None -LMS_BASE = None -LMS_ROOT_URL = None -CMS_ROOT_URL = None -AUTHORING_API_URL = '' +CELERY_ALWAYS_EAGER = False +CELERY_BROKER_HOSTNAME = "" +CELERY_BROKER_PASSWORD = "" +CELERY_BROKER_TRANSPORT = "" +CELERY_BROKER_USER = "" +CELERY_RESULT_BACKEND = 'django-cache' CHAT_COMPLETION_API = '' CHAT_COMPLETION_API_KEY = '' +CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION = True +CMS_BASE = None +CMS_ROOT_URL = None +DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False +IDA_LOGOUT_URI_LIST = [] LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT = '' LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT = '' LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT = '' LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT = '' -IDA_LOGOUT_URI_LIST = [] +LMS_BASE = None +LMS_ROOT_URL = None +OPENAPI_CACHE_TIMEOUT = 60 * 60 +PARSE_KEYS = {} +REGISTRATION_EMAIL_PATTERNS_ALLOWED = None SESSION_COOKIE_DOMAIN = None SESSION_COOKIE_HTTPONLY = True -REGISTRATION_EMAIL_PATTERNS_ALLOWED = None -AWS_STORAGE_BUCKET_NAME = 'edxuploads' -AWS_QUERYSTRING_AUTH = True -AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' -CELERY_ALWAYS_EAGER = False -CELERY_BROKER_TRANSPORT = "" -CELERY_BROKER_HOSTNAME = "" -CELERY_BROKER_USER = "" -CELERY_BROKER_PASSWORD = "" -CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION = True -PARSE_KEYS = {} -VIDEO_CDN_URL = {} +SESSION_ENGINE = 'django.contrib.sessions.backends.cache' SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None +STATIC_ROOT_BASE = None +STATIC_URL_BASE = None +VIDEO_CDN_URL = {} +# Derived defaults (alphabetical) +BROKER_USE_SSL = Derived(lambda settings: settings.CELERY_BROKER_USE_SSL) EMAIL_FILE_PATH = Derived(lambda settings: settings.DATA_DIR / "emails" / "studio") -LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) ENTERPRISE_API_URL = Derived(lambda settings: settings.LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/') ENTERPRISE_CONSENT_API_URL = Derived(lambda settings: settings.LMS_INTERNAL_ROOT_URL + '/consent/api/v1/') +LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) POLICY_CHANGE_GRADES_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) SHARED_COOKIE_DOMAIN = Derived(lambda settings: settings.SESSION_COOKIE_DOMAIN) SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) -VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) -SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) UPDATE_SEARCH_INDEX_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) -BROKER_USE_SSL = Derived(lambda settings: settings.CELERY_BROKER_USE_SSL) +VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) ####################################################################################################################### diff --git a/lms/envs/production.py b/lms/envs/production.py index 5fb41f9f47fb..4884e0a22c87 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -51,18 +51,20 @@ def get_env_setting(setting): #### DEBUG = False -DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False - -SESSION_ENGINE = 'django.contrib.sessions.backends.cache' -# IMPORTANT: With this enabled, the server must always be behind a proxy that -# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise, -# a user can fool our server into thinking it was an https connection. -# See -# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header -# for other warnings. +# IMPORTANT: With this enabled, the server must always be behind a proxy that strips the header HTTP_X_FORWARDED_PROTO +# from client requests. Otherwise, a user can fool our server into thinking it was an https connection. See +# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +# TODO: We believe these were part of the DEPR'd sysadmin dashboard, and can likely be removed. +SSL_AUTH_EMAIL_DOMAIN = "MIT.EDU" +SSL_AUTH_DN_FORMAT_STRING = ( + "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" +) + +DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False +SESSION_ENGINE = 'django.contrib.sessions.backends.cache' CELERY_RESULT_BACKEND = 'django-cache' BROKER_HEARTBEAT = 60.0 BROKER_HEARTBEAT_CHECKRATE = 2 @@ -119,65 +121,43 @@ def get_env_setting(setting): MAINTENANCE_BANNER_TEXT = None DASHBOARD_COURSE_LIMIT = None -# TODO: We believe these were part of the DEPR'd sysadmin dashboard, and can likely be removed. -SSL_AUTH_EMAIL_DOMAIN = "MIT.EDU" -SSL_AUTH_DN_FORMAT_STRING = ( - "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" -) - -CC_MERCHANT_NAME = Derived(lambda settings: settings.PLATFORM_NAME) -EMAIL_FILE_PATH = Derived(lambda settings: settings.DATA_DIR / "emails" / "lms") -LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) - -CSRF_TRUSTED_ORIGINS = Derived(lambda settings: settings.CSRF_TRUSTED_ORIGINS_WITH_SCHEME) - -# This is the domain that is used to set shared cookies between various sub-domains. -# By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN -SHARED_COOKIE_DOMAIN = Derived(lambda settings: settings.SESSION_COOKIE_DOMAIN) - -# We want Bulk Email running on the high-priority queue, so we define the -# routing key that points to it. At the moment, the name is the same. -# We have to reset the value here, since we have changed the value of the queue name. +# Derived defaults (alphabetical) +ACTIVATION_EMAIL_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) BULK_EMAIL_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) - -# We can run smaller jobs on the low priority queue. See note above for why -# we have to reset the value here. BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) - -# Queue to use for expiring old entitlements -ENTITLEMENTS_EXPIRATION_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) - -ID_VERIFICATION_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) -PASSWORD_RESET_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) -ACTIVATION_EMAIL_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) -LOGIN_ISSUE_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) - -# Default queues for various routes -GRADES_DOWNLOAD_ROUTING_KEY = Derived(lambda settings: settings.HIGH_MEM_QUEUE) +CC_MERCHANT_NAME = Derived(lambda settings: settings.PLATFORM_NAME) CREDENTIALS_GENERATION_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) -PROGRAM_CERTIFICATES_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) -SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) - -ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = Derived( - lambda settings: (settings.LMS_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH -) -ENTERPRISE_ENROLLMENT_API_URL = Derived( - lambda settings: (settings.LMS_INTERNAL_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH -) +CSRF_TRUSTED_ORIGINS = Derived(lambda settings: settings.CSRF_TRUSTED_ORIGINS_WITH_SCHEME) DEFAULT_ENTERPRISE_API_URL = Derived( lambda settings: ( None if settings.LMS_INTERNAL_ROOT_URL is None else settings.LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/' ) ) -ENTERPRISE_API_URL = DEFAULT_ENTERPRISE_API_URL DEFAULT_ENTERPRISE_CONSENT_API_URL = Derived( lambda settings: ( None if settings.LMS_INTERNAL_ROOT_URL is None else settings.LMS_INTERNAL_ROOT_URL + '/consent/api/v1/' ) ) +ENTERPRISE_API_URL = DEFAULT_ENTERPRISE_API_URL ENTERPRISE_CONSENT_API_URL = DEFAULT_ENTERPRISE_CONSENT_API_URL +ENTERPRISE_ENROLLMENT_API_URL = Derived( + lambda settings: (settings.LMS_INTERNAL_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH +) +ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = Derived( + lambda settings: (settings.LMS_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH +) +EMAIL_FILE_PATH = Derived(lambda settings: settings.DATA_DIR / "emails" / "lms") +ENTITLEMENTS_EXPIRATION_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +GRADES_DOWNLOAD_ROUTING_KEY = Derived(lambda settings: settings.HIGH_MEM_QUEUE) +ID_VERIFICATION_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) +LMS_INTERNAL_ROOT_URL = Derived(lambda settings: settings.LMS_ROOT_URL) +LOGIN_ISSUE_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) +PASSWORD_RESET_SUPPORT_LINK = Derived(lambda settings: settings.SUPPORT_SITE_LINK) +PROGRAM_CERTIFICATES_ROUTING_KEY = Derived(lambda settings: settings.DEFAULT_PRIORITY_QUEUE) +SHARED_COOKIE_DOMAIN = Derived(lambda settings: settings.SESSION_COOKIE_DOMAIN) +SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = Derived(lambda settings: settings.HIGH_PRIORITY_QUEUE) #######################################################################################################################