feat: experimental atlas and oep-58 i18n service updates FC-0012

.gitignore

@@ -61,6 +61,7 @@ conf/locale/fake*/LC_MESSAGES/*.po
 conf/locale/fake*/LC_MESSAGES/*.mo
 # this was a mistake in i18n_tools, now fixed.
 conf/locale/messages.mo
+conf/plugins-locale/
 
 ### Testing artifacts
 .testids/

Makefile

@@ -43,7 +43,16 @@ extract_translations: ## extract localizable strings from sources
 push_translations: ## push source strings to Transifex for translation
 	i18n_tool transifex push
 
-pull_translations:  ## pull translations from Transifex
+pull_xblock_translations:
+ifeq ($(OPENEDX_ATLAS_PULL),)
+	@echo "atlas is not used due to empty OPENEDX_ATLAS_PULL environment variable"
+else
+	rm -rf conf/xblocks/locale
+	mkdir conf/xblocks/locale
+	python manage.py lms xblocks_atlas_pull_module
+endif
+
+pull_translations: pull_xblock_translations ## pull translations from Transifex
 	git clean -fdX conf/locale
 	i18n_tool transifex pull
 	i18n_tool extract

cms/envs/production.py

@@ -262,6 +262,8 @@ def get_env_setting(setting):
 #    ],
 PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', [])
 
+PLUGINS_TRANSLATIONS_ROOT = REPO_ROOT / 'conf/plugins-locale'
+
 #Timezone overrides
 TIME_ZONE = ENV_TOKENS.get('CELERY_TIMEZONE', CELERY_TIMEZONE)
 

common/djangoapps/xblock_django/api.py

@@ -2,6 +2,7 @@
 API methods related to xblock state.
 """
 
+import pkg_resources
 
 from openedx.core.lib.cache_utils import CacheInvalidationManager
 from common.djangoapps.xblock_django.models import XBlockConfiguration, XBlockStudioConfiguration
@@ -27,6 +28,14 @@ def disabled_xblocks():
     return XBlockConfiguration.objects.current_set().filter(enabled=False)
 
 
+def get_xblocks_entry_points():
+    return [entry_point for entry_point in pkg_resources.iter_entry_points(group='xblock.v1')]
+
+def get_xblocks():
+    return [entry_point.resolve() for entry_point in get_xblocks_entry_points()]
+
+
+
 def authorable_xblocks(allow_unsupported=False, name=None):
     """
     This method returns the QuerySet of XBlocks that can be created in Studio (by default, only fully supported

common/djangoapps/xblock_django/management/commands/pull_plugins_translations.py

@@ -0,0 +1,71 @@
+"""
+This command downloads the translations for the XBlocks that are installed.
+"""
+import shutil
+import subprocess
+import os.path
+
+from django.conf import settings
+
+from django.core.management.base import BaseCommand
+
+from ...api import get_xblocks_entry_points
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose|-v',
+            action='store_true',
+            default=False,
+            dest='verbose',
+            help='Verbose output.'
+        )
+
+        parser.add_argument(
+            '--list|-l',
+            action='store_true',
+            default=False,
+            dest='list',
+            help='List plugins module names.'
+        )
+
+    def handle(self, *args, **options):
+        # Remove previous translations
+        for dir_name in os.listdir(settings.PLUGINS_TRANSLATIONS_ROOT):
+            dir_path = os.path.join(settings.PLUGINS_TRANSLATIONS_ROOT, dir_name)
+
+            if os.path.isdir(dir_path):
+                shutil.rmtree(dir_path, ignore_errors=True)
+
+        xblock_module_names = set()
+        for entry_point in get_xblocks_entry_points():
+            xblock = entry_point.resolve()
+            module_import_path = xblock.__module__
+            parent_module, _rest = module_import_path.split('.', maxsplit=1)
+            if parent_module == 'xmodule':
+                if options['verbose']:
+                    self.stdout.write(f'INFO: Skipped edx-platform XBlock "{entry_point.name}" '
+                                      f'in module={module_import_path}')
+            else:
+                xblock_module_names.add(parent_module)
+
+        sorted_xblock_module_names = list(sorted(xblock_module_names))
+
+        if options['list']:
+            self.stdout.write('\n'.join(sorted_xblock_module_names))
+        else:
+            xblock_atlas_args_list = [
+                f'translations/edx-platform-plugins/{module_name}:{module_name}'
+                for module_name in sorted_xblock_module_names
+            ]
+
+            subprocess.run(
+                [
+                    'atlas', 'pull',
+                    '--repository=Zeit-Labs/openedx-translations',
+                    '--branch=plugins-fix'
+                ] + xblock_atlas_args_list,
+                cwd=settings.PLUGINS_TRANSLATIONS_ROOT,
+            )

xmodule/modulestore/django.py

@@ -8,6 +8,7 @@
 from importlib import import_module
 import gettext
 import logging
+from os import path
 
 from pkg_resources import resource_filename
 import re  # lint-amnesty, pylint: disable=wrong-import-order
@@ -29,6 +30,7 @@
 from xmodule.modulestore.draft_and_published import BranchSettingMixin  # lint-amnesty, pylint: disable=wrong-import-position
 from xmodule.modulestore.mixed import MixedModuleStore  # lint-amnesty, pylint: disable=wrong-import-position
 from xmodule.util.xmodule_django import get_current_request_hostname  # lint-amnesty, pylint: disable=wrong-import-position
+from xmodule.waffle import ENABLE_ATLAS_TRANSLATIONS
 
 # We also may not always have the current request user (crum) module available
 try:
@@ -369,30 +371,74 @@ class XBlockI18nService:
     def __init__(self, block=None):
         """
         Attempt to load an XBlock-specific GNU gettext translator using the XBlock's own domain
-        translation catalog, currently expected to be found at:
-            <xblock_root>/conf/locale/<language>/LC_MESSAGES/<domain>.po|mo
+        translation catalog.
         If we can't locate the domain translation catalog then we fall-back onto
         django.utils.translation, which will point to the system's own domain translation catalog
         This effectively achieves translations by coincidence for an XBlock which does not provide
         its own dedicated translation catalog along with its implementation.
         """
         self.translator = django.utils.translation
         if block:
-            xblock_class = getattr(block, 'unmixed_class', block.__class__)
-            xblock_resource = xblock_class.__module__
+            if ENABLE_ATLAS_TRANSLATIONS.is_enabled():
+                xblock_domain = 'django'
+            else:
+                xblock_domain = 'text'
+
+            selected_language = get_language()
+
+            xblock_locale_path = self.get_python_locale_directory(block)
+            if xblock_locale_path:
+                try:
+                    self.translator = gettext.translation(
+                        xblock_domain,
+                        xblock_locale_path,
+                        [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)]
+                    )
+                except OSError:
+                    # Fall back to the default Django translator if the XBlock translator is not found.
+                    pass
+
+    @staticmethod
+    def get_python_locale_directory(block):
+        """
+        Return the XBlock locale directory with support for OEP-58 updated translation infrastructure.
+
+        This function works in two modes:
+            1. With ENABLE_ATLAS_TRANSLATIONS enabled it loads OEP-58 external translations.
+            2. With ENABLE_ATLAS_TRANSLATIONS disabled it uses the old in-xblock translations which are
+              typically found at <xblock_root>/conf/locale/<language>/LC_MESSAGES/<domain>.po|mo
+
+        The second mode is going to be eventually deprecated when OEP-58 is fully in use.
+        """
+        xblock_class = getattr(block, 'unmixed_class', block.__class__)
+        xblock_resource = xblock_class.__module__
+
+        if ENABLE_ATLAS_TRANSLATIONS.is_enabled():
+            xblock_module_name = xblock_resource
+            xblock_locale_path = path.join(settings.PLUGINS_TRANSLATIONS_ROOT, xblock_module_name)
+        else:
             xblock_locale_dir = 'translations'
             xblock_locale_path = resource_filename(xblock_resource, xblock_locale_dir)
-            xblock_domain = 'text'
-            selected_language = get_language()
-            try:
-                self.translator = gettext.translation(
-                    xblock_domain,
-                    xblock_locale_path,
-                    [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)]
-                )
-            except OSError:
-                # Fall back to the default Django translator if the XBlock translator is not found.
-                pass
+
+        return xblock_locale_path
+
+    @staticmethod
+    def get_javascript_locale_path(block):
+        """
+        Return the XBlock compiled javascript i18n path with support for OEP-58 updated translation infrastructure.
+
+        This function only works With ENABLE_ATLAS_TRANSLATIONS waffle switch enabled.
+        """
+        xblock_class = getattr(block, 'unmixed_class', block.__class__)
+        xblock_resource = xblock_class.__module__
+        selected_language = get_language()
+
+        if ENABLE_ATLAS_TRANSLATIONS.is_enabled():
+            xblock_module_name = xblock_resource
+            translations_dir = settings.XBLOCK_TRANSLATIONS_DIRECTORY
+            xblock_locale_path = path.join(translations_dir, xblock_module_name, selected_language, 'django.js')
+            if path.exists(xblock_locale_path):
+                return xblock_locale_path
 
     def __getattr__(self, name):
         name = 'gettext' if name == 'ugettext' else name

xmodule/waffle.py

@@ -0,0 +1,20 @@
+"""
+This module contains configuration settings via waffle flags for the xmodule modulestore.
+"""
+
+from edx_toggles.toggles import WaffleSwitch
+
+# XModule Namespace
+WAFFLE_NAMESPACE = 'xmodule'
+
+# .. toggle_name: xmodule.enable_atlas_translations
+# .. toggle_implementation: WaffleSwitch
+# .. toggle_default: False
+# .. toggle_description: Waffle switch for loading XBlock translations from external directory in line with to OEP-58.
+# .. toggle_use_cases: temporary
+# .. toggle_creation_date: 2023-03-28
+# .. toggle_tickets: TODO: add here the docs.openedx.org document link.
+
+ENABLE_ATLAS_TRANSLATIONS = WaffleSwitch(
+    f'{WAFFLE_NAMESPACE}.enable_atlas_translations', __name__
+)