Maurits/character sets

.meta.toml

@@ -41,4 +41,7 @@ ignore-bad-ideas = [
     "src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo",
     "src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo",
     "src/zope/i18n/tests/pl-default.mo",
+    "src/zope/i18n/tests/sr-default.mo",
+    "src/zope/i18n/tests/sr@Cyrl-default.mo",
+    "src/zope/i18n/tests/sr@Latn-default.mo",
     ]

CHANGES.rst

@@ -5,6 +5,12 @@
 4.7.1 (unreleased)
 ==================
 
+- Support character sets.
+  Example: ``sr@Latn`` and ``sr@Cyrl`` will be added to language ``sr`` (Serbian).
+  See https://github.com/collective/plone.app.locales/issues/326
+  You can choose which one to use by setting either ``sr@Latn`` or ``sr@Cyrl``
+  in environment variable ``zope_i18n_allowed_languages``.
+
 - Support and test Python 3.8 and 3.9.
   Full supported list is now: 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, PyPy, PyPy3.
 

setup.cfg

@@ -21,3 +21,6 @@ ignore-bad-ideas =
     src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo
     src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo
     src/zope/i18n/tests/pl-default.mo
+    src/zope/i18n/tests/sr-default.mo
+    src/zope/i18n/tests/sr@Cyrl-default.mo
+    src/zope/i18n/tests/sr@Latn-default.mo

src/zope/i18n/tests/sr-default.po

@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in Serbian Standard!"

src/zope/i18n/tests/sr@Cyrl-default.po

@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in српски!"

src/zope/i18n/tests/sr@Latn-default.po

@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in Serbian Latin!"

src/zope/i18n/tests/test_translationdomain.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 ##############################################################################
 #
 # Copyright (c) 2001-2008 Zope Foundation and Contributors.
@@ -226,3 +227,73 @@ def test_releoadCatalogs(self):
 
         with self.assertRaises(KeyError):
             self._domain.reloadCatalogs(('dne',))
+
+    def test_character_sets(self):
+        """Test two character sets for the same language.
+
+        Serbian can be written in Latin or Cyrillic,
+        where Latin is currently most used.
+        Interestingly, every Latin character can actually be mapped to
+        one Cyrillic character, and the other way around,
+        so you could write a script to turn one po-file into the other.
+
+        But most practical is to have two locales with names
+        sr@Latn and sr@Cyrl.  These are then two character sets
+        for the same language.  So they should end up together
+        under the same language in a translation domain.
+
+        The best way for an integrator to choose which one to use,
+        is to add an environment variable 'zope_i18n_allowed_languages'
+        and let this contain either sr@Latn or sr@Cyrl.
+
+        See https://github.com/collective/plone.app.locales/issues/326
+        """
+        standard_file = os.path.join(testdir, 'sr-default.mo')
+        latin_file = os.path.join(testdir, 'sr@Latn-default.mo')
+        cyrillic_file = os.path.join(testdir, 'sr@Cyrl-default.mo')
+        standard_catalog = GettextMessageCatalog('sr', 'char', standard_file)
+        latin_catalog = GettextMessageCatalog('sr@Latn', 'char', latin_file)
+        cyrillic_catalog = GettextMessageCatalog(
+            'sr@Cyrl', 'char', cyrillic_file
+        )
+
+        # Test the standard file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(standard_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Standard!",
+        )
+
+        # Test the Latin file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(latin_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Latin!",
+        )
+        # Note that sr@Latn is not recognizes as language id.
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr@Latn'),
+            u"short_greeting",
+        )
+
+        # Test the Cyrillic file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(cyrillic_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in српски!",
+        )
+
+        # When I have all three locales, this is the order that
+        # os.listdir gives them in:
+        domain = TranslationDomain('char')
+        domain.addCatalog(latin_catalog)
+        domain.addCatalog(cyrillic_catalog)
+        domain.addCatalog(standard_catalog)
+        # The Latin one is first, so it wins.
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Latin!",
+        )

src/zope/i18n/translationdomain.py

@@ -50,6 +50,11 @@ def __init__(self, domain, fallbacks=None):
 
     def _registerMessageCatalog(self, language, catalog_name):
         key = language
+        if "@" in key:
+            # sr@Latn and sr@Cyrl are two character set variants of
+            # the same Serbian language.
+            # See https://github.com/collective/plone.app.locales/issues/326
+            key = key.split("@")[0]
         mc = self._catalogs.setdefault(key, [])
         mc.append(catalog_name)