From 9a05f81e35694e589081d8faaa32a74953c2a853 Mon Sep 17 00:00:00 2001 From: Maurits van Rees Date: Thu, 2 Sep 2021 15:28:05 +0200 Subject: [PATCH] Support character sets. sr@Latn and sr@Cyrl will be added to language sr. See https://github.com/collective/plone.app.locales/issues/326 --- .meta.toml | 3 + CHANGES.rst | 6 ++ setup.cfg | 3 + src/zope/i18n/tests/sr-default.mo | Bin 0 -> 328 bytes src/zope/i18n/tests/sr-default.po | 12 +++ src/zope/i18n/tests/sr@Cyrl-default.mo | Bin 0 -> 324 bytes src/zope/i18n/tests/sr@Cyrl-default.po | 12 +++ src/zope/i18n/tests/sr@Latn-default.mo | Bin 0 -> 325 bytes src/zope/i18n/tests/sr@Latn-default.po | 12 +++ src/zope/i18n/tests/test_translationdomain.py | 71 ++++++++++++++++++ src/zope/i18n/translationdomain.py | 5 ++ 11 files changed, 124 insertions(+) create mode 100644 src/zope/i18n/tests/sr-default.mo create mode 100644 src/zope/i18n/tests/sr-default.po create mode 100644 src/zope/i18n/tests/sr@Cyrl-default.mo create mode 100644 src/zope/i18n/tests/sr@Cyrl-default.po create mode 100644 src/zope/i18n/tests/sr@Latn-default.mo create mode 100644 src/zope/i18n/tests/sr@Latn-default.po diff --git a/.meta.toml b/.meta.toml index 877be7c..53d84cb 100644 --- a/.meta.toml +++ b/.meta.toml @@ -41,4 +41,7 @@ ignore-bad-ideas = [ "src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo", "src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo", "src/zope/i18n/tests/pl-default.mo", + "src/zope/i18n/tests/sr-default.mo", + "src/zope/i18n/tests/sr@Cyrl-default.mo", + "src/zope/i18n/tests/sr@Latn-default.mo", ] diff --git a/CHANGES.rst b/CHANGES.rst index c554ca2..2dad4eb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,12 @@ 4.7.1 (unreleased) ================== +- Support character sets. + Example: ``sr@Latn`` and ``sr@Cyrl`` will be added to language ``sr`` (Serbian). + See https://github.com/collective/plone.app.locales/issues/326 + You can choose which one to use by setting either ``sr@Latn`` or ``sr@Cyrl`` + in environment variable ``zope_i18n_allowed_languages``. + - Support and test Python 3.8 and 3.9. Full supported list is now: 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, PyPy, PyPy3. diff --git a/setup.cfg b/setup.cfg index 43050e9..e159e42 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,3 +21,6 @@ ignore-bad-ideas = src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo src/zope/i18n/tests/pl-default.mo + src/zope/i18n/tests/sr-default.mo + src/zope/i18n/tests/sr@Cyrl-default.mo + src/zope/i18n/tests/sr@Latn-default.mo diff --git a/src/zope/i18n/tests/sr-default.mo b/src/zope/i18n/tests/sr-default.mo new file mode 100644 index 0000000000000000000000000000000000000000..28aba7673bc6fc1f493602b213d2105a4f4f4e0d GIT binary patch literal 328 zcmYL@%}T>S5XYl<%E5z2@i4u+=0k*n4$i^+BIr%qiSWQ~> zgcOW-1w8u>;Ub>jseAk|eNNqI#scy*&yw^qN%MHB1FHpT7p!Ja|3l}{c*kVwwd6uP zo6JVLwb@x3UrQOCOn!B;kMWa}&T8XIXjYm6a`m_vtKqJ%zwnBrMy~O#e7JyNZ8)CW sHEFBHp58+NuG@s6%1hX~m4iVASs6#SXsrk1-~p*NT9mbGHMN824_2OD4*&oF literal 0 HcmV?d00001 diff --git a/src/zope/i18n/tests/sr-default.po b/src/zope/i18n/tests/sr-default.po new file mode 100644 index 0000000..c8366f2 --- /dev/null +++ b/src/zope/i18n/tests/sr-default.po @@ -0,0 +1,12 @@ +msgid "" +msgstr "" +"Project-Id-Version: Zope 5\n" +"PO-Revision-Date: 2021/09/02\n" +"Last-Translator: Zope 3 contributors\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" + +msgid "short_greeting" +msgstr "Hello in Serbian Standard!" diff --git a/src/zope/i18n/tests/sr@Cyrl-default.mo b/src/zope/i18n/tests/sr@Cyrl-default.mo new file mode 100644 index 0000000000000000000000000000000000000000..fef80124d58c28106ae098b71d4a7bce0240c956 GIT binary patch literal 324 zcmYL@Jx;?g6o3O_^1#5r#PGUf@`I2{LMCWMrB+0!Ft8vueY8f#j%*jvjUIx#AQrBW zTmW$tURtC+>3jcwKL>js0m&|LL>#vP$;HO3?LPTA@kKl@L(#0GgjR2TYQymj$a9nDI`%6htYW$C2T01=VL3hQ^FhjKXeME z(ca3*f^v?H2BY3~ZF~~3ODdzL$z?_RJihs`QbOts%DJ!(eSSCY^X|5c3o8`w8~fr?(A7G0d6L1#%{6p#h%**kqf!Q>hGx~g*Wb;m{;EGa!5{m| BV2l6& literal 0 HcmV?d00001 diff --git a/src/zope/i18n/tests/sr@Cyrl-default.po b/src/zope/i18n/tests/sr@Cyrl-default.po new file mode 100644 index 0000000..7ffcaf7 --- /dev/null +++ b/src/zope/i18n/tests/sr@Cyrl-default.po @@ -0,0 +1,12 @@ +msgid "" +msgstr "" +"Project-Id-Version: Zope 5\n" +"PO-Revision-Date: 2021/09/02\n" +"Last-Translator: Zope 3 contributors\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" + +msgid "short_greeting" +msgstr "Hello in српски!" diff --git a/src/zope/i18n/tests/sr@Latn-default.mo b/src/zope/i18n/tests/sr@Latn-default.mo new file mode 100644 index 0000000000000000000000000000000000000000..51173d720e8dfe4638ed45bf26fc9b3ec52ab481 GIT binary patch literal 325 zcmYL^%}&EG495%NlmiDYTsU}lLO&4FXz2+yF{u;P7#z5uv@=^!NGc~o@F2V%;!$w7 zNwf6x-*#dr{yjPRiP#RAGv>Tc*lxD6(PNHF=8vhEQ$`-J^D9T;bxDXp+htUfd&f4& z$wEFOdF^ZtFRnwl67_?e;iv9v@=gWJAxp9}POjr56P5Bo&PiEsRB*Iwx`fu*K)P9T z&5P+|I@*s-FA{Of$7s3pyW@EQze3y@rELN2N|DD>@K70S5BwW(~ literal 0 HcmV?d00001 diff --git a/src/zope/i18n/tests/sr@Latn-default.po b/src/zope/i18n/tests/sr@Latn-default.po new file mode 100644 index 0000000..65b1c97 --- /dev/null +++ b/src/zope/i18n/tests/sr@Latn-default.po @@ -0,0 +1,12 @@ +msgid "" +msgstr "" +"Project-Id-Version: Zope 5\n" +"PO-Revision-Date: 2021/09/02\n" +"Last-Translator: Zope 3 contributors\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" + +msgid "short_greeting" +msgstr "Hello in Serbian Latin!" diff --git a/src/zope/i18n/tests/test_translationdomain.py b/src/zope/i18n/tests/test_translationdomain.py index 4687107..2018427 100644 --- a/src/zope/i18n/tests/test_translationdomain.py +++ b/src/zope/i18n/tests/test_translationdomain.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- ############################################################################## # # Copyright (c) 2001-2008 Zope Foundation and Contributors. @@ -226,3 +227,73 @@ def test_releoadCatalogs(self): with self.assertRaises(KeyError): self._domain.reloadCatalogs(('dne',)) + + def test_character_sets(self): + """Test two character sets for the same language. + + Serbian can be written in Latin or Cyrillic, + where Latin is currently most used. + Interestingly, every Latin character can actually be mapped to + one Cyrillic character, and the other way around, + so you could write a script to turn one po-file into the other. + + But most practical is to have two locales with names + sr@Latn and sr@Cyrl. These are then two character sets + for the same language. So they should end up together + under the same language in a translation domain. + + The best way for an integrator to choose which one to use, + is to add an environment variable 'zope_i18n_allowed_languages' + and let this contain either sr@Latn or sr@Cyrl. + + See https://github.com/collective/plone.app.locales/issues/326 + """ + standard_file = os.path.join(testdir, 'sr-default.mo') + latin_file = os.path.join(testdir, 'sr@Latn-default.mo') + cyrillic_file = os.path.join(testdir, 'sr@Cyrl-default.mo') + standard_catalog = GettextMessageCatalog('sr', 'char', standard_file) + latin_catalog = GettextMessageCatalog('sr@Latn', 'char', latin_file) + cyrillic_catalog = GettextMessageCatalog( + 'sr@Cyrl', 'char', cyrillic_file + ) + + # Test the standard file. + domain = TranslationDomain('char') + domain.addCatalog(standard_catalog) + self.assertEqual( + domain.translate('short_greeting', target_language='sr'), + u"Hello in Serbian Standard!", + ) + + # Test the Latin file. + domain = TranslationDomain('char') + domain.addCatalog(latin_catalog) + self.assertEqual( + domain.translate('short_greeting', target_language='sr'), + u"Hello in Serbian Latin!", + ) + # Note that sr@Latn is not recognizes as language id. + self.assertEqual( + domain.translate('short_greeting', target_language='sr@Latn'), + u"short_greeting", + ) + + # Test the Cyrillic file. + domain = TranslationDomain('char') + domain.addCatalog(cyrillic_catalog) + self.assertEqual( + domain.translate('short_greeting', target_language='sr'), + u"Hello in српски!", + ) + + # When I have all three locales, this is the order that + # os.listdir gives them in: + domain = TranslationDomain('char') + domain.addCatalog(latin_catalog) + domain.addCatalog(cyrillic_catalog) + domain.addCatalog(standard_catalog) + # The Latin one is first, so it wins. + self.assertEqual( + domain.translate('short_greeting', target_language='sr'), + u"Hello in Serbian Latin!", + ) diff --git a/src/zope/i18n/translationdomain.py b/src/zope/i18n/translationdomain.py index 9915deb..70c1cc0 100644 --- a/src/zope/i18n/translationdomain.py +++ b/src/zope/i18n/translationdomain.py @@ -50,6 +50,11 @@ def __init__(self, domain, fallbacks=None): def _registerMessageCatalog(self, language, catalog_name): key = language + if "@" in key: + # sr@Latn and sr@Cyrl are two character set variants of + # the same Serbian language. + # See https://github.com/collective/plone.app.locales/issues/326 + key = key.split("@")[0] mc = self._catalogs.setdefault(key, []) mc.append(catalog_name)