WIP - firmware translations

WIP - refactor and extend font generation for non-ascii characters WIP - add czech characters mapping between UTF8 value and index WIP - regenerate font files with czech characters WIP - shorten czech button text, it was causing SHUTDOWN for some reason WIP - support UTF8 characters in fonts.c WIP - account for translation in tests WIP - small fixes WIP - fix last test WIP - support UTF8 also in Rust font operations WIP - add a script to find non-translated english strings in micropython code WIP - add a validator script for checking missing micropython translations WIP - translate remaining altcoins and other apps in core (fido, sdcard, TT layouts, ...) WIP - generate czech glyphs for TT fonts WIP - modify gen_font.py to account for negative bearing czech characters WIP - extend translation validation scripts, move them into core/tools WIP - translate TT layouts in Rust WIP - fix tests WIP - fix inverse coloring of nonprintable glyph WIP - add build and test pipelines for Czech language WIP - merge both JSON files together WIP - run new isort WIP - unify all the translation in Rust, expose to micropython TEMP - leave en_merged.json file, so it is accessible by translators with old link WIP - fixes WIP - add french characters and translation via Google Translator WIP - skip rustfmt in mako-created files WIP - revert all the font height changes causing false-positive UI diff WIP - fixes after rebase
trezor · Sep 12, 2023 · 8b2081e · 8b2081e
1 parent a30c59c
commit 8b2081e
Show file tree

Hide file tree

Showing 169 changed files with 13,803 additions and 1,707 deletions.
diff --git a/ci/build.yml b/ci/build.yml
@@ -297,6 +297,20 @@ core unix frozen debug build:
     untracked: true
     expire_in: 1 week
 
+core unix frozen debug build czech:
+  stage: build
+  <<: *gitlab_caching
+  needs: []
+  variables:
+    PYOPT: "0"
+    TREZOR_LANG: "cs"
+  script:
+    - $NIX_SHELL --run "poetry run make -C core build_unix_frozen"
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    untracked: true
+    expire_in: 1 week
+
 core unix frozen R debug build:
   stage: build
   <<: *gitlab_caching
@@ -312,6 +326,21 @@ core unix frozen R debug build:
     untracked: true
     expire_in: 10 weeks
 
+core unix frozen R debug build czech:
+  stage: build
+  <<: *gitlab_caching
+  needs: []
+  variables:
+    PYOPT: "0"
+    TREZOR_MODEL: "R"
+    TREZOR_LANG: "cs"
+  script:
+    - nix-shell --run "poetry run make -C core build_unix_frozen"
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    untracked: true
+    expire_in: 10 weeks
+
 core unix frozen R debug build arm:
   image: nixos/nix
   stage: build

diff --git a/ci/test.yml b/ci/test.yml
@@ -82,6 +82,37 @@ core device test:
     reports:
       junit: tests/junit.xml
 
+core device test czech:
+  stage: test
+  <<: *gitlab_caching
+  needs:
+    - core unix frozen debug build czech
+  variables:
+    TREZOR_PROFILING: 1  # so that we get coverage data
+    MULTICORE: 4  # more could interfere with other jobs
+  script:
+    - $NIX_SHELL --run "poetry run make -C core test_emu_ui_multicore | ts -s"
+  after_script:
+    - mv core/src/.coverage.* core  # there will be more coverage files (one per core)
+    - mv tests/ui_tests/reports/test/ test_ui_report
+    - $NIX_SHELL --run "poetry run python ci/prepare_ui_artifacts.py | ts -s"
+    - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - ci/ui_test_records/
+      - test_ui_report
+      - tests/ui_tests/screens/
+      - tests/ui_tests/fixtures.suggestion.json
+      - tests/ui_tests/fixtures.results.json
+      - tests/junit.xml
+      - tests/trezor.log
+      - core/.coverage.*
+    when: always
+    expire_in: 1 week
+    reports:
+      junit: tests/junit.xml
+
 core device R test:
   stage: test
   <<: *gitlab_caching
@@ -114,6 +145,38 @@ core device R test:
     reports:
       junit: tests/junit.xml
 
+core device R test czech:
+  stage: test
+  <<: *gitlab_caching
+  needs:
+    - core unix frozen R debug build czech
+  variables:
+    TREZOR_PROFILING: "1"
+    TREZOR_MODEL: "R"
+    MULTICORE: 4  # more could interfere with other jobs
+  script:
+    - $NIX_SHELL --run "poetry run make -C core test_emu_ui_multicore | ts -s"
+  after_script:
+    - mv core/src/.coverage.* core  # there will be more coverage files (one per core)
+    - mv tests/ui_tests/reports/test/ test_ui_report
+    - $NIX_SHELL --run "poetry run python ci/prepare_ui_artifacts.py | ts -s"
+    - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - ci/ui_test_records/
+      - test_ui_report
+      - tests/ui_tests/screens/
+      - tests/ui_tests/fixtures.suggestion.json
+      - tests/ui_tests/fixtures.results.json
+      - tests/junit.xml
+      - tests/trezor.log
+      - core/.coverage.*
+    when: always
+    expire_in: 1 week
+    reports:
+      junit: tests/junit.xml
+
 core device asan test:
   stage: test
   <<: *gitlab_caching

diff --git a/common/tools/cointool.py b/common/tools/cointool.py
@@ -113,6 +113,10 @@ def ascii_filter(s: str) -> str:
     return re.sub("[^ -\x7e]", "_", s)
 
 
+def utf8_str_filter(s: str) -> str:
+    return '"' + repr(s)[1:-1] + '"'
+
+
 def make_support_filter(
     support_info: SupportInfo,
 ) -> Callable[[str, Coins], Iterator[Coin]]:
@@ -123,6 +127,7 @@ def supported_on(device: str, coins: Coins) -> Iterator[Coin]:
 
 
 MAKO_FILTERS = {
+    "utf8_str": utf8_str_filter,
     "c_str": c_str_filter,
     "ascii": ascii_filter,
     "black_repr": black_repr_filter,

diff --git a/core/Makefile b/core/Makefile
@@ -28,6 +28,7 @@ TREZOR_MODEL ?= T
 TREZOR_MEMPERF ?= 0
 ADDRESS_SANITIZER ?= 0
 CMAKELISTS ?= 0
+TREZOR_LANG ?= en
 PYTEST_TIMEOUT ?= 400
 
 # OpenOCD interface default. Alternative: ftdi/olimex-arm-usb-tiny-h
@@ -199,24 +200,24 @@ build_reflash: ## build reflash firmware + reflash image
 
 build_firmware: templates build_cross ## build firmware with frozen modules
 	$(SCONS) CFLAGS="$(CFLAGS)" PRODUCTION="$(PRODUCTION)" \
-		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" \
+		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" TREZOR_LANG="$(TREZOR_LANG)" \
 		PYOPT="$(PYOPT)" BITCOIN_ONLY="$(BITCOIN_ONLY)" \
 		BOOTLOADER_QA="$(BOOTLOADER_QA)" BOOTLOADER_DEVEL="$(BOOTLOADER_DEVEL)" $(FIRMWARE_BUILD_DIR)/firmware.bin
 
 build_unix: templates ## build unix port
 	$(SCONS) CFLAGS="$(CFLAGS)" $(UNIX_BUILD_DIR)/trezor-emu-core $(UNIX_PORT_OPTS) \
-		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" \
+		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" TREZOR_LANG="$(TREZOR_LANG)" \
 		PYOPT="0" BITCOIN_ONLY="$(BITCOIN_ONLY)" TREZOR_EMULATOR_ASAN="$(ADDRESS_SANITIZER)"
 
 build_unix_frozen: templates build_cross ## build unix port with frozen modules
 	$(SCONS) CFLAGS="$(CFLAGS)" $(UNIX_BUILD_DIR)/trezor-emu-core $(UNIX_PORT_OPTS) \
-		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" \
+		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" TREZOR_LANG="$(TREZOR_LANG)" \
 		PYOPT="$(PYOPT)" BITCOIN_ONLY="$(BITCOIN_ONLY)" TREZOR_EMULATOR_ASAN="$(ADDRESS_SANITIZER)" \
 		TREZOR_MEMPERF="$(TREZOR_MEMPERF)" TREZOR_EMULATOR_FROZEN=1
 
 build_unix_debug: templates ## build unix port
 	$(SCONS) --max-drift=1 CFLAGS="$(CFLAGS)" $(UNIX_BUILD_DIR)/trezor-emu-core $(UNIX_PORT_OPTS) \
-		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" \
+		TREZOR_MODEL="$(TREZOR_MODEL)" CMAKELISTS="$(CMAKELISTS)" TREZOR_LANG="$(TREZOR_LANG)" \
 		BITCOIN_ONLY="$(BITCOIN_ONLY)" TREZOR_EMULATOR_ASAN=1 TREZOR_EMULATOR_DEBUGGABLE=1
 
 build_cross: ## build mpy-cross port

diff --git a/core/SConscript.firmware b/core/SConscript.firmware
@@ -10,6 +10,7 @@ BOOTLOADER_QA = ARGUMENTS.get('BOOTLOADER_QA', '0') == '1'
 BOOTLOADER_DEVEL = ARGUMENTS.get('BOOTLOADER_DEVEL', '0') == '1'
 EVERYTHING = BITCOIN_ONLY != '1'
 TREZOR_MODEL = ARGUMENTS.get('TREZOR_MODEL', 'T')
+TREZOR_LANG = ARGUMENTS.get('TREZOR_LANG', 'en')
 CMAKELISTS = int(ARGUMENTS.get('CMAKELISTS', 0))
 
 
@@ -195,6 +196,7 @@ SOURCE_MOD += [
 
 CPPDEFINES_MOD += [
     'TREZOR_UI2',
+    ('TREZOR_LANG_QSTR', 'MP_QSTR_{}'.format(TREZOR_LANG)),
 ]
 
 if TREZOR_MODEL not in ('1', ):
@@ -550,6 +552,8 @@ if FROZEN:
         ] if "sd_card" not in FEATURES_AVAILABLE else []
     ))
 
+    SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/translations/*.py'))
+
     SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/__init__.py'))
     SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/*.py',
         exclude=[
@@ -688,6 +692,11 @@ def cargo_build():
     if PYOPT == '0':
         features.append('ui_debug')
 
+    if TREZOR_LANG in ('en', 'cs', 'fr'):
+        features.append(f'lang_{TREZOR_LANG}')
+    else:
+        raise Exception(f'Unknown language: {TREZOR_LANG}')
+
     features.extend(FEATURES_AVAILABLE)
 
     cargo_opts = [

diff --git a/core/SConscript.unix b/core/SConscript.unix
@@ -7,6 +7,7 @@ import tools
 BITCOIN_ONLY = ARGUMENTS.get('BITCOIN_ONLY', '0')
 EVERYTHING = BITCOIN_ONLY != '1'
 TREZOR_MODEL = ARGUMENTS.get('TREZOR_MODEL', 'T')
+TREZOR_LANG = ARGUMENTS.get('TREZOR_LANG', 'en')
 DMA2D =  TREZOR_MODEL in ('T', )
 CMAKELISTS = int(ARGUMENTS.get('CMAKELISTS', 0))
 
@@ -215,6 +216,7 @@ elif TREZOR_MODEL in ('R', ):
 
 CPPDEFINES_MOD += [
     'TREZOR_UI2',
+    ('TREZOR_LANG_QSTR', 'MP_QSTR_{}'.format(TREZOR_LANG)),
 ]
 if TREZOR_MODEL not in ('1', ):
     CPPDEFINES_MOD += [
@@ -625,6 +627,8 @@ if FROZEN:
         ] if TREZOR_MODEL not in ('T',) else []
     ))
 
+    SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/translations/*.py'))
+
     SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/messages/__init__.py'))
     SOURCE_PY.extend(Glob(SOURCE_PY_DIR + 'trezor/enums/*.py',
         exclude=[
@@ -761,6 +765,11 @@ def cargo_build():
     if DMA2D:
         features.append('dma2d')
 
+    if TREZOR_LANG in ('en', 'cs', 'fr'):
+        features.append(f'lang_{TREZOR_LANG}')
+    else:
+        raise Exception(f'Unknown language: {TREZOR_LANG}')
+
     if TREZOR_MODEL in ('T',):
         features.append('touch')
         features.append('sd_card')

diff --git a/core/embed/extmod/modtrezorutils/modtrezorutils.c b/core/embed/extmod/modtrezorutils/modtrezorutils.c
@@ -264,6 +264,7 @@ STATIC mp_obj_str_t mod_trezorutils_revision_obj = {
 /// USE_SD_CARD: bool
 /// USE_BACKLIGHT: bool
 /// MODEL: str
+/// LANG: str
 /// INTERNAL_MODEL: str
 /// EMULATOR: bool
 /// BITCOIN_ONLY: bool
@@ -300,6 +301,8 @@ STATIC const mp_rom_map_elem_t mp_module_trezorutils_globals_table[] = {
     {MP_ROM_QSTR(MP_QSTR_USE_BACKLIGHT), mp_const_false},
 #endif
     {MP_ROM_QSTR(MP_QSTR_MODEL), MP_ROM_QSTR(MODEL_NAME_QSTR)},
+    // TREZOR_LANG_QSTR created in SConscript
+    {MP_ROM_QSTR(MP_QSTR_LANG), MP_ROM_QSTR(TREZOR_LANG_QSTR)},
     {MP_ROM_QSTR(MP_QSTR_INTERNAL_MODEL),
      MP_ROM_QSTR(MODEL_INTERNAL_NAME_QSTR)},
 #ifdef TREZOR_EMULATOR

diff --git a/core/embed/extmod/rustmods.c b/core/embed/extmod/rustmods.c
@@ -27,3 +27,7 @@ MP_REGISTER_MODULE(MP_QSTR_trezorui2, mp_module_trezorui2);
 #if MICROPY_PY_TREZORPROTO
 MP_REGISTER_MODULE(MP_QSTR_trezorproto, mp_module_trezorproto);
 #endif
+
+#if MICROPY_PY_TREZORTRANSLATE
+MP_REGISTER_MODULE(MP_QSTR_trezortranslate, mp_module_trezortranslate);
+#endif
diff --git a/core/embed/firmware/mpconfigport.h b/core/embed/firmware/mpconfigport.h
@@ -163,6 +163,7 @@
 #define MICROPY_PY_TREZORUI         (1)
 #define MICROPY_PY_TREZORUTILS      (1)
 #define MICROPY_PY_TREZORPROTO      (1)
+#define MICROPY_PY_TREZORTRANSLATE  (1)
 #define MICROPY_PY_TREZORUI2        (1)
 
 #ifdef SYSTEM_VIEW