Use moz.l10n for serialization

pontoon/checks/libraries/compare_locales.py

@@ -9,8 +9,6 @@
 from compare_locales.parser.properties import PropertiesEntityMixin
 from compare_locales.paths import File
 
-from pontoon.sync.formats.utils import escape_apostrophes
-
 
 CommentEntity = namedtuple("Comment", ("all",))
 
@@ -141,8 +139,8 @@ def cast_to_compare_locales(resource_ext, entity, string):
             </resources>
         """.format(
             key=entity.key,
-            original=escape_apostrophes(entity.string),
-            translation=escape_apostrophes(string),
+            original=entity.string.replace("'", "\\'"),
+            translation=string.replace("'", "\\'"),
         )
 
         parser.readUnicode(content)

pontoon/checks/libraries/pontoon_db.py

@@ -3,8 +3,6 @@
 from fluent.syntax import FluentParser, ast
 from fluent.syntax.visitor import Visitor
 
-from pontoon.sync.formats.ftl import localizable_entries
-
 
 parser = FluentParser()
 
@@ -64,7 +62,7 @@ def run_checks(entity, original, string):
             checks["pErrors"].append(translation_ast.annotations[0].message)
 
         # Not a localizable entry
-        elif not isinstance(translation_ast, localizable_entries):
+        elif not isinstance(translation_ast, (ast.Message, ast.Term)):
             checks["pErrors"].append(
                 "Translation needs to be a valid localizable entry"
             )

pontoon/sync/core/entities.py

@@ -13,9 +13,8 @@
 from pontoon.base.models import Entity, Locale, Project, Resource, TranslatedResource
 from pontoon.base.models.entity import get_word_count
 from pontoon.sync.core.checkout import Checkout
-from pontoon.sync.formats import parse
-from pontoon.sync.formats.exceptions import ParseError
-from pontoon.sync.formats.silme import SilmeEntity, SilmeResource  # Approximate types
+from pontoon.sync.formats import parse_translations
+from pontoon.sync.formats.common import ParseError, VCSTranslation
 
 
 log = logging.getLogger(__name__)
@@ -35,39 +34,34 @@ def sync_entities_from_repo(
         return 0, set(), set()
     log.info(f"[{project.slug}] Syncing entities from repo...")
     # db_path -> parsed_resource
-    updates: dict[str, SilmeResource | None] = {}
+    updates: dict[str, list[VCSTranslation]] = {}
     source_paths = set(paths.ref_paths)
-    source_locale = Locale.objects.get(code="en-US")
     for co_path in checkout.changed:
         path = join(checkout.path, co_path)
         if path in source_paths and exists(path):
             db_path = get_db_path(paths, path)
             try:
-                res = parse(path, locale=source_locale)
+                translations = parse_translations(path)
             except ParseError as error:
                 log.error(
                     f"[{project.slug}:{db_path}] Skipping resource with parse error: {error}"
                 )
-                res = None
+                translations = []
             except ValueError as error:
                 if str(error).startswith("Translation format"):
                     log.warning(
                         f"[{project.slug}:{db_path}] Skipping resource with unsupported format"
                     )
-                    res = None
+                    translations = []
                 else:
                     raise error
-            updates[db_path] = res
+            updates[db_path] = translations
 
     with transaction.atomic():
         renamed_paths = rename_resources(project, paths, checkout)
         removed_paths = remove_resources(project, paths, checkout)
-        old_res_added_ent_count, changed_paths = update_resources(
-            project, locale_map, paths, updates, now
-        )
-        new_res_added_ent_count, _ = add_resources(
-            project, locale_map, paths, updates, changed_paths, now
-        )
+        old_res_added_ent_count, changed_paths = update_resources(project, updates, now)
+        new_res_added_ent_count, _ = add_resources(project, updates, changed_paths, now)
         update_translated_resources(project, locale_map, paths)
 
     return (
@@ -122,9 +116,7 @@ def remove_resources(
 
 def update_resources(
     project: Project,
-    locale_map: dict[str, Locale],
-    paths: L10nConfigPaths | L10nDiscoverPaths,
-    updates: dict[str, SilmeResource | None],
+    updates: dict[str, list[VCSTranslation]],
     now: datetime,
 ) -> tuple[int, set[str]]:
     changed_resources = (
@@ -147,7 +139,7 @@ def update_resources(
         for path, entity in (
             (cr.path, entity_from_source(cr, now, 0, tx))
             for cr in changed_resources
-            for tx in updates[cr.path].translations
+            for tx in updates[cr.path]
         )
     }
 
@@ -202,16 +194,14 @@ def update_resources(
 
 def add_resources(
     project: Project,
-    locale_map: dict[str, Locale],
-    paths: L10nConfigPaths | L10nDiscoverPaths,
-    updates: dict[str, SilmeResource | None],
+    updates: dict[str, list[VCSTranslation]],
     changed_paths: set[str],
     now: datetime,
 ) -> tuple[int, set[str]]:
     added_resources = [
         Resource(project=project, path=db_path, format=get_path_format(db_path))
-        for db_path, res in updates.items()
-        if res is not None and db_path not in changed_paths
+        for db_path, translations in updates.items()
+        if translations and db_path not in changed_paths
     ]
     if not added_resources:
         return 0, set()
@@ -226,7 +216,7 @@ def add_resources(
         (
             entity_from_source(resource, now, idx, tx)
             for resource in added_resources
-            for idx, tx in enumerate(updates[resource.path].translations)
+            for idx, tx in enumerate(updates[resource.path])
         )
     )
 
@@ -291,16 +281,16 @@ def is_translated_resource(
     if resource.format in BILINGUAL_FORMATS:
         # For bilingual formats, only create TranslatedResource
         # if the resource exists for the locale.
-        target = paths.target(resource.path)  # , locale_code)
+        target, _ = paths.target(resource.path)
         if target is None:
             return False
-        target_path = paths.format_target_path(target[0], locale.code)
+        target_path = paths.format_target_path(target, locale.code)
         return isfile(target_path)
     return True
 
 
 def entity_from_source(
-    resource: Resource, now: datetime, idx: int, tx: SilmeEntity
+    resource: Resource, now: datetime, idx: int, tx: VCSTranslation
 ) -> Entity:
     comments = getattr(tx, "comments", None)
     group_comments = getattr(tx, "group_comments", None)

pontoon/sync/core/translations_from_repo.py

@@ -4,11 +4,10 @@
 from collections.abc import Iterable, Sized
 from datetime import datetime
 from os.path import join, relpath, splitext
-from typing import cast
 
 from fluent.syntax import FluentParser
+from moz.l10n.formats import bilingual_extensions, l10n_extensions
 from moz.l10n.paths import L10nConfigPaths, L10nDiscoverPaths, parse_android_locale
-from moz.l10n.resource import bilingual_extensions, l10n_extensions
 
 from django.core.paginator import Paginator
 from django.db import transaction
@@ -31,8 +30,7 @@
 from pontoon.checks.utils import bulk_run_checks
 from pontoon.sync.core.checkout import Checkout, Checkouts
 from pontoon.sync.core.paths import UploadPaths
-from pontoon.sync.formats import parse
-from pontoon.sync.vcs.translation import VCSTranslation
+from pontoon.sync.formats import parse_translations
 
 
 log = logging.getLogger(__name__)
@@ -154,11 +152,7 @@ def find_db_updates(
                 db_path = relpath(ref_path, paths.ref_root)
                 lc_scope = f"[{project.slug}:{db_path}, {locale.code}]"
                 try:
-                    res = parse(
-                        target_path,
-                        None if isinstance(paths, UploadPaths) else ref_path,
-                        locale,
-                    )
+                    repo_translations = parse_translations(target_path)
                 except Exception as error:
                     log.error(f"{lc_scope} Skipping resource with parse error: {error}")
                     continue
@@ -168,7 +162,7 @@ def find_db_updates(
                 translated_resources[db_path].add(locale.pk)
                 translations.update(
                     ((db_path, tx.key, locale.pk), (tx.strings, tx.fuzzy))
-                    for tx in cast(list[VCSTranslation], res.translations)
+                    for tx in repo_translations
                     if tx.strings
                 )
         elif splitext(target_path)[1] in l10n_extensions and not isinstance(
@@ -385,7 +379,6 @@ def update_db_translations(
         # Add new approved translations for the remainder
         for (entity_id, locale_id), (strings, fuzzy) in repo_translations.items():
             for plural_form, string in strings.items():
-                # Note: no tx.entity.resource, which would be required by tx.save()
                 tx = Translation(
                     entity_id=entity_id,
                     locale_id=locale_id,