Refactor sync

docker/compile_requirements.sh

@@ -5,7 +5,7 @@
 
 export CUSTOM_COMPILE_COMMAND="./docker/compile_requirements.sh"
 
-uv pip compile --generate-hashes $@ requirements/default.in -o requirements/default.txt
-uv pip compile --generate-hashes $@ requirements/dev.in -o requirements/dev.txt
-uv pip compile --generate-hashes $@ requirements/lint.in -o requirements/lint.txt
-uv pip compile --generate-hashes $@ requirements/test.in -o requirements/test.txt
+uv pip compile --generate-hashes --no-strip-extras $@ requirements/default.in -o requirements/default.txt
+uv pip compile --generate-hashes --no-strip-extras $@ requirements/dev.in -o requirements/dev.txt
+uv pip compile --generate-hashes --no-strip-extras $@ requirements/lint.in -o requirements/lint.txt
+uv pip compile --generate-hashes --no-strip-extras $@ requirements/test.in -o requirements/test.txt

pontoon/administration/views.py

@@ -32,7 +32,7 @@
 from pontoon.base.utils import require_AJAX
 from pontoon.pretranslation.tasks import pretranslate
 from pontoon.sync.models import SyncLog
-from pontoon.sync.tasks import sync_project
+from pontoon.sync.tasks import sync_project_task
 
 
 log = logging.getLogger(__name__)
@@ -542,9 +542,9 @@ def manually_sync_project(request, slug):
             "Forbidden: You don't have permission for syncing projects"
         )
 
-    sync_log = SyncLog.objects.create(start_time=timezone.now())
     project = Project.objects.get(slug=slug)
-    sync_project.delay(project.pk, sync_log.pk)
+    sync_log = SyncLog.objects.create(start_time=timezone.now())
+    sync_project_task.delay(project.pk, sync_log.pk)
 
     return HttpResponse("ok")
 

pontoon/base/__init__.py

@@ -1,13 +0,0 @@
-MOZILLA_REPOS = (
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-for-android-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/thunderbird-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/lightning-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/seamonkey-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-for-android-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/thunderbird-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/lightning-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/seamonkey-central/",
-    "git@gitlab.com:seamonkey-project/seamonkey-central-l10n.git",
-)

pontoon/base/migrations/0018_populate_entity_context.py

@@ -3,22 +3,20 @@
 from django.db import migrations
 from django.db.models import F, Func, TextField, Value
 
-from pontoon.sync import KEY_SEPARATOR
-
 
 def add_entity_context(apps, schema_editor):
     Entity = apps.get_model("base", "Entity")
 
     split_key_po = Func(
         F("key"),
-        Value(KEY_SEPARATOR),
+        Value("\x04"),
         Value(1),
         function="split_part",
         output_field=TextField(),
     )
     split_key_xliff = Func(
         F("key"),
-        Value(KEY_SEPARATOR),
+        Value("\x04"),
         Value(2),
         function="split_part",
         output_field=TextField(),

pontoon/base/models/changed_entity_locale.py

@@ -1,15 +1,21 @@
+from typing import TYPE_CHECKING
+
 from django.db import models
 from django.utils import timezone
 
 
+if TYPE_CHECKING:
+    from pontoon.base.models import Entity, Locale
+
+
 class ChangedEntityLocale(models.Model):
     """
     ManyToMany model for storing what locales have changed translations for a
     specific entity since the last sync.
     """
 
-    entity = models.ForeignKey("Entity", models.CASCADE)
-    locale = models.ForeignKey("Locale", models.CASCADE)
+    entity: models.ForeignKey["Entity"] = models.ForeignKey("Entity", models.CASCADE)
+    locale: models.ForeignKey["Locale"] = models.ForeignKey("Locale", models.CASCADE)
     when = models.DateTimeField(default=timezone.now)
 
     class Meta:

pontoon/base/models/entity.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterable
 from functools import reduce
 from operator import ior
 from re import escape, findall, match
@@ -14,7 +15,6 @@
 from pontoon.base.models.project import Project
 from pontoon.base.models.project_locale import ProjectLocale
 from pontoon.base.models.resource import Resource
-from pontoon.sync import KEY_SEPARATOR
 
 
 def get_word_count(string):
@@ -532,18 +532,6 @@ class Meta:
             models.Index(fields=["resource", "obsolete", "string_plural"]),
         ]
 
-    @property
-    def cleaned_key(self):
-        """
-        Get cleaned key, without the source string and Translate Toolkit
-        separator.
-        """
-        key = self.key.split(KEY_SEPARATOR)[0]
-        if key == self.string:
-            key = ""
-
-        return key
-
     def __str__(self):
         return self.string
 
@@ -942,7 +930,9 @@ def map_entities(
     ):
         entities_array = []
 
-        entities = entities.prefetch_entities_data(locale, preferred_source_locale)
+        entities: Iterable[Entity] = entities.prefetch_entities_data(
+            locale, preferred_source_locale
+        )
 
         # If requested entity not in the current page
         if requested_entity and requested_entity not in [e.pk for e in entities]:
@@ -971,13 +961,18 @@ def map_entities(
                 if original_plural != "":
                     original_plural = entity.alternative_originals[-1].string
 
+            key_separator = "\x04"
+            cleaned_key = entity.key.split(key_separator)[0]
+            if cleaned_key == entity.string:
+                cleaned_key = ""
+
             entities_array.append(
                 {
                     "pk": entity.pk,
                     "original": original,
                     "original_plural": original_plural,
                     "machinery_original": entity.string,
-                    "key": entity.cleaned_key,
+                    "key": cleaned_key,
                     "context": entity.context,
                     "path": entity.resource.path,
                     "project": entity.resource.project.serialize(),

pontoon/base/models/project.py

@@ -1,20 +1,25 @@
 from collections import defaultdict
 from os.path import basename, join, normpath
+from typing import TYPE_CHECKING
 from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db import models
 from django.db.models import Prefetch
+from django.db.models.manager import BaseManager
 from django.utils import timezone
 from django.utils.functional import cached_property
 
-from pontoon.base import utils
 from pontoon.base.models.aggregated_stats import AggregatedStats
 from pontoon.base.models.changed_entity_locale import ChangedEntityLocale
 from pontoon.base.models.locale import Locale
 
 
+if TYPE_CHECKING:
+    from pontoon.base.models import Resource
+
+
 class Priority(models.IntegerChoices):
     LOWEST = 1, "Lowest"
     LOW = 2, "Low"
@@ -103,6 +108,8 @@ class Project(AggregatedStats):
     slug = models.SlugField(unique=True)
     locales = models.ManyToManyField(Locale, through="ProjectLocale")
 
+    resources: BaseManager["Resource"]
+
     class DataSource(models.TextChoices):
         REPOSITORY = "repository", "Repository"
         DATABASE = "database", "Database"
@@ -325,14 +332,14 @@ def repository_for_path(self, path):
         Return the repository instance whose checkout contains the given
         path. If no matching repo is found, raise a ValueError.
         """
-        repo = utils.first(
-            self.repositories.all(), lambda r: path.startswith(r.checkout_path)
-        )
-
-        if repo is None:
+        try:
+            return next(
+                repo
+                for repo in self.repositories.all()
+                if path.startswith(repo.checkout_path)
+            )
+        except StopIteration:
             raise ValueError(f"Could not find repo matching path {path}.")
-        else:
-            return repo
 
     @property
     def has_multi_locale_repositories(self):
@@ -352,10 +359,7 @@ def source_repository(self):
         Returns an instance of repository which contains the path to source files.
         """
         if not self.has_single_repo:
-            from pontoon.sync.vcs.project import VCSProject
-
-            source_directories = VCSProject.SOURCE_DIR_SCORES.keys()
-
+            source_directories = {"templates", "en-US", "en-us", "en_US", "en_us", "en"}
             for repo in self.repositories.all():
                 last_directory = basename(normpath(urlparse(repo.url).path))
                 if repo.source_repo or last_directory in source_directories:

pontoon/base/models/repository.py

@@ -1,8 +1,7 @@
 import logging
 import re
 
-from os import sep
-from os.path import join
+from os.path import join, normpath
 from urllib.parse import urlparse
 
 from jsonfield import JSONField
@@ -115,21 +114,20 @@ def checkout_path(self):
         Path where the checkout for this repo is located. Does not
         include a trailing path separator.
         """
-        path_components = [self.project.checkout_path]
 
         # Include path components from the URL in case it has locale
         # information, like https://hg.mozilla.org/gaia-l10n/fr/.
         # No worry about overlap between repos, any overlap of locale
         # directories is an error already.
-        path_components += urlparse(self.url).path.split("/")
+        path_components = [
+            self.project.checkout_path,
+            *urlparse(self.url).path.split("/"),
+        ]
         if self.multi_locale:
             path_components = [c for c in path_components if c != "{locale_code}"]
 
-        if self.source_repo:
-            path_components.append("templates")
-
-        # Remove trailing separator for consistency.
-        return join(*path_components).rstrip(sep)
+        # Normalize path for consistency.
+        return normpath(join(*path_components))
 
     @cached_property
     def api_config(self):
@@ -234,7 +232,7 @@ def pull(self, locales=None):
 
             return current_revisions
 
-    def commit(self, message, author, path):
+    def commit(self, message: str, author: str, path: str):
         """Commit changes to VCS."""
         # For multi-locale repos, figure out which sub-repo corresponds
         # to the given path.

pontoon/base/models/resource.py

@@ -1,12 +1,9 @@
-from os.path import splitext
-
 from django.db import models
 from django.utils import timezone
 
 
 class ResourceQuerySet(models.QuerySet):
-    def asymmetric(self):
-        return self.filter(format__in=Resource.ASYMMETRIC_FORMATS)
+    pass
 
 
 class Resource(models.Model):
@@ -46,19 +43,6 @@ class Format(models.TextChoices):
 
     deadline = models.DateField(blank=True, null=True)
 
-    SOURCE_EXTENSIONS = ["pot"]  # Extensions of source-only formats.
-    ALLOWED_EXTENSIONS = Format.values + SOURCE_EXTENSIONS
-
-    ASYMMETRIC_FORMATS = {
-        Format.DTD,
-        Format.FTL,
-        Format.INC,
-        Format.INI,
-        Format.JSON,
-        Format.PROPERTIES,
-        Format.XML,
-    }
-
     # Formats that allow empty translations
     EMPTY_TRANSLATION_FORMATS = {
         Format.DTD,
@@ -72,11 +56,6 @@ class Format(models.TextChoices):
     class Meta:
         unique_together = (("project", "path"),)
 
-    @property
-    def is_asymmetric(self):
-        """Return True if this resource is in an asymmetric format."""
-        return self.format in self.ASYMMETRIC_FORMATS
-
     @property
     def allows_empty_translations(self):
         """Return True if this resource allows empty translations."""
@@ -91,16 +70,3 @@ def __str__(self):
             project=self.project.name,
             resource=self.path,
         )
-
-    @classmethod
-    def get_path_format(self, path):
-        filename, extension = splitext(path)
-        path_format = extension[1:].lower()
-
-        # Special case: pot files are considered the po format
-        if path_format == "pot":
-            return "po"
-        elif path_format == "xlf":
-            return "xliff"
-        else:
-            return path_format

pontoon/base/tests/__init__.py

@@ -90,7 +90,9 @@ def locales(self, create, extracted, **kwargs):
 
         if extracted:
             for locale in extracted:
-                ProjectLocaleFactory.create(project=self, locale=locale)
+                ProjectLocaleFactory.create(
+                    project=self, locale=locale, total_strings=self.total_strings
+                )
 
     @factory.post_generation
     def repositories(self, create, extracted, **kwargs):

pontoon/base/tests/models/test_entity.py

@@ -1,7 +1,6 @@
 import pytest
 
 from pontoon.base.models import ChangedEntityLocale, Entity, Project
-from pontoon.sync import KEY_SEPARATOR
 from pontoon.test.factories import (
     EntityFactory,
     ResourceFactory,
@@ -40,7 +39,7 @@ def entity_test_models(translation_a, locale_b):
     entity_b = EntityFactory(
         resource=resourceX,
         string="entity_b",
-        key="Key%sentity_b" % KEY_SEPARATOR,
+        key="Key\x04entity_b",
         order=0,
     )
     translation_a_pl = TranslationFactory(

pontoon/base/tests/models/test_repository.py

@@ -43,15 +43,14 @@ def test_repo_checkout_path_multi_locale(settings, repo_git):
 @pytest.mark.django_db
 def test_repo_checkout_path_source_repo(settings, repo_git):
     """
-    The checkout_path for a source repo should end with a templates
+    The checkout_path for a source repo should not end with a templates
     directory.
     """
     repo_git.source_repo = True
     repo_git.url = "https://example.com/path/to/locale/"
     repo_git.save()
     assert repo_git.checkout_path == (
-        "%s/projects/%s/path/to/locale/templates"
-        % (settings.MEDIA_ROOT, repo_git.project.slug)
+        "%s/projects/%s/path/to/locale" % (settings.MEDIA_ROOT, repo_git.project.slug)
     )