diff --git a/package-lock.json b/package-lock.json
index 3eaf963056..b6d199a7f7 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2570,9 +2570,9 @@
"license": "MIT"
},
"node_modules/@mozilla/l10n": {
- "version": "0.14.0",
- "resolved": "https://registry.npmjs.org/@mozilla/l10n/-/l10n-0.14.0.tgz",
- "integrity": "sha512-07aqvhPiUQrMrQMmAqdZh7UNKaNwkPS7PSaNdcBlHv401HD3tAWdlDLfv4smslMeU7Aqa2qvO9nhnF3/+kI2ng==",
+ "version": "0.14.1",
+ "resolved": "https://registry.npmjs.org/@mozilla/l10n/-/l10n-0.14.1.tgz",
+ "integrity": "sha512-cpoTYv0Jv4SanrT4GR9geBaQOZ5QAiPgU2Yx+G3bOHzFDyCD13+6xLOjfIEUdQrRvg4hKgkOVFh3r7efgs7v4Q==",
"license": "Apache-2.0",
"dependencies": {
"@fluent/syntax": "^0.19.0",
@@ -10654,7 +10654,7 @@
"@fluent/langneg": "^0.7.0",
"@fluent/react": "^0.15.1",
"@lezer/highlight": "^1.1.6",
- "@mozilla/l10n": "^0.14.0",
+ "@mozilla/l10n": "^0.14.1",
"@reduxjs/toolkit": "^1.6.1",
"classnames": "^2.3.1",
"date-and-time": "^0.14.2",
diff --git a/pontoon/base/models/section.py b/pontoon/base/models/section.py
index 72e901fb48..05733b049d 100644
--- a/pontoon/base/models/section.py
+++ b/pontoon/base/models/section.py
@@ -15,6 +15,7 @@ class Section(models.Model):
resource: models.ForeignKey[Resource] = models.ForeignKey(
Resource, models.CASCADE, related_name="sections"
)
+ resource_id: int
key = ArrayField(models.TextField())
meta = ArrayField(ArrayField(models.TextField(), size=2), default=list)
comment = models.TextField(blank=True)
diff --git a/pontoon/base/models/translation.py b/pontoon/base/models/translation.py
index 532046bfbd..6e8f3f52c2 100644
--- a/pontoon/base/models/translation.py
+++ b/pontoon/base/models/translation.py
@@ -121,10 +121,13 @@ def bulk_mark_changed(self):
class Translation(DirtyFieldsMixin, models.Model):
entity: models.ForeignKey[Entity] = models.ForeignKey(Entity, models.CASCADE)
+ entity_id: int
locale: models.ForeignKey[Locale] = models.ForeignKey(Locale, models.CASCADE)
+ locale_id: int
user: models.ForeignKey[User | None] = models.ForeignKey(
User, models.SET_NULL, null=True, blank=True
)
+ user_id: int | None
string = models.TextField()
value = models.JSONField()
properties = models.JSONField(null=True, blank=True)
@@ -146,6 +149,7 @@ class Translation(DirtyFieldsMixin, models.Model):
null=True,
blank=True,
)
+ approved_user_id: int | None
approved_date = models.DateTimeField(null=True, blank=True)
unapproved_user = models.ForeignKey(
@@ -155,6 +159,7 @@ class Translation(DirtyFieldsMixin, models.Model):
null=True,
blank=True,
)
+ unapproved_user_id: int | None
unapproved_date = models.DateTimeField(null=True, blank=True)
rejected = models.BooleanField(default=False)
@@ -165,6 +170,7 @@ class Translation(DirtyFieldsMixin, models.Model):
null=True,
blank=True,
)
+ rejected_user_id: int | None
rejected_date = models.DateTimeField(null=True, blank=True)
unrejected_user = models.ForeignKey(
@@ -174,6 +180,7 @@ class Translation(DirtyFieldsMixin, models.Model):
null=True,
blank=True,
)
+ unrejected_user_id: int | None
unrejected_date = models.DateTimeField(null=True, blank=True)
class MachinerySource(models.TextChoices):
diff --git a/pontoon/base/tests/views/test_download.py b/pontoon/base/tests/views/test_download.py
index 0ad4b9ac79..a4e698ae3a 100644
--- a/pontoon/base/tests/views/test_download.py
+++ b/pontoon/base/tests/views/test_download.py
@@ -1,88 +1,133 @@
-from os import makedirs
-from tempfile import TemporaryDirectory
-from unittest.mock import patch
+from textwrap import dedent
import pytest
-from django.conf import settings
from django.test import RequestFactory
-from pontoon.base.models import Project, Resource
+from pontoon.base.models import Project
from pontoon.base.views import download_translations
-from pontoon.sync.tests.test_checkouts import MockVersionControl
-from pontoon.sync.tests.utils import build_file_tree
from pontoon.test.factories import (
+ EntityFactory,
LocaleFactory,
ProjectFactory,
- RepositoryFactory,
ResourceFactory,
+ SectionFactory,
TranslatedResourceFactory,
+ TranslationFactory,
UserFactory,
)
@pytest.mark.django_db
-@pytest.mark.parametrize("two_repos", [True, False])
-@pytest.mark.parametrize(
- "repo_url,expected_location",
- [
- (
- "https://github.com:gh-org/gh-repo.git",
- "https://raw.githubusercontent.com/gh-org/gh-repo/HEAD/de-Test/a.ftl",
- ),
- (
- "git@gitlab.com:gl-org/gl-repo.git",
- "https://gitlab.com/gl-org/gl-repo/-/raw/HEAD/de-Test/a.ftl?inline=false",
- ),
- ("http://example.com/tgt-repo", "https://example.com/tgt-repo"),
- ],
-)
-def test_download(two_repos, repo_url, expected_location):
- with (
- TemporaryDirectory() as root,
- patch("pontoon.sync.core.checkout.get_repo", return_value=MockVersionControl()),
- ):
- settings.MEDIA_ROOT = root
- locale = LocaleFactory.create(code="de-Test")
- if two_repos:
- repo_src = RepositoryFactory(
- url="http://example.com/src-repo", source_repo=True
- )
- repo_tgt = RepositoryFactory(url=repo_url)
- project = ProjectFactory.create(
- name="test-dl",
- locales=[locale],
- repositories=[repo_src, repo_tgt],
- visibility=Project.Visibility.PUBLIC,
- )
- src_root = repo_src.checkout_path
- tgt_root = repo_tgt.checkout_path
- makedirs(src_root)
- build_file_tree(src_root, {"en-US": {"a.ftl": ""}})
- makedirs(tgt_root)
- build_file_tree(tgt_root, {"de-Test": {"a.ftl": ""}})
- else:
- repo = RepositoryFactory(url=repo_url)
- project = ProjectFactory.create(
- name="test-dl",
- locales=[locale],
- repositories=[repo],
- visibility=Project.Visibility.PUBLIC,
- )
- repo_root = repo.checkout_path
- makedirs(repo_root)
- build_file_tree(
- repo_root, {"en-US": {"a.ftl": ""}, "de-Test": {"a.ftl": ""}}
- )
- res = ResourceFactory.create(
- project=project, path="a.ftl", format=Resource.Format.FLUENT
- )
- TranslatedResourceFactory.create(locale=locale, resource=res)
+def test_download_fluent():
+ locale = LocaleFactory.create(code="de-Test")
+ project = ProjectFactory.create(
+ name="test-dl",
+ locales=[locale],
+ visibility=Project.Visibility.PUBLIC,
+ )
+ res = ResourceFactory.create(
+ project=project, format="fluent", path="path/to/file.ftl"
+ )
+ TranslatedResourceFactory.create(locale=locale, resource=res)
+ section = SectionFactory.create(resource=res, key=[], comment="Group")
+ e1 = EntityFactory.create(resource=res, section=section, key=["e1"], value=["E1"])
+ e2 = EntityFactory.create(
+ resource=res,
+ section=section,
+ key=["e2"],
+ value=[],
+ properties={"attr": ["E2"]},
+ )
+ EntityFactory.create(resource=res, section=section, key=["e3"], value=["E3"])
+ TranslationFactory.create(
+ locale=locale, entity=e1, value=["T1"], active=True, approved=True
+ )
+ TranslationFactory.create(
+ locale=locale,
+ entity=e2,
+ value=[],
+ properties={"attr": ["T2"]},
+ active=True,
+ approved=True,
+ )
+
+ request = RequestFactory().get(
+ "/translations/?code=de-Test&slug=test-dl&part=path/to/file.ftl"
+ )
+ request.user = UserFactory()
+ response = download_translations(request)
+ assert response.status_code == 200
+ assert response["Content-Type"] == "text/plain"
+ assert (
+ response["Content-Disposition"]
+ == "attachment; filename=de-Test_test-dl_path_to_file.ftl"
+ )
+ assert response.content.decode("utf-8") == dedent("""\
+ ## Group
+
+ e1 = T1
+ e2 =
+ .attr = T2
+ """)
+
+
+@pytest.mark.django_db
+def test_download_xliff():
+ locale = LocaleFactory.create(code="de-Test")
+ project = ProjectFactory.create(
+ name="test-dlx",
+ locales=[locale],
+ visibility=Project.Visibility.PUBLIC,
+ )
+ res = ResourceFactory.create(project=project, format="xliff", path="file.xlf")
+ TranslatedResourceFactory.create(locale=locale, resource=res)
+ section = SectionFactory.create(resource=res, key=["file.foo"], comment="Group")
+ e1 = EntityFactory.create(
+ resource=res, section=section, key=["file.foo", "e1"], value=["E1"]
+ )
+ e2 = EntityFactory.create(
+ resource=res, section=section, key=["file.foo", "e2"], value=["E2"]
+ )
+ EntityFactory.create(
+ resource=res, section=section, key=["file.foo", "e3"], value=["E3"]
+ )
+ TranslationFactory.create(
+ locale=locale, entity=e1, value=["T1"], active=True, approved=True
+ )
+ TranslationFactory.create(
+ locale=locale, entity=e2, value=["T2"], active=True, approved=True
+ )
- request = RequestFactory().get(
- "/translations/?code=de-Test&slug=test-dl&part=a.ftl"
- )
- request.user = UserFactory()
- response = download_translations(request)
- assert response.status_code == 302
- assert response.get("Location") == expected_location
+ request = RequestFactory().get(
+ "/translations/?code=de-Test&slug=test-dlx&part=file.xlf"
+ )
+ request.user = UserFactory()
+ response = download_translations(request)
+ assert response.status_code == 200
+ assert response["Content-Type"] == "text/plain"
+ assert (
+ response["Content-Disposition"]
+ == "attachment; filename=de-Test_test-dlx_file.xlf"
+ )
+ assert response.content.decode("utf-8") == dedent("""\
+
+
+
+
+
+
+ E1
+ T1
+
+
+ E2
+ T2
+
+
+ E3
+
+
+
+
+ """)
diff --git a/pontoon/base/views.py b/pontoon/base/views.py
index 2609d7fa90..b2d68849fc 100755
--- a/pontoon/base/views.py
+++ b/pontoon/base/views.py
@@ -20,7 +20,6 @@
HttpRequest,
HttpResponse,
HttpResponseForbidden,
- HttpResponseRedirect,
JsonResponse,
StreamingHttpResponse,
)
@@ -1008,9 +1007,9 @@ def perform_checks(request):
@transaction.atomic
def download_translations(request):
- """Download translated resource from its backing repository."""
+ """Download translated resource."""
- from pontoon.sync.utils import translations_target_url
+ from pontoon.sync.utils import serialize_translated_resource
try:
slug = request.GET["slug"]
@@ -1019,17 +1018,26 @@ def download_translations(request):
except MultiValueDictKeyError:
raise Http404
- project = get_object_or_404(Project.objects.visible_for(request.user), slug=slug)
+ project = get_object_or_404(
+ Project.objects.visible_for(request.user), slug=slug, disabled=False
+ )
+ resource = get_object_or_404(
+ Resource, project=project, path=res_path, obsolete=False
+ )
locale = get_object_or_404(Locale, code=code)
+ if not TranslatedResource.objects.filter(locale=locale, resource=resource).exists():
+ raise Http404(
+ f"{resource.path} of {project.slug} not available for locale {locale.code}"
+ )
- # FIXME This is a temporary hack, to be replaced by 04/2025 with proper downloads.
- # Once fixed, we should remove SSH credentials from the web pod
- # https://github.com/mozilla/webservices-infra/pull/9295
- url = translations_target_url(project, locale, res_path)
- if url and url.startswith("https://"):
- return HttpResponseRedirect(url)
- else:
- raise Http404
+ str_res = serialize_translated_resource(resource, locale)
+ filename = re.sub(r"[/\\]", "_", f"{locale.code}_{slug}_{res_path}")
+
+ response = HttpResponse()
+ response.content = str_res.encode("utf-8")
+ response["Content-Type"] = "text/plain"
+ response["Content-Disposition"] = f"attachment; filename={filename}"
+ return response
@login_required(redirect_field_name="", login_url="/403")
diff --git a/pontoon/pretranslation/tests/test_pretranslate.py b/pontoon/pretranslation/tests/test_pretranslate.py
index 1a580df779..add028a288 100644
--- a/pontoon/pretranslation/tests/test_pretranslate.py
+++ b/pontoon/pretranslation/tests/test_pretranslate.py
@@ -228,8 +228,8 @@ def test_get_pretranslations_fluent_accesskeys_label_attribute(
expected = dedent(
"""
title = gt_translation
- .label = gt_translation
.aria-label = gt_translation
+ .label = gt_translation
.value = gt_translation
.accesskey = g
"""
@@ -760,8 +760,8 @@ def gt_mock_fn(**kwargs):
expected = dedent(
"""
batman = GT: The { $dark } Knight
- .weapon = GT: Brain and { -wayne-enterprise }
.history = GT: Lost { 2 } parents, has { 1 } "$alfred"
+ .weapon = GT: Brain and { -wayne-enterprise }
"""
)
diff --git a/pontoon/sync/core/entities.py b/pontoon/sync/core/entities.py
index 4a336ac403..90d7b1abea 100644
--- a/pontoon/sync/core/entities.py
+++ b/pontoon/sync/core/entities.py
@@ -56,6 +56,11 @@ def sync_resources_from_repo(
xliff_source_entries=True,
)
assert res.format
+ # XLIFF templates _should_ not contain elements,
+ # but if they do, we strip them out.
+ if res.format == L10nFormat.xliff:
+ for entry in res.all_entries():
+ entry.del_meta("target")
try:
Resource.Format(res.format.name)
updates[db_path] = res
diff --git a/pontoon/sync/core/translations_to_repo.py b/pontoon/sync/core/translations_to_repo.py
index cc615b1e92..11fb02b442 100644
--- a/pontoon/sync/core/translations_to_repo.py
+++ b/pontoon/sync/core/translations_to_repo.py
@@ -1,17 +1,20 @@
+import html
import logging
from collections import defaultdict
+from copy import deepcopy
from datetime import datetime
from os import makedirs, remove
-from os.path import commonpath, dirname, isfile, join, normpath
+from os.path import commonpath, dirname, isfile
from moz.l10n.formats import Format
from moz.l10n.formats.xliff import xliff_is_xcode
-from moz.l10n.message import parse_message
+from moz.l10n.message import message_from_json, serialize_message
from moz.l10n.model import (
CatchallKey,
Entry,
Id,
+ Message,
Metadata,
PatternMessage,
Resource,
@@ -19,13 +22,20 @@
SelectMessage,
)
from moz.l10n.paths import L10nConfigPaths, L10nDiscoverPaths
-from moz.l10n.resource import parse_resource, serialize_resource
+from moz.l10n.resource import serialize_resource
from django.conf import settings
from django.db.models import Q
from django.db.models.query import QuerySet
-from pontoon.base.models import Locale, Project, Translation, User
+from pontoon.base.models import (
+ Entity,
+ Locale,
+ Project,
+ Resource as DbResource,
+ Translation,
+ User,
+)
from pontoon.base.models.changed_entity_locale import ChangedEntityLocale
from pontoon.sync.core.checkout import Checkouts
from pontoon.sync.repositories import CommitToRepositoryException, get_repo
@@ -159,20 +169,24 @@ def update_changed_resources(
now: datetime,
) -> tuple[int, set[Locale], dict[User, set[str]]]:
count = 0
- # db_path -> {Locale}, empty set stands for "all locales"
- changed_resources: dict[str, set[Locale]] = {
- path: set() for path in changed_source_paths
+ # db_path -> (db_res, {Locale})
+ # Where an empty set stands for "all locales"
+ changed_resources: dict[str, tuple[DbResource, set[Locale]]] = {
+ db_res.path: (db_res, set())
+ for db_res in DbResource.objects.filter(
+ project=project, path__in=changed_source_paths
+ )
}
for change in db_changes:
if change.locale in readonly_locales:
continue
- path = str(change.entity.resource.path)
- if path not in changed_resources:
- changed_resources[path] = {change.locale}
+ db_res = change.entity.resource
+ if db_res.path not in changed_resources:
+ changed_resources[db_res.path] = (db_res, {change.locale})
else:
- prev = changed_resources[path]
- if prev:
- prev.add(change.locale)
+ prev = changed_resources[db_res.path]
+ if prev[1]:
+ prev[1].add(change.locale)
changed_entities = {change.entity for change in db_changes}
if changed_resources:
n = len(changed_resources)
@@ -181,7 +195,7 @@ def update_changed_resources(
updated_locales: set[Locale] = set()
translators: dict[User, set[str]] = defaultdict(set)
- for path, locales_ in changed_resources.items():
+ for path, (db_res, locales_) in changed_resources.items():
log_scope = f"[{project.slug}:{path}]"
target, locale_codes = paths.target(path)
if target is None:
@@ -198,12 +212,6 @@ def update_changed_resources(
}
if not locales:
continue
- ref_path = normpath(join(paths.ref_root, path))
- if ref_path.endswith(".po"):
- ref_path += "t"
- if not isfile(ref_path):
- log.error(f"{log_scope} Missing source file")
- continue
if locales_:
lc_str = ", ".join(locale.code for locale in locales_)
log.info(f"{log_scope} Updating locales: {lc_str}")
@@ -226,93 +234,121 @@ def update_changed_resources(
.exclude(approved_date__gt=now) # includes approved_date = None
.select_related("entity")
)
+ res = build_moz_l10n_resource(db_res)
for locale in locales:
- lc_scope = f"[{project.slug}:{path}, {locale.code}]"
- lc_translations = [tx for tx in translations if tx.locale_id == locale.pk]
+ lc_translations = {
+ tuple(tx.entity.key): tx
+ for tx in translations
+ if tx.locale_id == locale.pk
+ }
target_path = paths.format_target_path(target, locale.code)
if not lc_translations and not isfile(target_path):
continue
try:
lc_plurals = locale.cldr_plurals_list()
- res = parse_resource(ref_path)
- set_translations(locale, lc_translations, res)
+ tr_res = build_translated_resource(locale, lc_translations, res)
makedirs(dirname(target_path), exist_ok=True)
with open(target_path, "w", encoding="utf-8") as file:
- for line in serialize_resource(res, gettext_plurals=lc_plurals):
+ for line in serialize_resource(tr_res, gettext_plurals=lc_plurals):
file.write(line)
updated_locales.add(locale)
- for tx in lc_translations:
+ for tx in lc_translations.values():
if tx.approved and tx.entity in changed_entities and tx.user:
translators[tx.user].add(locale.code)
count += 1
except Exception as error:
- log.error(f"{lc_scope} Update failed: {error}")
+ log.error(
+ f"[{project.slug}:{path}, {locale.code}] Update failed: {error}"
+ )
continue
return count, updated_locales, translators
-def set_translations(
- locale: Locale, translations: list[Translation], res: Resource
-) -> None:
- if res.format == Format.fluent:
- trans_res = parse_resource(
- Format.fluent, "".join(tx.string for tx in translations)
- )
- trans_entries: dict[Id, Entry | None] = {
- entry.id: entry
- for section in trans_res.sections
- for entry in section.entries
- if isinstance(entry, Entry)
- }
- for section in res.sections:
- rm: list[Entry] = []
- for entry in section.entries:
- if isinstance(entry, Entry):
- te = trans_entries.get(entry.id, None)
- if te is None:
- rm.append(entry)
- else:
- entry.value = te.value
- entry.properties = (
- te.properties
- if entry.id[0].startswith("-")
- else {
- name: tp
- for name, tp in te.properties.items()
- if name in entry.properties
- }
- )
- if rm:
- section.entries = [e for e in section.entries if e not in rm]
+def build_moz_l10n_resource(db_res: DbResource) -> Resource:
+ if db_res.format == DbResource.Format.XCODE:
+ format = Format.xliff
else:
- # The iOS locale remapping is a hacky workaround for Xcode projects only,
- # so don't apply it to other XLIFF projects.
- is_xcode = res.format == Format.xliff and xliff_is_xcode(res)
- for section in res.sections:
- if (
- res.format == Format.xliff
- and section.get_meta("@source-language") is not None
- ):
- prev_tgt = next(
- (m for m in section.meta if m.key == "@target-language"), None
- )
- lc = (
- ios_locale_map.get(locale.code, locale.code)
- if is_xcode
- else locale.code
- )
- if prev_tgt is None:
- section.meta.append(Metadata("@target-language", lc))
- else:
- prev_tgt.value = lc
-
- rm: list[Entry] = []
- for entry in section.entries:
- if isinstance(entry, Entry):
- if not set_translation(translations, res.format, section, entry):
+ try:
+ format = Format[db_res.format]
+ except KeyError:
+ raise ValueError(f"Unsupported format: {db_res.format}")
+ db_sections = {s.pk: s for s in db_res.sections.iterator()}
+ sections: list[Section[Message]] = []
+ prev_s_pk = -1
+ for e in db_res.entities.filter(obsolete=False).order_by("order").iterator():
+ entry = _entry_from_entity(format, e)
+ s_pk = e.section_id
+ assert s_pk is not None
+ if s_pk == prev_s_pk:
+ sections[-1].entries.append(entry)
+ continue
+
+ # Comment-based sections (as in Fluent) can be non-contiguous,
+ # and should remain that way.
+ # Sections with a non-empty key should stay together,
+ # even if the Entity.order values are not in the right order.
+ db_section = db_sections[s_pk]
+ prev_s_pk = s_pk
+ s_id = tuple(db_section.key)
+ if s_id:
+ section = next((s for s in sections if s.id == s_id), None)
+ if section is not None:
+ section.entries.append(entry)
+ continue
+
+ section = Section(
+ id=s_id,
+ comment=db_section.comment,
+ meta=[Metadata(k, v) for k, v in db_section.meta],
+ entries=[entry],
+ )
+ sections.append(section)
+ return Resource(
+ format=format,
+ comment=db_res.comment,
+ meta=[Metadata(k, v) for k, v in db_res.meta],
+ sections=sections,
+ )
+
+
+def _entry_from_entity(format: Format, e: Entity) -> Entry[Message]:
+ key = e.key[1:] if format in {Format.ini, Format.xliff} else e.key
+ entry = Entry(
+ id=tuple(key),
+ comment=e.comment,
+ meta=[Metadata(k, v) for k, v in e.meta],
+ value=message_from_json(e.value),
+ properties={k: message_from_json(v) for k, v in e.properties.items()}
+ if e.properties
+ else {},
+ )
+ if format == Format.xliff:
+ source = serialize_message(Format.xliff, entry.value)
+ entry.set_meta("source", html.unescape(source))
+ return entry
+
+
+def build_translated_resource(
+ locale: Locale, translations: dict[Id, Translation], res: Resource[Message]
+) -> Resource[Message]:
+ res = deepcopy(res)
+ for section in res.sections:
+ rm = []
+ for entry in section.entries:
+ assert isinstance(entry, Entry)
+ tx = translations.get(section.id + entry.id, None)
+ if tx is not None:
+ _set_translation(res.format, entry, tx)
+ else:
+ match res.format:
+ case Format.gettext if isinstance(entry.value, SelectMessage):
+ entry.value.variants = {(CatchallKey(),): []}
+ case Format.gettext | Format.xliff:
+ entry.value = PatternMessage([])
+ case _:
rm.append(entry)
- if rm and res.format not in (Format.gettext, Format.xliff):
- section.entries = [e for e in section.entries if e not in rm]
+ if rm:
+ section.entries = [e for e in section.entries if e not in rm]
match res.format:
case Format.gettext:
@@ -327,37 +363,40 @@ def set_translations(
for key, value in header.items()
if key not in gettext_trim_headers
]
+
case Format.xliff:
- pass
+ lc = str(locale.code)
+ if xliff_is_xcode(res):
+ lc = ios_locale_map.get(lc, lc)
+ for section in res.sections:
+ if section.get_meta("@source-language") is not None:
+ section.set_meta("@target-language", lc)
+
case _:
res.sections = [
section
for section in res.sections
if any(isinstance(entry, Entry) for entry in section.entries)
]
+ return res
-def set_translation(
- translations: list[Translation],
- format: Format | None,
- section: Section,
- entry: Entry,
-) -> bool:
- key = list(section.id + entry.id)
- tx = next((tx for tx in translations if tx.entity.key == key), None)
- if tx is None:
- if format == Format.gettext:
- if isinstance(entry.value, SelectMessage):
- entry.value.variants = {(CatchallKey(),): []}
- else:
- entry.value = PatternMessage([])
- return True
- else:
- return False
-
+def _set_translation(format: Format | None, entry: Entry, tx: Translation) -> None:
match format:
+ case Format.fluent:
+ entry.value = message_from_json(tx.value)
+ is_term = entry.id[0].startswith("-")
+ entry.properties = (
+ {
+ name: message_from_json(pv)
+ for name, pv in tx.properties.items()
+ if is_term or name in entry.properties
+ }
+ if tx.properties
+ else {}
+ )
case Format.android | Format.gettext | Format.webext | Format.xliff:
- msg = parse_message(Format.mf2, tx.string)
+ msg = message_from_json(tx.value)
if isinstance(entry.value, SelectMessage):
entry.value.variants = (
{(CatchallKey(),): msg.pattern}
@@ -377,6 +416,4 @@ def set_translation(
entry.meta = [m for m in entry.meta if m != fuzzy_flag]
case _:
- entry.value = tx.string
-
- return True
+ entry.value = message_from_json(tx.value)
diff --git a/pontoon/sync/tests/test_e2e.py b/pontoon/sync/tests/test_e2e.py
index b186f08ed4..7f426e4a94 100644
--- a/pontoon/sync/tests/test_e2e.py
+++ b/pontoon/sync/tests/test_e2e.py
@@ -33,6 +33,7 @@
ProjectLocaleFactory,
RepositoryFactory,
ResourceFactory,
+ SectionFactory,
TranslatedResourceFactory,
TranslationFactory,
)
@@ -71,17 +72,19 @@ def test_kitchen_sink():
ResourceFactory.create(project=project, path="a.ftl", format="fluent")
ResourceFactory.create(project=project, path="b.po", format="gettext")
res_c = ResourceFactory.create(project=project, path="c.ftl", format="fluent")
+ section = SectionFactory.create(resource=res_c, key=[])
for i in range(3):
entity = EntityFactory.create(
resource=res_c,
+ section=section,
key=[f"key-{i}"],
- string=f"key-{i} = Message {i}\n",
+ value=[f"Message {i}"],
)
for locale in [locale_de, locale_fr]:
TranslationFactory.create(
entity=entity,
locale=locale,
- string=f"key-{i} = New translation {locale.code[:2]} {i}\n",
+ value=[f"New translation {locale.code[:2]} {i}"],
active=True,
approved=True,
)
@@ -255,7 +258,7 @@ def test_add_resources():
resource__project=project, resource__path="file.xliff"
),
locale=locale,
- string="xliff translation",
+ value=["xliff translation"],
active=True,
approved=True,
)
@@ -343,7 +346,7 @@ def test_xliff_html_translation():
TranslationFactory.create(
entity=Entity.objects.get(resource__project=project),
locale=locale,
- string="translation: Hello world!",
+ value=["translation: Hello world!"],
active=True,
approved=True,
)
@@ -430,7 +433,7 @@ def test_xliff_target_language():
resource__project=project, resource__path=path
),
locale=locale,
- string="translation",
+ value=["translation"],
active=True,
approved=True,
)
@@ -456,10 +459,10 @@ def test_translation_before_source():
res_a = ResourceFactory.create(project=project, path="a.ftl", format="fluent")
TranslationFactory.create(
entity=EntityFactory.create(
- resource=res_a, key=["a0"], string="a0 = Message 0\n"
+ resource=res_a, key=["a0"], value=["Message 0"]
),
locale=locale,
- string="a0 = Translation 0\n",
+ value=["Translation 0"],
active=True,
approved=True,
)
@@ -467,10 +470,10 @@ def test_translation_before_source():
res_b = ResourceFactory.create(project=project, path="b.ftl", format="fluent")
TranslationFactory.create(
entity=EntityFactory.create(
- resource=res_b, key=["b0"], string="b0 = Message 0\n"
+ resource=res_b, key=["b0"], value=["Message 0"]
),
locale=locale,
- string="b0 = Translation 0\n",
+ value=["Translation 0"],
active=True,
approved=True,
)
@@ -512,23 +515,23 @@ def test_android():
)
entity = EntityFactory.create(
- resource=res, key=["quotes"], string="Prev quotes"
+ resource=res, key=["quotes"], value=["Prev quotes"]
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string="'Hello' \"translation\"",
+ value=["'Hello' \"translation\""],
active=True,
approved=True,
)
entity = EntityFactory.create(
- resource=res, key=["newline"], string="Prev newline"
+ resource=res, key=["newline"], value=["Prev newline"]
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string="translated escaped \n newlines",
+ value=["translated escaped \n newlines"],
active=True,
approved=True,
)
@@ -576,11 +579,13 @@ def test_gettext_fuzzy():
for i in range(5):
string = f"Message {i}\n"
fuzzy = i < 3
- entity = EntityFactory.create(resource=res, key=[f"key-{i}"], string=string)
+ entity = EntityFactory.create(
+ resource=res, key=[f"key-{i}"], value=[string]
+ )
TranslationFactory.create(
entity=entity,
locale=locale,
- string=string.replace("Message", "Fuzzy" if fuzzy else "Translation"),
+ value=[string.replace("Message", "Fuzzy" if fuzzy else "Translation")],
active=True,
approved=not fuzzy,
fuzzy=fuzzy,
@@ -696,12 +701,14 @@ def test_plain_json(caplog):
res = ResourceFactory.create(
project=project, path="old-file.json", format="plain_json"
)
-
- entity = EntityFactory.create(resource=res, key=["o1"], string="Entity 1")
+ section = SectionFactory.create(resource=res, key=[])
+ entity = EntityFactory.create(
+ resource=res, section=section, key=["o1"], value=["Entity 1"]
+ )
TranslationFactory.create(
entity=entity,
locale=locale,
- string="Translation 1",
+ value=["Translation 1"],
active=True,
approved=True,
)
@@ -791,22 +798,24 @@ def test_webext():
project=project, path="messages.json", format="android"
)
- entity = EntityFactory.create(resource=res, key=["plain"], string="Entity")
+ entity = EntityFactory.create(resource=res, key=["plain"], value=["Entity"])
TranslationFactory.create(
entity=entity,
locale=locale,
- string="Translation",
+ value=["Translation"],
active=True,
approved=True,
)
entity = EntityFactory.create(
- resource=res, key=["number"], string="Entity for {$arg1 @source=|$1|}"
+ resource=res,
+ key=["number"],
+ value=["Entity for ", {"$": "arg1", "attr": {"source": "$1"}}],
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string="Translation for {$arg1 @source=|$1|}",
+ value=["Translation for ", {"$": "arg1", "attr": {"source": "$1"}}],
active=True,
approved=True,
)
@@ -814,14 +823,31 @@ def test_webext():
entity = EntityFactory.create(
resource=res,
key=["name"],
- string=".local $ORIGIN = {$arg1 @source=|$1| @example=developer.mozilla.org}\n"
- + "{{Entity for {$ORIGIN @source=|$ORIGIN$|}}}",
+ value={
+ "decl": {
+ "ORIGIN": {
+ "$": "arg1",
+ "attr": {"source": "$1", "example": "developer.mozilla.org"},
+ }
+ },
+ "msg": ["Entity for ", {"$": "ORIGIN", "attr": {"source": "ORIGIN"}}],
+ },
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string=".local $ORIGIN = {$arg1 @source=|$1| @example=developer.mozilla.org}\n"
- + "{{Translation for {$ORIGIN @source=|$ORIGIN$|}}}",
+ value={
+ "decl": {
+ "ORIGIN": {
+ "$": "arg1",
+ "attr": {"source": "$1", "example": "developer.mozilla.org"},
+ }
+ },
+ "msg": [
+ "Translation for ",
+ {"$": "ORIGIN", "attr": {"source": "ORIGIN"}},
+ ],
+ },
active=True,
approved=True,
)
diff --git a/pontoon/sync/tests/test_translations_to_repo.py b/pontoon/sync/tests/test_translations_to_repo.py
index b17f142a32..89759420fc 100644
--- a/pontoon/sync/tests/test_translations_to_repo.py
+++ b/pontoon/sync/tests/test_translations_to_repo.py
@@ -7,13 +7,19 @@
import pytest
+from moz.l10n.formats import Format
+from moz.l10n.model import Entry, Metadata, PatternMessage, Resource, Section
+
from django.conf import settings
from django.utils import timezone
-from pontoon.base.models import ChangedEntityLocale
+from pontoon.base.models import ChangedEntityLocale, Resource as DbResource
from pontoon.sync.core.checkout import Checkout, Checkouts
from pontoon.sync.core.paths import find_paths
-from pontoon.sync.core.translations_to_repo import sync_translations_to_repo
+from pontoon.sync.core.translations_to_repo import (
+ build_moz_l10n_resource,
+ sync_translations_to_repo,
+)
from pontoon.sync.tests.utils import build_file_tree
from pontoon.test.factories import (
EntityFactory,
@@ -21,6 +27,7 @@
ProjectFactory,
RepositoryFactory,
ResourceFactory,
+ SectionFactory,
TranslatedResourceFactory,
TranslationFactory,
)
@@ -104,17 +111,19 @@ def test_remove_entity():
TranslatedResourceFactory.create(locale=locale, resource=res_a)
TranslatedResourceFactory.create(locale=locale, resource=res_b)
TranslatedResourceFactory.create(locale=locale, resource=res_c, total_strings=3)
+ section = SectionFactory.create(resource=res_c, key=[])
for i in range(3):
if i != 1:
entity = EntityFactory.create(
resource=res_c,
+ section=section,
key=[f"key-{i}"],
- string=f"key-{i} = Message {i}\n",
+ value=[f"Message {i}"],
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string=f"key-{i} = Translation {i}\n",
+ value=[f"Translation {i}"],
active=True,
approved=True,
)
@@ -192,51 +201,59 @@ def test_add_translation():
TranslatedResourceFactory.create(locale=locale, resource=res_a)
TranslatedResourceFactory.create(locale=locale, resource=res_b)
TranslatedResourceFactory.create(locale=locale, resource=res_c, total_strings=3)
+ section = SectionFactory.create(resource=res_c, key=[])
ent_0 = EntityFactory.create(
resource=res_c,
+ section=section,
key=["key-0"],
- string="key-0 = Message 0\n",
+ value=["Message 0"],
)
TranslationFactory.create(
entity=ent_0,
locale=locale,
- string="key-0 = Translation 0\n",
+ value=["Translation 0"],
active=True,
approved=True,
)
ent_1 = EntityFactory.create(
resource=res_c,
+ section=section,
key=["key-1"],
- string="key-1 = Message 1\n",
+ value=["Message 1"],
)
TranslationFactory.create(
entity=ent_1,
locale=locale,
- string="key-1 = Translation 1\n",
+ value=["Translation 1"],
active=True,
approved=True,
)
ent_2 = EntityFactory.create(
resource=res_c,
+ section=section,
key=["key-2"],
- string="key-2 =\n .attr = Message 2\n",
+ value=[],
+ properties={"attr": ["Message 2"]},
)
TranslationFactory.create(
entity=ent_2,
locale=locale,
- string="key-2 =\n .attr = Translation 2\n",
+ value=[],
+ properties={"attr": ["Translation 2"]},
active=True,
approved=True,
)
ent_3 = EntityFactory.create(
resource=res_c,
+ section=section,
key=["-term-3"],
- string="-term-3 = Term 3\n",
+ value=["Term 3"],
)
TranslationFactory.create(
entity=ent_3,
locale=locale,
- string="-term-3 = Translation 3\n .attr = Term attribute\n",
+ value=["Translation 3"],
+ properties={"attr": ["Term attribute"]},
active=True,
approved=True,
)
@@ -316,12 +333,12 @@ def test_directory_creation_on_translation_update():
entity = EntityFactory.create(
resource=res_c,
key=["key-0"],
- string="key-0 = Message 0\n",
+ value=["Message 0"],
)
TranslationFactory.create(
entity=entity,
locale=locale,
- string="key-0 = Translation 0\n",
+ value=["Translation 0"],
active=True,
approved=True,
)
@@ -369,3 +386,93 @@ def test_directory_creation_on_translation_update():
assert exists(target_path), (
"Expected translated file to be created in nested directories."
)
+
+
+@pytest.mark.django_db
+def test_build_fluent_resource():
+ res = ResourceFactory(
+ format=DbResource.Format.FLUENT, path="path/to/file.ftl", comment="R"
+ )
+ s0 = SectionFactory(resource=res, key=[])
+ EntityFactory(resource=res, section=s0, order=0, key=["e0"], value=["E0"])
+ s1 = SectionFactory(resource=res, key=[], comment="S1")
+ e1 = EntityFactory(resource=res, section=s1, order=1, key=["e1"], value=["E1"])
+ s2 = SectionFactory(resource=res, key=[])
+ EntityFactory(resource=res, section=s2, order=2, key=["e2"], value=["E2"])
+
+ assert build_moz_l10n_resource(res) == Resource(
+ format=Format.fluent,
+ comment="R",
+ sections=[
+ Section(id=(), entries=[Entry(id=("e0",), value=PatternMessage(["E0"]))]),
+ Section(
+ id=(),
+ comment="S1",
+ entries=[Entry(id=("e1",), value=PatternMessage(["E1"]))],
+ ),
+ Section(id=(), entries=[Entry(id=("e2",), value=PatternMessage(["E2"]))]),
+ ],
+ )
+
+ e1.order = 3
+ e1.save()
+ assert build_moz_l10n_resource(res) == Resource(
+ format=Format.fluent,
+ comment="R",
+ sections=[
+ Section(id=(), entries=[Entry(id=("e0",), value=PatternMessage(["E0"]))]),
+ Section(id=(), entries=[Entry(id=("e2",), value=PatternMessage(["E2"]))]),
+ Section(
+ id=(),
+ comment="S1",
+ entries=[Entry(id=("e1",), value=PatternMessage(["E1"]))],
+ ),
+ ],
+ )
+
+
+@pytest.mark.django_db
+def test_build_xliff_resource():
+ res = ResourceFactory(format=DbResource.Format.XCODE, path="file.xliff")
+ s1 = SectionFactory(resource=res, key=["path/to/f2"])
+ EntityFactory(
+ resource=res, section=s1, order=1, key=["path/to/f2", "e1"], value=["E1"]
+ )
+ s0 = SectionFactory(resource=res, key=["path/to/f1"])
+ EntityFactory(
+ resource=res, section=s0, order=2, key=["path/to/f1", "e2"], value=["E2"]
+ )
+ EntityFactory(
+ resource=res, section=s0, order=0, key=["path/to/f1", "e0"], value=["E0"]
+ )
+
+ assert build_moz_l10n_resource(res) == Resource(
+ format=Format.xliff,
+ sections=[
+ Section(
+ id=("path/to/f1",),
+ entries=[
+ Entry(
+ id=("e0",),
+ meta=[Metadata("source", "E0")],
+ value=PatternMessage(["E0"]),
+ ),
+ Entry(
+ id=("e2",),
+ meta=[Metadata("source", "E2")],
+ value=PatternMessage(["E2"]),
+ ),
+ ],
+ ),
+ Section(
+ id=("path/to/f2",),
+ entries=[
+ Entry(
+ id=("e1",),
+ meta=[Metadata("source", "E1")],
+ value=PatternMessage(["E1"]),
+ ),
+ ],
+ ),
+ ],
+ )
diff --git a/pontoon/sync/utils.py b/pontoon/sync/utils.py
index 612c619f6f..746d1c2008 100644
--- a/pontoon/sync/utils.py
+++ b/pontoon/sync/utils.py
@@ -1,83 +1,53 @@
-import re
-
-from io import BytesIO
-from os.path import basename, exists, join, relpath
+from os.path import basename, join
from tempfile import TemporaryDirectory
-from zipfile import ZipFile
+
+from moz.l10n.resource import serialize_resource
from django.core.files import File
+from django.db.models import Q
from django.utils import timezone
from pontoon.base.badge_utils import badges_review_level, badges_translation_level
-from pontoon.base.models import ChangedEntityLocale, Locale, Project, User
-from pontoon.base.models.repository import Repository
+from pontoon.base.models import (
+ ChangedEntityLocale,
+ Locale,
+ Project,
+ Resource as DbResource,
+ Translation,
+ User,
+)
from pontoon.messaging.notifications import send_badge_notification
-from pontoon.sync.core.checkout import checkout_repos
-from pontoon.sync.core.paths import UploadPaths, find_paths
+from pontoon.sync.core.paths import UploadPaths
from pontoon.sync.core.stats import update_stats
from pontoon.sync.core.translations_from_repo import find_db_updates, write_db_updates
-from pontoon.sync.core.translations_to_repo import update_changed_resources
-
-
-# FIXME This is a temporary hack, to be replaced by 04/2025 with proper downloads.
-# Once fixed, we should remove SSH credentials from the web pod
-# https://github.com/mozilla/webservices-infra/pull/9295
-def translations_target_url(
- project: Project, locale: Locale, resource_path: str
-) -> str | None:
- """The target repository URL for a resource, for direct download."""
-
- if project.repositories.count() > 1:
- # HACK: Let's assume that no config is used, and the target repo root is the right base.
- target_repo: Repository = project.repositories.get(source_repo=False)
- rel_path = f"{locale.code}/{resource_path}"
- else:
- checkouts = checkout_repos(project, shallow=True)
- target_repo = checkouts.target.repo
- paths = find_paths(project, checkouts)
- target, _ = paths.target(resource_path)
- if not target:
- return None
- abs_path = paths.format_target_path(target, locale.code)
- rel_path = relpath(abs_path, checkouts.target.path).replace("\\", "/")
-
- github = re.search(r"\bgithub\.com[:/]([^/]+)/([^/]+)\.git$", target_repo.url)
- if github:
- org, repo = github.groups()
- ref = f"refs/heads/{target_repo.branch}" if target_repo.branch else "HEAD"
- return f"https://raw.githubusercontent.com/{org}/{repo}/{ref}/{rel_path}"
-
- gitlab = re.search(r"gitlab\.com[:/]([^/]+)/([^/]+)\.git$", target_repo.url)
- if gitlab:
- org, repo = gitlab.groups()
- ref = target_repo.branch or "HEAD"
- return f"https://gitlab.com/{org}/{repo}/-/raw/{ref}/{rel_path}?inline=false"
-
- # Default to bare repo link
- return re.sub(r"^.*?(://|@)", "https://", target_repo.url, count=1)
-
-
-# FIXME Currently not in use, to be refactored for proper download support
-def download_translations_zip(
- project: Project, locale: Locale
-) -> tuple[bytes, str] | tuple[None, None]:
- checkouts = checkout_repos(project, shallow=True)
- paths = find_paths(project, checkouts)
- db_changes = ChangedEntityLocale.objects.filter(
- entity__resource__project=project, locale=locale
- ).select_related("entity__resource", "locale")
- update_changed_resources(project, paths, {}, [], db_changes, set(), timezone.now())
-
- bytes_io = BytesIO()
- zipfile = ZipFile(bytes_io, "w")
- for _, tgt_path in paths.all():
- filename = paths.format_target_path(tgt_path, locale.code)
- if exists(filename):
- arcname = relpath(filename, checkouts.target.path)
- zipfile.write(filename, arcname)
- zipfile.close()
+from pontoon.sync.core.translations_to_repo import (
+ build_moz_l10n_resource,
+ build_translated_resource,
+)
+
+
+def serialize_translated_resource(db_res: DbResource, locale: Locale) -> str:
+ res = build_moz_l10n_resource(db_res)
+ translations = {
+ tuple(tx.entity.key): tx
+ for tx in Translation.objects.filter(
+ entity__obsolete=False,
+ entity__resource=db_res,
+ locale=locale,
+ active=True,
+ )
+ .filter(
+ Q(approved=True)
+ | Q(pretranslated=True, warnings__isnull=True)
+ | Q(fuzzy=True)
+ )
+ .select_related("entity")
+ .iterator()
+ }
+ tr_res = build_translated_resource(locale, translations, res)
- return bytes_io.getvalue(), f"{project.slug}.zip"
+ lc_plurals = locale.cldr_plurals_list()
+ return "".join(serialize_resource(tr_res, gettext_plurals=lc_plurals))
def import_uploaded_file(
diff --git a/pontoon/test/factories.py b/pontoon/test/factories.py
index 61f90891b5..9223e5a96e 100644
--- a/pontoon/test/factories.py
+++ b/pontoon/test/factories.py
@@ -129,7 +129,7 @@ class Meta:
@factory.post_generation
def parsed_value(entity, create, extracted, **kwargs):
- if entity.value:
+ if entity.value or entity.properties:
return
key, value, properties = parse_source_string_to_json(
entity.resource.format, entity.string
diff --git a/requirements/base.in b/requirements/base.in
index 9ab988cf73..68a46cf69d 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -37,7 +37,7 @@ drf-spectacular[sidecar]==0.29.0
google-cloud-translate==3.16.0
gunicorn==23.0.0
markupsafe==2.0.1
-moz.l10n[xml]==0.14.0
+moz.l10n[xml]==0.14.1
newrelic==9.6.0
openai==3.0.0
PyJWT==2.13.0
diff --git a/requirements/dev.txt b/requirements/dev.txt
index 9c61324135..3b33d38557 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -1277,9 +1277,9 @@ markupsafe==2.0.1 \
# -c prod.txt
# -r base.in
# jinja2
-moz-l10n[xml]==0.14.0 \
- --hash=sha256:451050fe26ad019f610813e9d9b702484fb0257cf84a6a57d3b9f36c3e99d3ff \
- --hash=sha256:65225e6781b29369b2aaf1e8c47a52e403252a8bc69e7792ce35a2c4f3762576
+moz-l10n[xml]==0.14.1 \
+ --hash=sha256:215c2fb80d591125d14dc15065fb861af3941fb24c58820d00d717287e72d23a \
+ --hash=sha256:b61e4767c136f08a240abcb3d4fb02ed0891ed654cec6228c36bc75656f3de44
# via
# -c prod.txt
# -r base.in
diff --git a/requirements/prod.txt b/requirements/prod.txt
index 3a54c7997d..d2672f6ba7 100644
--- a/requirements/prod.txt
+++ b/requirements/prod.txt
@@ -977,9 +977,9 @@ markupsafe==2.0.1 \
# via
# -r base.in
# jinja2
-moz-l10n[xml]==0.14.0 \
- --hash=sha256:451050fe26ad019f610813e9d9b702484fb0257cf84a6a57d3b9f36c3e99d3ff \
- --hash=sha256:65225e6781b29369b2aaf1e8c47a52e403252a8bc69e7792ce35a2c4f3762576
+moz-l10n[xml]==0.14.1 \
+ --hash=sha256:215c2fb80d591125d14dc15065fb861af3941fb24c58820d00d717287e72d23a \
+ --hash=sha256:b61e4767c136f08a240abcb3d4fb02ed0891ed654cec6228c36bc75656f3de44
# via -r base.in
newrelic==9.6.0 \
--hash=sha256:01c0eb630bb18261241a37aa0a70cb6f706079a1f58f59f2bb64f26fda54ffc5 \
diff --git a/translate/package.json b/translate/package.json
index 64e588531c..41d404b38a 100644
--- a/translate/package.json
+++ b/translate/package.json
@@ -11,7 +11,7 @@
"@fluent/langneg": "^0.7.0",
"@fluent/react": "^0.15.1",
"@lezer/highlight": "^1.1.6",
- "@mozilla/l10n": "^0.14.0",
+ "@mozilla/l10n": "^0.14.1",
"@reduxjs/toolkit": "^1.6.1",
"classnames": "^2.3.1",
"date-and-time": "^0.14.2",