Write resources without fetch, using data model fields

pontoon/base/models/section.py

@@ -15,6 +15,7 @@ class Section(models.Model):
     resource: models.ForeignKey[Resource] = models.ForeignKey(
         Resource, models.CASCADE, related_name="sections"
     )
+    resource_id: int
     key = ArrayField(models.TextField())
     meta = ArrayField(ArrayField(models.TextField(), size=2), default=list)
     comment = models.TextField(blank=True)

pontoon/base/models/translation.py

@@ -121,10 +121,13 @@ def bulk_mark_changed(self):
 
 class Translation(DirtyFieldsMixin, models.Model):
     entity: models.ForeignKey[Entity] = models.ForeignKey(Entity, models.CASCADE)
+    entity_id: int
     locale: models.ForeignKey[Locale] = models.ForeignKey(Locale, models.CASCADE)
+    locale_id: int
     user: models.ForeignKey[User | None] = models.ForeignKey(
         User, models.SET_NULL, null=True, blank=True
     )
+    user_id: int | None
     string = models.TextField()
     value = models.JSONField()
     properties = models.JSONField(null=True, blank=True)
@@ -146,6 +149,7 @@ class Translation(DirtyFieldsMixin, models.Model):
         null=True,
         blank=True,
     )
+    approved_user_id: int | None
     approved_date = models.DateTimeField(null=True, blank=True)
 
     unapproved_user = models.ForeignKey(
@@ -155,6 +159,7 @@ class Translation(DirtyFieldsMixin, models.Model):
         null=True,
         blank=True,
     )
+    unapproved_user_id: int | None
     unapproved_date = models.DateTimeField(null=True, blank=True)
 
     rejected = models.BooleanField(default=False)
@@ -165,6 +170,7 @@ class Translation(DirtyFieldsMixin, models.Model):
         null=True,
         blank=True,
     )
+    rejected_user_id: int | None
     rejected_date = models.DateTimeField(null=True, blank=True)
 
     unrejected_user = models.ForeignKey(
@@ -174,6 +180,7 @@ class Translation(DirtyFieldsMixin, models.Model):
         null=True,
         blank=True,
     )
+    unrejected_user_id: int | None
     unrejected_date = models.DateTimeField(null=True, blank=True)
 
     class MachinerySource(models.TextChoices):

pontoon/base/tests/views/test_download.py

@@ -1,88 +1,143 @@
-from os import makedirs
-from tempfile import TemporaryDirectory
-from unittest.mock import patch
+from io import BytesIO
+from textwrap import dedent
+from zipfile import ZipFile
 
 import pytest
 
-from django.conf import settings
 from django.test import RequestFactory
 
-from pontoon.base.models import Project, Resource
+from pontoon.base.models import Project
 from pontoon.base.views import download_translations
-from pontoon.sync.tests.test_checkouts import MockVersionControl
-from pontoon.sync.tests.utils import build_file_tree
 from pontoon.test.factories import (
+    EntityFactory,
     LocaleFactory,
     ProjectFactory,
-    RepositoryFactory,
     ResourceFactory,
+    SectionFactory,
     TranslatedResourceFactory,
+    TranslationFactory,
     UserFactory,
 )
 
 
 @pytest.mark.django_db
-@pytest.mark.parametrize("two_repos", [True, False])
-@pytest.mark.parametrize(
-    "repo_url,expected_location",
-    [
-        (
-            "https://github.com:gh-org/gh-repo.git",
-            "https://raw.githubusercontent.com/gh-org/gh-repo/HEAD/de-Test/a.ftl",
-        ),
-        (
-            "git@gitlab.com:gl-org/gl-repo.git",
-            "https://gitlab.com/gl-org/gl-repo/-/raw/HEAD/de-Test/a.ftl?inline=false",
-        ),
-        ("http://example.com/tgt-repo", "https://example.com/tgt-repo"),
-    ],
-)
-def test_download(two_repos, repo_url, expected_location):
-    with (
-        TemporaryDirectory() as root,
-        patch("pontoon.sync.core.checkout.get_repo", return_value=MockVersionControl()),
-    ):
-        settings.MEDIA_ROOT = root
-        locale = LocaleFactory.create(code="de-Test")
-        if two_repos:
-            repo_src = RepositoryFactory(
-                url="http://example.com/src-repo", source_repo=True
-            )
-            repo_tgt = RepositoryFactory(url=repo_url)
-            project = ProjectFactory.create(
-                name="test-dl",
-                locales=[locale],
-                repositories=[repo_src, repo_tgt],
-                visibility=Project.Visibility.PUBLIC,
-            )
-            src_root = repo_src.checkout_path
-            tgt_root = repo_tgt.checkout_path
-            makedirs(src_root)
-            build_file_tree(src_root, {"en-US": {"a.ftl": ""}})
-            makedirs(tgt_root)
-            build_file_tree(tgt_root, {"de-Test": {"a.ftl": ""}})
-        else:
-            repo = RepositoryFactory(url=repo_url)
-            project = ProjectFactory.create(
-                name="test-dl",
-                locales=[locale],
-                repositories=[repo],
-                visibility=Project.Visibility.PUBLIC,
-            )
-            repo_root = repo.checkout_path
-            makedirs(repo_root)
-            build_file_tree(
-                repo_root, {"en-US": {"a.ftl": ""}, "de-Test": {"a.ftl": ""}}
-            )
-        res = ResourceFactory.create(
-            project=project, path="a.ftl", format=Resource.Format.FLUENT
-        )
-        TranslatedResourceFactory.create(locale=locale, resource=res)
+def test_download_fluent():
+    locale = LocaleFactory.create(code="de-Test")
+    project = ProjectFactory.create(
+        name="test-dl",
+        locales=[locale],
+        visibility=Project.Visibility.PUBLIC,
+    )
+    res = ResourceFactory.create(
+        project=project, format="fluent", path="path/to/file.ftl"
+    )
+    TranslatedResourceFactory.create(locale=locale, resource=res)
+    section = SectionFactory.create(resource=res, key=[], comment="Group")
+    e1 = EntityFactory.create(resource=res, section=section, key=["e1"], value=["E1"])
+    e2 = EntityFactory.create(
+        resource=res,
+        section=section,
+        key=["e2"],
+        value=[],
+        properties={"attr": ["E2"]},
+    )
+    EntityFactory.create(resource=res, section=section, key=["e3"], value=["E3"])
+    TranslationFactory.create(
+        locale=locale, entity=e1, value=["T1"], active=True, approved=True
+    )
+    TranslationFactory.create(
+        locale=locale,
+        entity=e2,
+        value=[],
+        properties={"attr": ["T2"]},
+        active=True,
+        approved=True,
+    )
+
+    request = RequestFactory().get(
+        "/translations/?code=de-Test&slug=test-dl&part=path/to/file.ftl"
+    )
+    request.user = UserFactory()
+    response = download_translations(request)
+    assert response.status_code == 200
+    assert response["Content-Type"] == "application/zip"
+    assert (
+        response["Content-Disposition"]
+        == "attachment; filename=de-Test_test-dl_path_to_file.zip"
+    )
+    bytes_io = BytesIO(response.content)
+    with ZipFile(bytes_io, "r") as zipfile:
+        assert zipfile.namelist() == ["de-Test_test-dl_path_to_file.ftl"]
+        raw = zipfile.read("de-Test_test-dl_path_to_file.ftl")
+    assert raw.decode("utf-8") == dedent("""\
+        ## Group
+
+        e1 = T1
+        e2 =
+            .attr = T2
+        """)
+
+
+@pytest.mark.django_db
+def test_download_xliff():
+    locale = LocaleFactory.create(code="de-Test")
+    project = ProjectFactory.create(
+        name="test-dlx",
+        locales=[locale],
+        visibility=Project.Visibility.PUBLIC,
+    )
+    res = ResourceFactory.create(project=project, format="xliff", path="file.xlf")
+    TranslatedResourceFactory.create(locale=locale, resource=res)
+    section = SectionFactory.create(resource=res, key=["file.foo"], comment="Group")
+    e1 = EntityFactory.create(
+        resource=res, section=section, key=["file.foo", "e1"], value=["E1"]
+    )
+    e2 = EntityFactory.create(
+        resource=res, section=section, key=["file.foo", "e2"], value=["E2"]
+    )
+    EntityFactory.create(
+        resource=res, section=section, key=["file.foo", "e3"], value=["E3"]
+    )
+    TranslationFactory.create(
+        locale=locale, entity=e1, value=["T1"], active=True, approved=True
+    )
+    TranslationFactory.create(
+        locale=locale, entity=e2, value=["T2"], active=True, approved=True
+    )
 
-        request = RequestFactory().get(
-            "/translations/?code=de-Test&slug=test-dl&part=a.ftl"
-        )
-        request.user = UserFactory()
-        response = download_translations(request)
-        assert response.status_code == 302
-        assert response.get("Location") == expected_location
+    request = RequestFactory().get(
+        "/translations/?code=de-Test&slug=test-dlx&part=file.xlf"
+    )
+    request.user = UserFactory()
+    response = download_translations(request)
+    assert response.status_code == 200
+    assert response["Content-Type"] == "application/zip"
+    assert (
+        response["Content-Disposition"]
+        == "attachment; filename=de-Test_test-dlx_file.zip"
+    )
+    bytes_io = BytesIO(response.content)
+    with ZipFile(bytes_io, "r") as zipfile:
+        assert zipfile.namelist() == ["de-Test_test-dlx_file.xlf"]
+        raw = zipfile.read("de-Test_test-dlx_file.xlf")
+    assert raw.decode("utf-8") == dedent("""\
+        <?xml version="1.0" encoding="utf-8"?>
+        <xliff>
+          <file original="file.foo">
+            <!-- Group -->
+            <body>
+              <trans-unit id="e1">
+                <source>E1</source>
+                <target>T1</target>
+              </trans-unit>
+              <trans-unit id="e2">
+                <source>E2</source>
+                <target>T2</target>
+              </trans-unit>
+              <trans-unit id="e3">
+                <source>E3</source>
+              </trans-unit>
+            </body>
+          </file>
+        </xliff>
+        """)

pontoon/base/views.py

@@ -20,7 +20,6 @@
     HttpRequest,
     HttpResponse,
     HttpResponseForbidden,
-    HttpResponseRedirect,
     JsonResponse,
     StreamingHttpResponse,
 )
@@ -1008,9 +1007,12 @@ def perform_checks(request):
 
 @transaction.atomic
 def download_translations(request):
-    """Download translated resource from its backing repository."""
+    """Download translated resource."""
 
-    from pontoon.sync.utils import translations_target_url
+    from io import BytesIO
+    from zipfile import ZIP_DEFLATED, ZipFile
+
+    from pontoon.sync.utils import serialize_translated_resource
 
     try:
         slug = request.GET["slug"]
@@ -1020,16 +1022,21 @@ def download_translations(request):
         raise Http404
 
     project = get_object_or_404(Project.objects.visible_for(request.user), slug=slug)
+    resource = get_object_or_404(Resource, project=project, path=res_path)
     locale = get_object_or_404(Locale, code=code)
 
-    # FIXME This is a temporary hack, to be replaced by 04/2025 with proper downloads.
-    # Once fixed, we should remove SSH credentials from the web pod
-    # https://github.com/mozilla/webservices-infra/pull/9295
-    url = translations_target_url(project, locale, res_path)
-    if url and url.startswith("https://"):
-        return HttpResponseRedirect(url)
-    else:
-        raise Http404
+    filename = re.sub(r"[/\\]", "_", f"{locale.code}_{slug}_{res_path}")
+    bytes_io = BytesIO()
+    zipfile = ZipFile(bytes_io, "w", compression=ZIP_DEFLATED)
+    zipfile.writestr(filename, serialize_translated_resource(resource, locale))
+    zipfile.close()
+
+    response = HttpResponse()
+    response.content = bytes_io.getvalue()
+    response["Content-Type"] = "application/zip"
+    zip_name = re.sub(r"[^.]+$", "zip", filename)
+    response["Content-Disposition"] = f"attachment; filename={zip_name}"
+    return response
 
 
 @login_required(redirect_field_name="", login_url="/403")

pontoon/pretranslation/tests/test_pretranslate.py

@@ -228,8 +228,8 @@ def test_get_pretranslations_fluent_accesskeys_label_attribute(
     expected = dedent(
         """
         title = gt_translation
-            .label = gt_translation
             .aria-label = gt_translation
+            .label = gt_translation
             .value = gt_translation
             .accesskey = g
     """
@@ -760,8 +760,8 @@ def gt_mock_fn(**kwargs):
     expected = dedent(
         """
         batman = GT: The { $dark } Knight
-            .weapon = GT: Brain and { -wayne-enterprise }
             .history = GT: Lost { 2 } parents, has { 1 } "$alfred"
+            .weapon = GT: Brain and { -wayne-enterprise }
     """
     )
 

pontoon/sync/core/entities.py

@@ -56,6 +56,11 @@ def sync_resources_from_repo(
                     xliff_source_entries=True,
                 )
                 assert res.format
+                # XLIFF templates _should_ not contain <target> elements,
+                # but if they do, we strip them out.
+                if res.format == L10nFormat.xliff:
+                    for entry in res.all_entries():
+                        entry.del_meta("target")
                 try:
                     Resource.Format(res.format.name)
                     updates[db_path] = res