We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Refine pretranslation logic for strings with rejected suggestions
Currently, a string with suggestions is ignored by pretranslation.

We should:
* Pretranslate if all suggestions are submitted by users and rejected. 
* Not pretranslate if a pretranslation was already rejected.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pontoon/pretranslation/pretranslate.py`
Content:
```
1 import logging
2 import operator
3 import re
4 
5 from django.db.models import CharField, Value as V
6 from django.db.models.functions import Concat
7 
8 from fluent.syntax import FluentParser, FluentSerializer
9 from functools import reduce
10 
11 from pontoon.base.models import User, TranslatedResource
12 from pontoon.base.fluent import FlatTransformer, create_locale_plural_variants
13 from pontoon.machinery.utils import (
14     get_google_translate_data,
15     get_translation_memory_data,
16 )
17 
18 
19 log = logging.getLogger(__name__)
20 
21 parser = FluentParser()
22 serializer = FluentSerializer()
23 
24 
25 class PretranslationTransformer(FlatTransformer):
26     def __init__(self, locale):
27         self.services = []
28         self.locale = locale
29 
30     def visit_SelectExpression(self, node):
31         create_locale_plural_variants(node, self.locale)
32         return self.generic_visit(node)
33 
34     def visit_TextElement(self, node):
35         # Machine translation treats each line as separate sentence,
36         # hence we replace newline characters with spaces.
37         source = node.value.replace("\n", " ")
38 
39         pretranslation, service = get_pretranslated_data(source, self.locale)
40 
41         if pretranslation is None:
42             raise ValueError(
43                 f"Pretranslation for `{source}` to {self.locale.code} not available."
44             )
45 
46         node.value = pretranslation
47         self.services.append(service)
48         return node
49 
50 
51 def get_pretranslations(entity, locale):
52     """
53     Get pretranslations for the entity-locale pair using internal translation memory and
54     Google's machine translation.
55 
56     For Fluent strings, uplift SelectExpressions, serialize Placeables as TextElements
57     and then only pretranslate TextElements. Set the most frequent TextElement
58     pretranslation author as the author of the entire pretranslation.
59 
60     :arg Entity entity: the Entity object
61     :arg Locale locale: the Locale object
62 
63     :returns: a list of tuples, consisting of:
64         - a pretranslation of the entity
65         - a plural form
66         - a user (representing TM or GT service)
67     """
68     source = entity.string
69     services = {
70         "tm": User.objects.get(email="pontoon-tm@example.com"),
71         "gt": User.objects.get(email="pontoon-gt@example.com"),
72     }
73 
74     if entity.resource.format == "ftl":
75         source_ast = parser.parse_entry(source)
76         pt_transformer = PretranslationTransformer(locale)
77 
78         try:
79             pretranslated_ast = pt_transformer.visit(source_ast)
80         except ValueError as e:
81             log.info(f"Fluent pretranslation error: {e}")
82             return []
83 
84         pretranslation = serializer.serialize_entry(pretranslated_ast)
85 
86         authors = [services[service] for service in pt_transformer.services]
87         author = max(set(authors), key=authors.count) if authors else services["tm"]
88 
89         return [(pretranslation, None, author)]
90 
91     else:
92         pretranslation, service = get_pretranslated_data(source, locale)
93 
94         if pretranslation is None:
95             return []
96 
97         author = services[service]
98         if entity.string_plural == "":
99             return [(pretranslation, None, author)]
100         else:
101             plural_forms = range(0, locale.nplurals or 1)
102             return [
103                 (pretranslation, plural_form, author) for plural_form in plural_forms
104             ]
105 
106 
107 def get_pretranslated_data(source, locale):
108     # Empty strings do not need translation
109     if re.search("^\\s*$", source):
110         return source, "tm"
111 
112     # Try to get matches from Translation Memory
113     tm_response = get_translation_memory_data(text=source, locale=locale)
114     tm_perfect = [t for t in tm_response if int(t["quality"]) == 100]
115     if tm_perfect:
116         return tm_perfect[0]["target"], "tm"
117 
118     # Fetch from Google Translate
119     elif locale.google_translate_code:
120         gt_response = get_google_translate_data(text=source, locale=locale)
121         if gt_response["status"]:
122             return gt_response["translation"], "gt"
123 
124     return None, None
125 
126 
127 def update_changed_instances(tr_filter, tr_dict, translations):
128     """
129     Update the latest activity and stats for changed Locales, ProjectLocales
130     & TranslatedResources
131     """
132     tr_filter = tuple(tr_filter)
133     # Combine all generated filters with an OK operator.
134     # `operator.ior` is the '|' Python operator, which turns into a logical OR
135     # when used between django ORM query objects.
136     tr_query = reduce(operator.ior, tr_filter)
137 
138     translatedresources = TranslatedResource.objects.filter(tr_query).annotate(
139         locale_resource=Concat(
140             "locale_id", V("-"), "resource_id", output_field=CharField()
141         )
142     )
143 
144     translatedresources.update_stats()
145 
146     for tr in translatedresources:
147         index = tr_dict[tr.locale_resource]
148         translation = translations[index]
149         translation.update_latest_translation()
150
```

Path: `pontoon/pretranslation/__init__.py`
Content:
```
```

Path: `pontoon/pretranslation/tasks.py`
Content:
```
1 import logging
2 
3 from django.db.models import Q, CharField, Value as V
4 from django.db.models.functions import Concat
5 from django.conf import settings
6 from pontoon.base.models import (
7     Project,
8     Entity,
9     TranslatedResource,
10     Translation,
11 )
12 from pontoon.actionlog.models import ActionLog
13 from pontoon.pretranslation.pretranslate import (
14     get_pretranslations,
15     update_changed_instances,
16 )
17 from pontoon.base.tasks import PontoonTask
18 from pontoon.sync.core import serial_task
19 from pontoon.checks.utils import bulk_run_checks
20 
21 
22 log = logging.getLogger(__name__)
23 
24 
25 @serial_task(settings.SYNC_TASK_TIMEOUT, base=PontoonTask, lock_key="project={0}")
26 def pretranslate(self, project_pk, locales=None, entities=None):
27     """
28     Identifies strings without any translations and any suggestions.
29     Engages TheAlgorithm (bug 1552796) to gather pretranslations.
30     Stores pretranslations as suggestions (approved=False) to DB.
31 
32     :arg project_pk: the pk of the project to be pretranslated
33     :arg Queryset locales: the locales for the project to be pretranslated
34     :arg Queryset entites: the entities for the project to be pretranslated
35 
36     :returns: None
37     """
38     project = Project.objects.get(pk=project_pk)
39 
40     if not project.pretranslation_enabled:
41         log.info(f"Pretranslation not enabled for project {project.name}")
42         return
43 
44     if locales:
45         locales = project.locales.filter(pk__in=locales)
46     else:
47         locales = project.locales
48 
49     locales = locales.filter(
50         project_locale__project=project,
51         project_locale__pretranslation_enabled=True,
52         project_locale__readonly=False,
53     )
54 
55     if not locales:
56         log.info(
57             f"Pretranslation not enabled for any locale within project {project.name}"
58         )
59         return
60 
61     log.info(f"Fetching pretranslations for project {project.name} started")
62 
63     if not entities:
64         entities = Entity.objects.filter(
65             resource__project=project,
66             obsolete=False,
67         )
68 
69     entities = entities.prefetch_related("resource")
70 
71     # get available TranslatedResource pairs
72     tr_pairs = (
73         TranslatedResource.objects.filter(
74             resource__project=project,
75             locale__in=locales,
76         )
77         .annotate(
78             locale_resource=Concat(
79                 "locale_id", V("-"), "resource_id", output_field=CharField()
80             )
81         )
82         .values_list("locale_resource", flat=True)
83         .distinct()
84     )
85 
86     # Fetch all distinct locale-entity pairs for which translation exists
87     translated_entities = (
88         Translation.objects.filter(
89             locale__in=locales,
90             entity__in=entities,
91         )
92         .annotate(
93             locale_entity=Concat(
94                 "locale_id", V("-"), "entity_id", output_field=CharField()
95             )
96         )
97         .values_list("locale_entity", flat=True)
98         .distinct()
99     )
100 
101     translated_entities = list(translated_entities)
102 
103     translations = []
104 
105     # To keep track of changed TranslatedResources and their latest_translation
106     tr_dict = {}
107 
108     tr_filter = []
109     index = -1
110 
111     for locale in locales:
112         log.info(f"Fetching pretranslations for locale {locale.code} started")
113         for entity in entities:
114             locale_entity = f"{locale.id}-{entity.id}"
115             locale_resource = f"{locale.id}-{entity.resource.id}"
116             if locale_entity in translated_entities or locale_resource not in tr_pairs:
117                 continue
118 
119             pretranslations = get_pretranslations(entity, locale)
120 
121             if not pretranslations:
122                 continue
123 
124             for string, plural_form, user in pretranslations:
125                 t = Translation(
126                     entity=entity,
127                     locale=locale,
128                     string=string,
129                     user=user,
130                     approved=False,
131                     pretranslated=True,
132                     active=True,
133                     plural_form=plural_form,
134                 )
135 
136                 index += 1
137                 translations.append(t)
138 
139                 if locale_resource not in tr_dict:
140                     tr_dict[locale_resource] = index
141 
142                     # Add query for fetching respective TranslatedResource.
143                     tr_filter.append(
144                         Q(locale__id=locale.id) & Q(resource__id=entity.resource.id)
145                     )
146 
147                 # Update the latest translation index
148                 tr_dict[locale_resource] = index
149 
150         log.info(f"Fetching pretranslations for locale {locale.code} done")
151 
152     if len(translations) == 0:
153         return
154 
155     translations = Translation.objects.bulk_create(translations)
156 
157     # Log creating actions
158     actions_to_log = [
159         ActionLog(
160             action_type=ActionLog.ActionType.TRANSLATION_CREATED,
161             performed_by=t.user,
162             translation=t,
163         )
164         for t in translations
165     ]
166 
167     ActionLog.objects.bulk_create(actions_to_log)
168 
169     # Run checks on all translations
170     translation_pks = {translation.pk for translation in translations}
171     bulk_run_checks(Translation.objects.for_checks().filter(pk__in=translation_pks))
172 
173     # Mark translations as changed
174     changed_translations = Translation.objects.filter(
175         pk__in=translation_pks,
176         # Do not sync translations with errors and warnings
177         errors__isnull=True,
178         warnings__isnull=True,
179     )
180     changed_translations.bulk_mark_changed()
181 
182     # Update latest activity and stats for changed instances.
183     update_changed_instances(tr_filter, tr_dict, translations)
184 
185     log.info(f"Fetching pretranslations for project {project.name} done")
186
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pontoon/pretranslation/__init__.py b/pontoon/pretranslation/__init__.py
--- a/pontoon/pretranslation/__init__.py
+++ b/pontoon/pretranslation/__init__.py
@@ -0,0 +1,4 @@
+AUTHORS = {
+    "tm": "pontoon-tm@example.com",
+    "gt": "pontoon-gt@example.com",
+}
diff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py
--- a/pontoon/pretranslation/pretranslate.py
+++ b/pontoon/pretranslation/pretranslate.py
@@ -14,6 +14,7 @@
     get_google_translate_data,
     get_translation_memory_data,
 )
+from pontoon.pretranslation import AUTHORS
 
 
 log = logging.getLogger(__name__)
@@ -66,10 +67,7 @@
         - a user (representing TM or GT service)
     """
     source = entity.string
-    services = {
-        "tm": User.objects.get(email="pontoon-tm@example.com"),
-        "gt": User.objects.get(email="pontoon-gt@example.com"),
-    }
+    services = {k: User.objects.get(email=email) for k, email in AUTHORS.items()}
 
     if entity.resource.format == "ftl":
         source_ast = parser.parse_entry(source)
diff --git a/pontoon/pretranslation/tasks.py b/pontoon/pretranslation/tasks.py
--- a/pontoon/pretranslation/tasks.py
+++ b/pontoon/pretranslation/tasks.py
@@ -8,8 +8,10 @@
     Entity,
     TranslatedResource,
     Translation,
+    User,
 )
 from pontoon.actionlog.models import ActionLog
+from pontoon.pretranslation import AUTHORS
 from pontoon.pretranslation.pretranslate import (
     get_pretranslations,
     update_changed_instances,
@@ -68,7 +70,7 @@
 
     entities = entities.prefetch_related("resource")
 
-    # get available TranslatedResource pairs
+    # Fetch all available locale-resource pairs (TranslatedResource objects)
     tr_pairs = (
         TranslatedResource.objects.filter(
             resource__project=project,
@@ -83,12 +85,14 @@
         .distinct()
     )
 
-    # Fetch all distinct locale-entity pairs for which translation exists
+    # Fetch all locale-entity pairs with non-rejected or pretranslated translations
+    pt_authors = [User.objects.get(email=email) for email in AUTHORS.values()]
     translated_entities = (
         Translation.objects.filter(
             locale__in=locales,
             entity__in=entities,
         )
+        .filter(Q(rejected=False) | Q(user__in=pt_authors))
         .annotate(
             locale_entity=Concat(
                 "locale_id", V("-"), "entity_id", output_field=CharField()

{"golden_diff": "diff --git a/pontoon/pretranslation/__init__.py b/pontoon/pretranslation/__init__.py\n--- a/pontoon/pretranslation/__init__.py\n+++ b/pontoon/pretranslation/__init__.py\n@@ -0,0 +1,4 @@\n+AUTHORS = {\n+    \"tm\": \"pontoon-tm@example.com\",\n+    \"gt\": \"pontoon-gt@example.com\",\n+}\ndiff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py\n--- a/pontoon/pretranslation/pretranslate.py\n+++ b/pontoon/pretranslation/pretranslate.py\n@@ -14,6 +14,7 @@\n     get_google_translate_data,\n     get_translation_memory_data,\n )\n+from pontoon.pretranslation import AUTHORS\n \n \n log = logging.getLogger(__name__)\n@@ -66,10 +67,7 @@\n         - a user (representing TM or GT service)\n     \"\"\"\n     source = entity.string\n-    services = {\n-        \"tm\": User.objects.get(email=\"pontoon-tm@example.com\"),\n-        \"gt\": User.objects.get(email=\"pontoon-gt@example.com\"),\n-    }\n+    services = {k: User.objects.get(email=email) for k, email in AUTHORS.items()}\n \n     if entity.resource.format == \"ftl\":\n         source_ast = parser.parse_entry(source)\ndiff --git a/pontoon/pretranslation/tasks.py b/pontoon/pretranslation/tasks.py\n--- a/pontoon/pretranslation/tasks.py\n+++ b/pontoon/pretranslation/tasks.py\n@@ -8,8 +8,10 @@\n     Entity,\n     TranslatedResource,\n     Translation,\n+    User,\n )\n from pontoon.actionlog.models import ActionLog\n+from pontoon.pretranslation import AUTHORS\n from pontoon.pretranslation.pretranslate import (\n     get_pretranslations,\n     update_changed_instances,\n@@ -68,7 +70,7 @@\n \n     entities = entities.prefetch_related(\"resource\")\n \n-    # get available TranslatedResource pairs\n+    # Fetch all available locale-resource pairs (TranslatedResource objects)\n     tr_pairs = (\n         TranslatedResource.objects.filter(\n             resource__project=project,\n@@ -83,12 +85,14 @@\n         .distinct()\n     )\n \n-    # Fetch all distinct locale-entity pairs for which translation exists\n+    # Fetch all locale-entity pairs with non-rejected or pretranslated translations\n+    pt_authors = [User.objects.get(email=email) for email in AUTHORS.values()]\n     translated_entities = (\n         Translation.objects.filter(\n             locale__in=locales,\n             entity__in=entities,\n         )\n+        .filter(Q(rejected=False) | Q(user__in=pt_authors))\n         .annotate(\n             locale_entity=Concat(\n                 \"locale_id\", V(\"-\"), \"entity_id\", output_field=CharField()\n", "issue": "Refine pretranslation logic for strings with rejected suggestions\nCurrently, a string with suggestions is ignored by pretranslation.\r\n\r\nWe should:\r\n* Pretranslate if all suggestions are submitted by users and rejected. \r\n* Not pretranslate if a pretranslation was already rejected.\n", "before_files": [{"content": "import logging\nimport operator\nimport re\n\nfrom django.db.models import CharField, Value as V\nfrom django.db.models.functions import Concat\n\nfrom fluent.syntax import FluentParser, FluentSerializer\nfrom functools import reduce\n\nfrom pontoon.base.models import User, TranslatedResource\nfrom pontoon.base.fluent import FlatTransformer, create_locale_plural_variants\nfrom pontoon.machinery.utils import (\n    get_google_translate_data,\n    get_translation_memory_data,\n)\n\n\nlog = logging.getLogger(__name__)\n\nparser = FluentParser()\nserializer = FluentSerializer()\n\n\nclass PretranslationTransformer(FlatTransformer):\n    def __init__(self, locale):\n        self.services = []\n        self.locale = locale\n\n    def visit_SelectExpression(self, node):\n        create_locale_plural_variants(node, self.locale)\n        return self.generic_visit(node)\n\n    def visit_TextElement(self, node):\n        # Machine translation treats each line as separate sentence,\n        # hence we replace newline characters with spaces.\n        source = node.value.replace(\"\\n\", \" \")\n\n        pretranslation, service = get_pretranslated_data(source, self.locale)\n\n        if pretranslation is None:\n            raise ValueError(\n                f\"Pretranslation for `{source}` to {self.locale.code} not available.\"\n            )\n\n        node.value = pretranslation\n        self.services.append(service)\n        return node\n\n\ndef get_pretranslations(entity, locale):\n    \"\"\"\n    Get pretranslations for the entity-locale pair using internal translation memory and\n    Google's machine translation.\n\n    For Fluent strings, uplift SelectExpressions, serialize Placeables as TextElements\n    and then only pretranslate TextElements. Set the most frequent TextElement\n    pretranslation author as the author of the entire pretranslation.\n\n    :arg Entity entity: the Entity object\n    :arg Locale locale: the Locale object\n\n    :returns: a list of tuples, consisting of:\n        - a pretranslation of the entity\n        - a plural form\n        - a user (representing TM or GT service)\n    \"\"\"\n    source = entity.string\n    services = {\n        \"tm\": User.objects.get(email=\"pontoon-tm@example.com\"),\n        \"gt\": User.objects.get(email=\"pontoon-gt@example.com\"),\n    }\n\n    if entity.resource.format == \"ftl\":\n        source_ast = parser.parse_entry(source)\n        pt_transformer = PretranslationTransformer(locale)\n\n        try:\n            pretranslated_ast = pt_transformer.visit(source_ast)\n        except ValueError as e:\n            log.info(f\"Fluent pretranslation error: {e}\")\n            return []\n\n        pretranslation = serializer.serialize_entry(pretranslated_ast)\n\n        authors = [services[service] for service in pt_transformer.services]\n        author = max(set(authors), key=authors.count) if authors else services[\"tm\"]\n\n        return [(pretranslation, None, author)]\n\n    else:\n        pretranslation, service = get_pretranslated_data(source, locale)\n\n        if pretranslation is None:\n            return []\n\n        author = services[service]\n        if entity.string_plural == \"\":\n            return [(pretranslation, None, author)]\n        else:\n            plural_forms = range(0, locale.nplurals or 1)\n            return [\n                (pretranslation, plural_form, author) for plural_form in plural_forms\n            ]\n\n\ndef get_pretranslated_data(source, locale):\n    # Empty strings do not need translation\n    if re.search(\"^\\\\s*$\", source):\n        return source, \"tm\"\n\n    # Try to get matches from Translation Memory\n    tm_response = get_translation_memory_data(text=source, locale=locale)\n    tm_perfect = [t for t in tm_response if int(t[\"quality\"]) == 100]\n    if tm_perfect:\n        return tm_perfect[0][\"target\"], \"tm\"\n\n    # Fetch from Google Translate\n    elif locale.google_translate_code:\n        gt_response = get_google_translate_data(text=source, locale=locale)\n        if gt_response[\"status\"]:\n            return gt_response[\"translation\"], \"gt\"\n\n    return None, None\n\n\ndef update_changed_instances(tr_filter, tr_dict, translations):\n    \"\"\"\n    Update the latest activity and stats for changed Locales, ProjectLocales\n    & TranslatedResources\n    \"\"\"\n    tr_filter = tuple(tr_filter)\n    # Combine all generated filters with an OK operator.\n    # `operator.ior` is the '|' Python operator, which turns into a logical OR\n    # when used between django ORM query objects.\n    tr_query = reduce(operator.ior, tr_filter)\n\n    translatedresources = TranslatedResource.objects.filter(tr_query).annotate(\n        locale_resource=Concat(\n            \"locale_id\", V(\"-\"), \"resource_id\", output_field=CharField()\n        )\n    )\n\n    translatedresources.update_stats()\n\n    for tr in translatedresources:\n        index = tr_dict[tr.locale_resource]\n        translation = translations[index]\n        translation.update_latest_translation()\n", "path": "pontoon/pretranslation/pretranslate.py"}, {"content": "", "path": "pontoon/pretranslation/__init__.py"}, {"content": "import logging\n\nfrom django.db.models import Q, CharField, Value as V\nfrom django.db.models.functions import Concat\nfrom django.conf import settings\nfrom pontoon.base.models import (\n    Project,\n    Entity,\n    TranslatedResource,\n    Translation,\n)\nfrom pontoon.actionlog.models import ActionLog\nfrom pontoon.pretranslation.pretranslate import (\n    get_pretranslations,\n    update_changed_instances,\n)\nfrom pontoon.base.tasks import PontoonTask\nfrom pontoon.sync.core import serial_task\nfrom pontoon.checks.utils import bulk_run_checks\n\n\nlog = logging.getLogger(__name__)\n\n\n@serial_task(settings.SYNC_TASK_TIMEOUT, base=PontoonTask, lock_key=\"project={0}\")\ndef pretranslate(self, project_pk, locales=None, entities=None):\n    \"\"\"\n    Identifies strings without any translations and any suggestions.\n    Engages TheAlgorithm (bug 1552796) to gather pretranslations.\n    Stores pretranslations as suggestions (approved=False) to DB.\n\n    :arg project_pk: the pk of the project to be pretranslated\n    :arg Queryset locales: the locales for the project to be pretranslated\n    :arg Queryset entites: the entities for the project to be pretranslated\n\n    :returns: None\n    \"\"\"\n    project = Project.objects.get(pk=project_pk)\n\n    if not project.pretranslation_enabled:\n        log.info(f\"Pretranslation not enabled for project {project.name}\")\n        return\n\n    if locales:\n        locales = project.locales.filter(pk__in=locales)\n    else:\n        locales = project.locales\n\n    locales = locales.filter(\n        project_locale__project=project,\n        project_locale__pretranslation_enabled=True,\n        project_locale__readonly=False,\n    )\n\n    if not locales:\n        log.info(\n            f\"Pretranslation not enabled for any locale within project {project.name}\"\n        )\n        return\n\n    log.info(f\"Fetching pretranslations for project {project.name} started\")\n\n    if not entities:\n        entities = Entity.objects.filter(\n            resource__project=project,\n            obsolete=False,\n        )\n\n    entities = entities.prefetch_related(\"resource\")\n\n    # get available TranslatedResource pairs\n    tr_pairs = (\n        TranslatedResource.objects.filter(\n            resource__project=project,\n            locale__in=locales,\n        )\n        .annotate(\n            locale_resource=Concat(\n                \"locale_id\", V(\"-\"), \"resource_id\", output_field=CharField()\n            )\n        )\n        .values_list(\"locale_resource\", flat=True)\n        .distinct()\n    )\n\n    # Fetch all distinct locale-entity pairs for which translation exists\n    translated_entities = (\n        Translation.objects.filter(\n            locale__in=locales,\n            entity__in=entities,\n        )\n        .annotate(\n            locale_entity=Concat(\n                \"locale_id\", V(\"-\"), \"entity_id\", output_field=CharField()\n            )\n        )\n        .values_list(\"locale_entity\", flat=True)\n        .distinct()\n    )\n\n    translated_entities = list(translated_entities)\n\n    translations = []\n\n    # To keep track of changed TranslatedResources and their latest_translation\n    tr_dict = {}\n\n    tr_filter = []\n    index = -1\n\n    for locale in locales:\n        log.info(f\"Fetching pretranslations for locale {locale.code} started\")\n        for entity in entities:\n            locale_entity = f\"{locale.id}-{entity.id}\"\n            locale_resource = f\"{locale.id}-{entity.resource.id}\"\n            if locale_entity in translated_entities or locale_resource not in tr_pairs:\n                continue\n\n            pretranslations = get_pretranslations(entity, locale)\n\n            if not pretranslations:\n                continue\n\n            for string, plural_form, user in pretranslations:\n                t = Translation(\n                    entity=entity,\n                    locale=locale,\n                    string=string,\n                    user=user,\n                    approved=False,\n                    pretranslated=True,\n                    active=True,\n                    plural_form=plural_form,\n                )\n\n                index += 1\n                translations.append(t)\n\n                if locale_resource not in tr_dict:\n                    tr_dict[locale_resource] = index\n\n                    # Add query for fetching respective TranslatedResource.\n                    tr_filter.append(\n                        Q(locale__id=locale.id) & Q(resource__id=entity.resource.id)\n                    )\n\n                # Update the latest translation index\n                tr_dict[locale_resource] = index\n\n        log.info(f\"Fetching pretranslations for locale {locale.code} done\")\n\n    if len(translations) == 0:\n        return\n\n    translations = Translation.objects.bulk_create(translations)\n\n    # Log creating actions\n    actions_to_log = [\n        ActionLog(\n            action_type=ActionLog.ActionType.TRANSLATION_CREATED,\n            performed_by=t.user,\n            translation=t,\n        )\n        for t in translations\n    ]\n\n    ActionLog.objects.bulk_create(actions_to_log)\n\n    # Run checks on all translations\n    translation_pks = {translation.pk for translation in translations}\n    bulk_run_checks(Translation.objects.for_checks().filter(pk__in=translation_pks))\n\n    # Mark translations as changed\n    changed_translations = Translation.objects.filter(\n        pk__in=translation_pks,\n        # Do not sync translations with errors and warnings\n        errors__isnull=True,\n        warnings__isnull=True,\n    )\n    changed_translations.bulk_mark_changed()\n\n    # Update latest activity and stats for changed instances.\n    update_changed_instances(tr_filter, tr_dict, translations)\n\n    log.info(f\"Fetching pretranslations for project {project.name} done\")\n", "path": "pontoon/pretranslation/tasks.py"}], "after_files": [{"content": "import logging\nimport operator\nimport re\n\nfrom django.db.models import CharField, Value as V\nfrom django.db.models.functions import Concat\n\nfrom fluent.syntax import FluentParser, FluentSerializer\nfrom functools import reduce\n\nfrom pontoon.base.models import User, TranslatedResource\nfrom pontoon.base.fluent import FlatTransformer, create_locale_plural_variants\nfrom pontoon.machinery.utils import (\n    get_google_translate_data,\n    get_translation_memory_data,\n)\nfrom pontoon.pretranslation import AUTHORS\n\n\nlog = logging.getLogger(__name__)\n\nparser = FluentParser()\nserializer = FluentSerializer()\n\n\nclass PretranslationTransformer(FlatTransformer):\n    def __init__(self, locale):\n        self.services = []\n        self.locale = locale\n\n    def visit_SelectExpression(self, node):\n        create_locale_plural_variants(node, self.locale)\n        return self.generic_visit(node)\n\n    def visit_TextElement(self, node):\n        # Machine translation treats each line as separate sentence,\n        # hence we replace newline characters with spaces.\n        source = node.value.replace(\"\\n\", \" \")\n\n        pretranslation, service = get_pretranslated_data(source, self.locale)\n\n        if pretranslation is None:\n            raise ValueError(\n                f\"Pretranslation for `{source}` to {self.locale.code} not available.\"\n            )\n\n        node.value = pretranslation\n        self.services.append(service)\n        return node\n\n\ndef get_pretranslations(entity, locale):\n    \"\"\"\n    Get pretranslations for the entity-locale pair using internal translation memory and\n    Google's machine translation.\n\n    For Fluent strings, uplift SelectExpressions, serialize Placeables as TextElements\n    and then only pretranslate TextElements. Set the most frequent TextElement\n    pretranslation author as the author of the entire pretranslation.\n\n    :arg Entity entity: the Entity object\n    :arg Locale locale: the Locale object\n\n    :returns: a list of tuples, consisting of:\n        - a pretranslation of the entity\n        - a plural form\n        - a user (representing TM or GT service)\n    \"\"\"\n    source = entity.string\n    services = {k: User.objects.get(email=email) for k, email in AUTHORS.items()}\n\n    if entity.resource.format == \"ftl\":\n        source_ast = parser.parse_entry(source)\n        pt_transformer = PretranslationTransformer(locale)\n\n        try:\n            pretranslated_ast = pt_transformer.visit(source_ast)\n        except ValueError as e:\n            log.info(f\"Fluent pretranslation error: {e}\")\n            return []\n\n        pretranslation = serializer.serialize_entry(pretranslated_ast)\n\n        authors = [services[service] for service in pt_transformer.services]\n        author = max(set(authors), key=authors.count) if authors else services[\"tm\"]\n\n        return [(pretranslation, None, author)]\n\n    else:\n        pretranslation, service = get_pretranslated_data(source, locale)\n\n        if pretranslation is None:\n            return []\n\n        author = services[service]\n        if entity.string_plural == \"\":\n            return [(pretranslation, None, author)]\n        else:\n            plural_forms = range(0, locale.nplurals or 1)\n            return [\n                (pretranslation, plural_form, author) for plural_form in plural_forms\n            ]\n\n\ndef get_pretranslated_data(source, locale):\n    # Empty strings do not need translation\n    if re.search(\"^\\\\s*$\", source):\n        return source, \"tm\"\n\n    # Try to get matches from Translation Memory\n    tm_response = get_translation_memory_data(text=source, locale=locale)\n    tm_perfect = [t for t in tm_response if int(t[\"quality\"]) == 100]\n    if tm_perfect:\n        return tm_perfect[0][\"target\"], \"tm\"\n\n    # Fetch from Google Translate\n    elif locale.google_translate_code:\n        gt_response = get_google_translate_data(text=source, locale=locale)\n        if gt_response[\"status\"]:\n            return gt_response[\"translation\"], \"gt\"\n\n    return None, None\n\n\ndef update_changed_instances(tr_filter, tr_dict, translations):\n    \"\"\"\n    Update the latest activity and stats for changed Locales, ProjectLocales\n    & TranslatedResources\n    \"\"\"\n    tr_filter = tuple(tr_filter)\n    # Combine all generated filters with an OK operator.\n    # `operator.ior` is the '|' Python operator, which turns into a logical OR\n    # when used between django ORM query objects.\n    tr_query = reduce(operator.ior, tr_filter)\n\n    translatedresources = TranslatedResource.objects.filter(tr_query).annotate(\n        locale_resource=Concat(\n            \"locale_id\", V(\"-\"), \"resource_id\", output_field=CharField()\n        )\n    )\n\n    translatedresources.update_stats()\n\n    for tr in translatedresources:\n        index = tr_dict[tr.locale_resource]\n        translation = translations[index]\n        translation.update_latest_translation()\n", "path": "pontoon/pretranslation/pretranslate.py"}, {"content": "AUTHORS = {\n    \"tm\": \"pontoon-tm@example.com\",\n    \"gt\": \"pontoon-gt@example.com\",\n}\n", "path": "pontoon/pretranslation/__init__.py"}, {"content": "import logging\n\nfrom django.db.models import Q, CharField, Value as V\nfrom django.db.models.functions import Concat\nfrom django.conf import settings\nfrom pontoon.base.models import (\n    Project,\n    Entity,\n    TranslatedResource,\n    Translation,\n    User,\n)\nfrom pontoon.actionlog.models import ActionLog\nfrom pontoon.pretranslation import AUTHORS\nfrom pontoon.pretranslation.pretranslate import (\n    get_pretranslations,\n    update_changed_instances,\n)\nfrom pontoon.base.tasks import PontoonTask\nfrom pontoon.sync.core import serial_task\nfrom pontoon.checks.utils import bulk_run_checks\n\n\nlog = logging.getLogger(__name__)\n\n\n@serial_task(settings.SYNC_TASK_TIMEOUT, base=PontoonTask, lock_key=\"project={0}\")\ndef pretranslate(self, project_pk, locales=None, entities=None):\n    \"\"\"\n    Identifies strings without any translations and any suggestions.\n    Engages TheAlgorithm (bug 1552796) to gather pretranslations.\n    Stores pretranslations as suggestions (approved=False) to DB.\n\n    :arg project_pk: the pk of the project to be pretranslated\n    :arg Queryset locales: the locales for the project to be pretranslated\n    :arg Queryset entites: the entities for the project to be pretranslated\n\n    :returns: None\n    \"\"\"\n    project = Project.objects.get(pk=project_pk)\n\n    if not project.pretranslation_enabled:\n        log.info(f\"Pretranslation not enabled for project {project.name}\")\n        return\n\n    if locales:\n        locales = project.locales.filter(pk__in=locales)\n    else:\n        locales = project.locales\n\n    locales = locales.filter(\n        project_locale__project=project,\n        project_locale__pretranslation_enabled=True,\n        project_locale__readonly=False,\n    )\n\n    if not locales:\n        log.info(\n            f\"Pretranslation not enabled for any locale within project {project.name}\"\n        )\n        return\n\n    log.info(f\"Fetching pretranslations for project {project.name} started\")\n\n    if not entities:\n        entities = Entity.objects.filter(\n            resource__project=project,\n            obsolete=False,\n        )\n\n    entities = entities.prefetch_related(\"resource\")\n\n    # Fetch all available locale-resource pairs (TranslatedResource objects)\n    tr_pairs = (\n        TranslatedResource.objects.filter(\n            resource__project=project,\n            locale__in=locales,\n        )\n        .annotate(\n            locale_resource=Concat(\n                \"locale_id\", V(\"-\"), \"resource_id\", output_field=CharField()\n            )\n        )\n        .values_list(\"locale_resource\", flat=True)\n        .distinct()\n    )\n\n    # Fetch all locale-entity pairs with non-rejected or pretranslated translations\n    pt_authors = [User.objects.get(email=email) for email in AUTHORS.values()]\n    translated_entities = (\n        Translation.objects.filter(\n            locale__in=locales,\n            entity__in=entities,\n        )\n        .filter(Q(rejected=False) | Q(user__in=pt_authors))\n        .annotate(\n            locale_entity=Concat(\n                \"locale_id\", V(\"-\"), \"entity_id\", output_field=CharField()\n            )\n        )\n        .values_list(\"locale_entity\", flat=True)\n        .distinct()\n    )\n\n    translated_entities = list(translated_entities)\n\n    translations = []\n\n    # To keep track of changed TranslatedResources and their latest_translation\n    tr_dict = {}\n\n    tr_filter = []\n    index = -1\n\n    for locale in locales:\n        log.info(f\"Fetching pretranslations for locale {locale.code} started\")\n        for entity in entities:\n            locale_entity = f\"{locale.id}-{entity.id}\"\n            locale_resource = f\"{locale.id}-{entity.resource.id}\"\n            if locale_entity in translated_entities or locale_resource not in tr_pairs:\n                continue\n\n            pretranslations = get_pretranslations(entity, locale)\n\n            if not pretranslations:\n                continue\n\n            for string, plural_form, user in pretranslations:\n                t = Translation(\n                    entity=entity,\n                    locale=locale,\n                    string=string,\n                    user=user,\n                    approved=False,\n                    pretranslated=True,\n                    active=True,\n                    plural_form=plural_form,\n                )\n\n                index += 1\n                translations.append(t)\n\n                if locale_resource not in tr_dict:\n                    tr_dict[locale_resource] = index\n\n                    # Add query for fetching respective TranslatedResource.\n                    tr_filter.append(\n                        Q(locale__id=locale.id) & Q(resource__id=entity.resource.id)\n                    )\n\n                # Update the latest translation index\n                tr_dict[locale_resource] = index\n\n        log.info(f\"Fetching pretranslations for locale {locale.code} done\")\n\n    if len(translations) == 0:\n        return\n\n    translations = Translation.objects.bulk_create(translations)\n\n    # Log creating actions\n    actions_to_log = [\n        ActionLog(\n            action_type=ActionLog.ActionType.TRANSLATION_CREATED,\n            performed_by=t.user,\n            translation=t,\n        )\n        for t in translations\n    ]\n\n    ActionLog.objects.bulk_create(actions_to_log)\n\n    # Run checks on all translations\n    translation_pks = {translation.pk for translation in translations}\n    bulk_run_checks(Translation.objects.for_checks().filter(pk__in=translation_pks))\n\n    # Mark translations as changed\n    changed_translations = Translation.objects.filter(\n        pk__in=translation_pks,\n        # Do not sync translations with errors and warnings\n        errors__isnull=True,\n        warnings__isnull=True,\n    )\n    changed_translations.bulk_mark_changed()\n\n    # Update latest activity and stats for changed instances.\n    update_changed_instances(tr_filter, tr_dict, translations)\n\n    log.info(f\"Fetching pretranslations for project {project.name} done\")\n", "path": "pontoon/pretranslation/tasks.py"}]}