We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Severe performance degradation of sync_stores
Earlier today we updated production translation server (merged 'Raw font' PR branch with master and switched to master). So this release includes recent changes related to optimizations. Immediately after that our sync cycle time increased from typical 18-20 minutes to 1.5 hours. I had to write a tool to analyze and compare our logs from past sync cycles and extract timing information. Here's the output:

![screenclip](https://cloud.githubusercontent.com/assets/1728158/18117500/035f5506-6f04-11e6-98db-5303e76872f7.png)

Some comments on the screenshot. It compares our main steps of sync cycle between four logs (vertical columns). The first log is from a morning run, some time before the release. The second one is soon after the release, and the last two are most recent ones. As one might see, the `pull-ts` step is the culprit. It went from mere 3 minutes up to more than an hour. During this step all we do is run `manage.py sync_stores --skip-missing --project=<project_id>` and wait for it to finish.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pootle/apps/pootle_store/syncer.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (C) Pootle contributors.
4 #
5 # This file is a part of the Pootle project. It is distributed under the GPL3
6 # or later license. See the LICENSE file for a copy of the license and the
7 # AUTHORS file for copyright and authorship information.
8 
9 import logging
10 import os
11 from collections import namedtuple
12 
13 from translate.storage.factory import getclass
14 
15 from django.utils.functional import cached_property
16 
17 from pootle.core.delegate import format_classes
18 from pootle.core.log import log
19 from pootle.core.url_helpers import split_pootle_path
20 
21 from .models import Unit
22 from .util import get_change_str
23 
24 
25 class UnitSyncer(object):
26 
27     def __init__(self, unit):
28         self.unit = unit
29 
30     @property
31     def context(self):
32         return self.unit.getcontext()
33 
34     @property
35     def developer_notes(self):
36         return self.unit.getnotes(origin="developer")
37 
38     @property
39     def isfuzzy(self):
40         return self.unit.isfuzzy()
41 
42     @property
43     def isobsolete(self):
44         return self.unit.isobsolete()
45 
46     @property
47     def locations(self):
48         return self.unit.getlocations()
49 
50     @property
51     def source(self):
52         return self.unit.source
53 
54     @property
55     def target(self):
56         return self.unit.target
57 
58     @property
59     def translator_notes(self):
60         return self.unit.getnotes(origin="translator")
61 
62     @property
63     def unitid(self):
64         return self.unit.getid()
65 
66     @property
67     def unit_class(self):
68         return self.unit.store.syncer.unit_class
69 
70     def convert(self, unitclass=None):
71         newunit = self.create_unit(
72             unitclass or self.unit_class)
73         self.set_target(newunit)
74         self.set_fuzzy(newunit)
75         self.set_locations(newunit)
76         self.set_developer_notes(newunit)
77         self.set_translator_notes(newunit)
78         self.set_unitid(newunit)
79         self.set_context(newunit)
80         self.set_obsolete(newunit)
81         return newunit
82 
83     def create_unit(self, unitclass):
84         return unitclass(self.source)
85 
86     def set_context(self, newunit):
87         newunit.setcontext(self.context)
88 
89     def set_developer_notes(self, newunit):
90         notes = self.developer_notes
91         if notes:
92             newunit.addnote(notes, origin="developer")
93 
94     def set_fuzzy(self, newunit):
95         newunit.markfuzzy(self.isfuzzy)
96 
97     def set_locations(self, newunit):
98         locations = self.locations
99         if locations:
100             newunit.addlocations(locations)
101 
102     def set_obsolete(self, newunit):
103         if self.isobsolete:
104             newunit.makeobsolete()
105 
106     def set_target(self, newunit):
107         newunit.target = self.target
108 
109     def set_translator_notes(self, newunit):
110         notes = self.translator_notes
111         if notes:
112             newunit.addnote(notes, origin="translator")
113 
114     def set_unitid(self, newunit):
115         newunit.setid(self.unitid)
116 
117 
118 class StoreSyncer(object):
119     unit_sync_class = UnitSyncer
120 
121     def __init__(self, store):
122         self.store = store
123 
124     @cached_property
125     def disk_store(self):
126         return self.store.file.store
127 
128     @property
129     def translation_project(self):
130         return self.store.translation_project
131 
132     @property
133     def language(self):
134         return self.translation_project.language
135 
136     @property
137     def project(self):
138         return self.translation_project.project
139 
140     @property
141     def source_language(self):
142         return self.project.source_language
143 
144     @property
145     def store_file_path(self):
146         return os.path.join(
147             self.translation_project.abs_real_path,
148             *split_pootle_path(self.store.pootle_path)[2:])
149 
150     @property
151     def relative_file_path(self):
152         path_parts = split_pootle_path(self.store.pootle_path)
153         path_prefix = [path_parts[1]]
154         if self.project.get_treestyle() != "gnu":
155             path_prefix.append(path_parts[0])
156         return os.path.join(*(path_prefix + list(path_parts[2:])))
157 
158     @property
159     def unit_class(self):
160         return self.file_class.UnitClass
161 
162     @cached_property
163     def file_class(self):
164         # get a plugin adapted file_class
165         fileclass = format_classes.gather().get(
166             str(self.store.filetype.extension))
167         if fileclass:
168             return fileclass
169         if self.store.is_template:
170             # namedtuple is equiv here of object() with name attr
171             return self._getclass(
172                 namedtuple("instance", "name")(
173                     name=".".join(
174                         [os.path.splitext(self.store.name)[0],
175                          str(self.store.filetype.extension)])))
176         return self._getclass(self.store)
177 
178     def convert(self, fileclass=None):
179         """export to fileclass"""
180         fileclass = fileclass or self.file_class
181         logging.debug(
182             u"Converting %s to %s",
183             self.store.pootle_path,
184             fileclass)
185         output = fileclass()
186         output.settargetlanguage(self.language.code)
187         # FIXME: we should add some headers
188         for unit in self.store.units.iterator():
189             output.addunit(
190                 self.unit_sync_class(unit).convert(output.UnitClass))
191         return output
192 
193     def _getclass(self, obj):
194         try:
195             return getclass(obj)
196         except ValueError:
197             raise ValueError(
198                 "Unable to find conversion class for Store '%s'"
199                 % self.store.name)
200 
201     def get_new_units(self, old_ids, new_ids):
202         return self.store.findid_bulk(
203             [self.dbid_index.get(uid)
204              for uid
205              in new_ids - old_ids])
206 
207     def get_units_to_obsolete(self, old_ids, new_ids):
208         for uid in old_ids - new_ids:
209             unit = self.disk_store.findid(uid)
210             if unit and not unit.isobsolete():
211                 yield unit
212 
213     def obsolete_unit(self, unit, conservative):
214         deleted = not unit.istranslated()
215         obsoleted = (
216             not deleted
217             and not conservative)
218         if obsoleted:
219             unit.makeobsolete()
220             deleted = not unit.isobsolete()
221         if deleted:
222             del unit
223         return obsoleted, deleted
224 
225     def update_structure(self, obsolete_units, new_units, conservative):
226         obsolete = 0
227         deleted = 0
228         added = 0
229         for unit in obsolete_units:
230             _obsolete, _deleted = self.obsolete_unit(unit, conservative)
231             if _obsolete:
232                 obsolete += 1
233             if _deleted:
234                 deleted += 1
235         for unit in new_units:
236             newunit = unit.convert(self.disk_store.UnitClass)
237             self.disk_store.addunit(newunit)
238             added += 1
239         return obsolete, deleted, added
240 
241     def create_store_file(self, last_revision, user):
242         logging.debug(u"Creating file %s", self.store.pootle_path)
243         store = self.convert()
244         if not os.path.exists(os.path.dirname(self.store_file_path)):
245             os.makedirs(os.path.dirname(self.store_file_path))
246         self.store.file = self.relative_file_path
247         store.savefile(self.store_file_path)
248         log(u"Created file for %s [revision: %d]" %
249             (self.store.pootle_path, last_revision))
250         self.update_store_header(user=user)
251         self.store.file.savestore()
252         self.store.file_mtime = self.store.get_file_mtime()
253         self.store.last_sync_revision = last_revision
254         self.store.save()
255 
256     def update_newer(self, last_revision):
257         return (
258             not self.store.file.exists()
259             or (last_revision >= self.store.last_sync_revision))
260 
261     @cached_property
262     def dbid_index(self):
263         """build a quick mapping index between unit ids and database ids"""
264         return dict(
265             self.store.unit_set.live().values_list('unitid', 'id'))
266 
267     def sync(self, update_structure=False, conservative=True,
268              user=None, only_newer=True):
269         last_revision = self.store.get_max_unit_revision()
270 
271         # TODO only_newer -> not force
272         if only_newer and not self.update_newer(last_revision):
273             logging.info(
274                 u"[sync] No updates for %s after [revision: %d]",
275                 self.store.pootle_path, self.store.last_sync_revision)
276             return
277 
278         if not self.store.file.exists():
279             self.create_store_file(last_revision, user)
280             return
281 
282         if conservative and self.store.is_template:
283             return
284 
285         file_changed, changes = self.sync_store(
286             last_revision,
287             update_structure,
288             conservative)
289         self.save_store(
290             last_revision,
291             user,
292             changes,
293             (file_changed or not conservative))
294 
295     def sync_store(self, last_revision, update_structure, conservative):
296         logging.info(u"Syncing %s", self.store.pootle_path)
297         old_ids = set(self.disk_store.getids())
298         new_ids = set(self.dbid_index.keys())
299         file_changed = False
300         changes = {}
301         if update_structure:
302             obsolete_units = self.get_units_to_obsolete(old_ids, new_ids)
303             new_units = self.get_new_units(old_ids, new_ids)
304             if obsolete_units or new_units:
305                 file_changed = True
306                 (changes['obsolete'],
307                  changes['deleted'],
308                  changes['added']) = self.update_structure(
309                     obsolete_units,
310                     new_units,
311                     conservative=conservative)
312         changes["updated"] = self.sync_units(
313             self.get_common_units(
314                 set(self.dbid_index.get(uid)
315                     for uid
316                     in old_ids & new_ids),
317                 last_revision,
318                 conservative))
319         return bool(file_changed or any(changes.values())), changes
320 
321     def save_store(self, last_revision, user, changes, updated):
322         # TODO conservative -> not overwrite
323         if updated:
324             self.update_store_header(user=user)
325             self.store.file.savestore()
326             self.store.file_mtime = self.store.get_file_mtime()
327             log(u"[sync] File saved; %s units in %s [revision: %d]" %
328                 (get_change_str(changes),
329                  self.store.pootle_path,
330                  last_revision))
331         else:
332             logging.info(
333                 u"[sync] nothing changed in %s [revision: %d]",
334                 self.store.pootle_path,
335                 last_revision)
336         self.store.last_sync_revision = last_revision
337         self.store.save()
338 
339     def get_revision_filters(self, last_revision):
340         # Get units modified after last sync and before this sync started
341         filter_by = {
342             'revision__lte': last_revision,
343             'store': self.store}
344         # Sync all units if first sync
345         if self.store.last_sync_revision is not None:
346             filter_by.update({'revision__gt': self.store.last_sync_revision})
347         return filter_by
348 
349     def get_modified_units(self, last_revision):
350         return set(
351             Unit.objects.filter(**self.get_revision_filters(last_revision))
352                         .values_list('id', flat=True).distinct()
353             if last_revision > self.store.last_sync_revision
354             else [])
355 
356     def get_common_units(self, common_dbids, last_revision, conservative):
357         if conservative:
358             # Sync only modified units
359             common_dbids &= self.get_modified_units(last_revision)
360         return self.store.findid_bulk(list(common_dbids))
361 
362     def sync_units(self, units):
363         updated = 0
364         for unit in units:
365             match = self.disk_store.findid(unit.getid())
366             if match is not None:
367                 changed = unit.sync(match)
368                 if changed:
369                     updated += 1
370         return updated
371 
372     def update_store_header(self, **kwargs_):
373         self.disk_store.settargetlanguage(self.language.code)
374         self.disk_store.setsourcelanguage(self.source_language.code)
375
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pootle/apps/pootle_store/syncer.py b/pootle/apps/pootle_store/syncer.py
--- a/pootle/apps/pootle_store/syncer.py
+++ b/pootle/apps/pootle_store/syncer.py
@@ -256,7 +256,8 @@
     def update_newer(self, last_revision):
         return (
             not self.store.file.exists()
-            or (last_revision >= self.store.last_sync_revision))
+            or last_revision > self.store.last_sync_revision
+        )
 
     @cached_property
     def dbid_index(self):

{"golden_diff": "diff --git a/pootle/apps/pootle_store/syncer.py b/pootle/apps/pootle_store/syncer.py\n--- a/pootle/apps/pootle_store/syncer.py\n+++ b/pootle/apps/pootle_store/syncer.py\n@@ -256,7 +256,8 @@\n     def update_newer(self, last_revision):\n         return (\n             not self.store.file.exists()\n-            or (last_revision >= self.store.last_sync_revision))\n+            or last_revision > self.store.last_sync_revision\n+        )\n \n     @cached_property\n     def dbid_index(self):\n", "issue": "Severe performance degradation of sync_stores\nEarlier today we updated production translation server (merged 'Raw font' PR branch with master and switched to master). So this release includes recent changes related to optimizations. Immediately after that our sync cycle time increased from typical 18-20 minutes to 1.5 hours. I had to write a tool to analyze and compare our logs from past sync cycles and extract timing information. Here's the output:\n\n![screenclip](https://cloud.githubusercontent.com/assets/1728158/18117500/035f5506-6f04-11e6-98db-5303e76872f7.png)\n\nSome comments on the screenshot. It compares our main steps of sync cycle between four logs (vertical columns). The first log is from a morning run, some time before the release. The second one is soon after the release, and the last two are most recent ones. As one might see, the `pull-ts` step is the culprit. It went from mere 3 minutes up to more than an hour. During this step all we do is run `manage.py sync_stores --skip-missing --project=<project_id>` and wait for it to finish.\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nimport logging\nimport os\nfrom collections import namedtuple\n\nfrom translate.storage.factory import getclass\n\nfrom django.utils.functional import cached_property\n\nfrom pootle.core.delegate import format_classes\nfrom pootle.core.log import log\nfrom pootle.core.url_helpers import split_pootle_path\n\nfrom .models import Unit\nfrom .util import get_change_str\n\n\nclass UnitSyncer(object):\n\n    def __init__(self, unit):\n        self.unit = unit\n\n    @property\n    def context(self):\n        return self.unit.getcontext()\n\n    @property\n    def developer_notes(self):\n        return self.unit.getnotes(origin=\"developer\")\n\n    @property\n    def isfuzzy(self):\n        return self.unit.isfuzzy()\n\n    @property\n    def isobsolete(self):\n        return self.unit.isobsolete()\n\n    @property\n    def locations(self):\n        return self.unit.getlocations()\n\n    @property\n    def source(self):\n        return self.unit.source\n\n    @property\n    def target(self):\n        return self.unit.target\n\n    @property\n    def translator_notes(self):\n        return self.unit.getnotes(origin=\"translator\")\n\n    @property\n    def unitid(self):\n        return self.unit.getid()\n\n    @property\n    def unit_class(self):\n        return self.unit.store.syncer.unit_class\n\n    def convert(self, unitclass=None):\n        newunit = self.create_unit(\n            unitclass or self.unit_class)\n        self.set_target(newunit)\n        self.set_fuzzy(newunit)\n        self.set_locations(newunit)\n        self.set_developer_notes(newunit)\n        self.set_translator_notes(newunit)\n        self.set_unitid(newunit)\n        self.set_context(newunit)\n        self.set_obsolete(newunit)\n        return newunit\n\n    def create_unit(self, unitclass):\n        return unitclass(self.source)\n\n    def set_context(self, newunit):\n        newunit.setcontext(self.context)\n\n    def set_developer_notes(self, newunit):\n        notes = self.developer_notes\n        if notes:\n            newunit.addnote(notes, origin=\"developer\")\n\n    def set_fuzzy(self, newunit):\n        newunit.markfuzzy(self.isfuzzy)\n\n    def set_locations(self, newunit):\n        locations = self.locations\n        if locations:\n            newunit.addlocations(locations)\n\n    def set_obsolete(self, newunit):\n        if self.isobsolete:\n            newunit.makeobsolete()\n\n    def set_target(self, newunit):\n        newunit.target = self.target\n\n    def set_translator_notes(self, newunit):\n        notes = self.translator_notes\n        if notes:\n            newunit.addnote(notes, origin=\"translator\")\n\n    def set_unitid(self, newunit):\n        newunit.setid(self.unitid)\n\n\nclass StoreSyncer(object):\n    unit_sync_class = UnitSyncer\n\n    def __init__(self, store):\n        self.store = store\n\n    @cached_property\n    def disk_store(self):\n        return self.store.file.store\n\n    @property\n    def translation_project(self):\n        return self.store.translation_project\n\n    @property\n    def language(self):\n        return self.translation_project.language\n\n    @property\n    def project(self):\n        return self.translation_project.project\n\n    @property\n    def source_language(self):\n        return self.project.source_language\n\n    @property\n    def store_file_path(self):\n        return os.path.join(\n            self.translation_project.abs_real_path,\n            *split_pootle_path(self.store.pootle_path)[2:])\n\n    @property\n    def relative_file_path(self):\n        path_parts = split_pootle_path(self.store.pootle_path)\n        path_prefix = [path_parts[1]]\n        if self.project.get_treestyle() != \"gnu\":\n            path_prefix.append(path_parts[0])\n        return os.path.join(*(path_prefix + list(path_parts[2:])))\n\n    @property\n    def unit_class(self):\n        return self.file_class.UnitClass\n\n    @cached_property\n    def file_class(self):\n        # get a plugin adapted file_class\n        fileclass = format_classes.gather().get(\n            str(self.store.filetype.extension))\n        if fileclass:\n            return fileclass\n        if self.store.is_template:\n            # namedtuple is equiv here of object() with name attr\n            return self._getclass(\n                namedtuple(\"instance\", \"name\")(\n                    name=\".\".join(\n                        [os.path.splitext(self.store.name)[0],\n                         str(self.store.filetype.extension)])))\n        return self._getclass(self.store)\n\n    def convert(self, fileclass=None):\n        \"\"\"export to fileclass\"\"\"\n        fileclass = fileclass or self.file_class\n        logging.debug(\n            u\"Converting %s to %s\",\n            self.store.pootle_path,\n            fileclass)\n        output = fileclass()\n        output.settargetlanguage(self.language.code)\n        # FIXME: we should add some headers\n        for unit in self.store.units.iterator():\n            output.addunit(\n                self.unit_sync_class(unit).convert(output.UnitClass))\n        return output\n\n    def _getclass(self, obj):\n        try:\n            return getclass(obj)\n        except ValueError:\n            raise ValueError(\n                \"Unable to find conversion class for Store '%s'\"\n                % self.store.name)\n\n    def get_new_units(self, old_ids, new_ids):\n        return self.store.findid_bulk(\n            [self.dbid_index.get(uid)\n             for uid\n             in new_ids - old_ids])\n\n    def get_units_to_obsolete(self, old_ids, new_ids):\n        for uid in old_ids - new_ids:\n            unit = self.disk_store.findid(uid)\n            if unit and not unit.isobsolete():\n                yield unit\n\n    def obsolete_unit(self, unit, conservative):\n        deleted = not unit.istranslated()\n        obsoleted = (\n            not deleted\n            and not conservative)\n        if obsoleted:\n            unit.makeobsolete()\n            deleted = not unit.isobsolete()\n        if deleted:\n            del unit\n        return obsoleted, deleted\n\n    def update_structure(self, obsolete_units, new_units, conservative):\n        obsolete = 0\n        deleted = 0\n        added = 0\n        for unit in obsolete_units:\n            _obsolete, _deleted = self.obsolete_unit(unit, conservative)\n            if _obsolete:\n                obsolete += 1\n            if _deleted:\n                deleted += 1\n        for unit in new_units:\n            newunit = unit.convert(self.disk_store.UnitClass)\n            self.disk_store.addunit(newunit)\n            added += 1\n        return obsolete, deleted, added\n\n    def create_store_file(self, last_revision, user):\n        logging.debug(u\"Creating file %s\", self.store.pootle_path)\n        store = self.convert()\n        if not os.path.exists(os.path.dirname(self.store_file_path)):\n            os.makedirs(os.path.dirname(self.store_file_path))\n        self.store.file = self.relative_file_path\n        store.savefile(self.store_file_path)\n        log(u\"Created file for %s [revision: %d]\" %\n            (self.store.pootle_path, last_revision))\n        self.update_store_header(user=user)\n        self.store.file.savestore()\n        self.store.file_mtime = self.store.get_file_mtime()\n        self.store.last_sync_revision = last_revision\n        self.store.save()\n\n    def update_newer(self, last_revision):\n        return (\n            not self.store.file.exists()\n            or (last_revision >= self.store.last_sync_revision))\n\n    @cached_property\n    def dbid_index(self):\n        \"\"\"build a quick mapping index between unit ids and database ids\"\"\"\n        return dict(\n            self.store.unit_set.live().values_list('unitid', 'id'))\n\n    def sync(self, update_structure=False, conservative=True,\n             user=None, only_newer=True):\n        last_revision = self.store.get_max_unit_revision()\n\n        # TODO only_newer -> not force\n        if only_newer and not self.update_newer(last_revision):\n            logging.info(\n                u\"[sync] No updates for %s after [revision: %d]\",\n                self.store.pootle_path, self.store.last_sync_revision)\n            return\n\n        if not self.store.file.exists():\n            self.create_store_file(last_revision, user)\n            return\n\n        if conservative and self.store.is_template:\n            return\n\n        file_changed, changes = self.sync_store(\n            last_revision,\n            update_structure,\n            conservative)\n        self.save_store(\n            last_revision,\n            user,\n            changes,\n            (file_changed or not conservative))\n\n    def sync_store(self, last_revision, update_structure, conservative):\n        logging.info(u\"Syncing %s\", self.store.pootle_path)\n        old_ids = set(self.disk_store.getids())\n        new_ids = set(self.dbid_index.keys())\n        file_changed = False\n        changes = {}\n        if update_structure:\n            obsolete_units = self.get_units_to_obsolete(old_ids, new_ids)\n            new_units = self.get_new_units(old_ids, new_ids)\n            if obsolete_units or new_units:\n                file_changed = True\n                (changes['obsolete'],\n                 changes['deleted'],\n                 changes['added']) = self.update_structure(\n                    obsolete_units,\n                    new_units,\n                    conservative=conservative)\n        changes[\"updated\"] = self.sync_units(\n            self.get_common_units(\n                set(self.dbid_index.get(uid)\n                    for uid\n                    in old_ids & new_ids),\n                last_revision,\n                conservative))\n        return bool(file_changed or any(changes.values())), changes\n\n    def save_store(self, last_revision, user, changes, updated):\n        # TODO conservative -> not overwrite\n        if updated:\n            self.update_store_header(user=user)\n            self.store.file.savestore()\n            self.store.file_mtime = self.store.get_file_mtime()\n            log(u\"[sync] File saved; %s units in %s [revision: %d]\" %\n                (get_change_str(changes),\n                 self.store.pootle_path,\n                 last_revision))\n        else:\n            logging.info(\n                u\"[sync] nothing changed in %s [revision: %d]\",\n                self.store.pootle_path,\n                last_revision)\n        self.store.last_sync_revision = last_revision\n        self.store.save()\n\n    def get_revision_filters(self, last_revision):\n        # Get units modified after last sync and before this sync started\n        filter_by = {\n            'revision__lte': last_revision,\n            'store': self.store}\n        # Sync all units if first sync\n        if self.store.last_sync_revision is not None:\n            filter_by.update({'revision__gt': self.store.last_sync_revision})\n        return filter_by\n\n    def get_modified_units(self, last_revision):\n        return set(\n            Unit.objects.filter(**self.get_revision_filters(last_revision))\n                        .values_list('id', flat=True).distinct()\n            if last_revision > self.store.last_sync_revision\n            else [])\n\n    def get_common_units(self, common_dbids, last_revision, conservative):\n        if conservative:\n            # Sync only modified units\n            common_dbids &= self.get_modified_units(last_revision)\n        return self.store.findid_bulk(list(common_dbids))\n\n    def sync_units(self, units):\n        updated = 0\n        for unit in units:\n            match = self.disk_store.findid(unit.getid())\n            if match is not None:\n                changed = unit.sync(match)\n                if changed:\n                    updated += 1\n        return updated\n\n    def update_store_header(self, **kwargs_):\n        self.disk_store.settargetlanguage(self.language.code)\n        self.disk_store.setsourcelanguage(self.source_language.code)\n", "path": "pootle/apps/pootle_store/syncer.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nimport logging\nimport os\nfrom collections import namedtuple\n\nfrom translate.storage.factory import getclass\n\nfrom django.utils.functional import cached_property\n\nfrom pootle.core.delegate import format_classes\nfrom pootle.core.log import log\nfrom pootle.core.url_helpers import split_pootle_path\n\nfrom .models import Unit\nfrom .util import get_change_str\n\n\nclass UnitSyncer(object):\n\n    def __init__(self, unit):\n        self.unit = unit\n\n    @property\n    def context(self):\n        return self.unit.getcontext()\n\n    @property\n    def developer_notes(self):\n        return self.unit.getnotes(origin=\"developer\")\n\n    @property\n    def isfuzzy(self):\n        return self.unit.isfuzzy()\n\n    @property\n    def isobsolete(self):\n        return self.unit.isobsolete()\n\n    @property\n    def locations(self):\n        return self.unit.getlocations()\n\n    @property\n    def source(self):\n        return self.unit.source\n\n    @property\n    def target(self):\n        return self.unit.target\n\n    @property\n    def translator_notes(self):\n        return self.unit.getnotes(origin=\"translator\")\n\n    @property\n    def unitid(self):\n        return self.unit.getid()\n\n    @property\n    def unit_class(self):\n        return self.unit.store.syncer.unit_class\n\n    def convert(self, unitclass=None):\n        newunit = self.create_unit(\n            unitclass or self.unit_class)\n        self.set_target(newunit)\n        self.set_fuzzy(newunit)\n        self.set_locations(newunit)\n        self.set_developer_notes(newunit)\n        self.set_translator_notes(newunit)\n        self.set_unitid(newunit)\n        self.set_context(newunit)\n        self.set_obsolete(newunit)\n        return newunit\n\n    def create_unit(self, unitclass):\n        return unitclass(self.source)\n\n    def set_context(self, newunit):\n        newunit.setcontext(self.context)\n\n    def set_developer_notes(self, newunit):\n        notes = self.developer_notes\n        if notes:\n            newunit.addnote(notes, origin=\"developer\")\n\n    def set_fuzzy(self, newunit):\n        newunit.markfuzzy(self.isfuzzy)\n\n    def set_locations(self, newunit):\n        locations = self.locations\n        if locations:\n            newunit.addlocations(locations)\n\n    def set_obsolete(self, newunit):\n        if self.isobsolete:\n            newunit.makeobsolete()\n\n    def set_target(self, newunit):\n        newunit.target = self.target\n\n    def set_translator_notes(self, newunit):\n        notes = self.translator_notes\n        if notes:\n            newunit.addnote(notes, origin=\"translator\")\n\n    def set_unitid(self, newunit):\n        newunit.setid(self.unitid)\n\n\nclass StoreSyncer(object):\n    unit_sync_class = UnitSyncer\n\n    def __init__(self, store):\n        self.store = store\n\n    @cached_property\n    def disk_store(self):\n        return self.store.file.store\n\n    @property\n    def translation_project(self):\n        return self.store.translation_project\n\n    @property\n    def language(self):\n        return self.translation_project.language\n\n    @property\n    def project(self):\n        return self.translation_project.project\n\n    @property\n    def source_language(self):\n        return self.project.source_language\n\n    @property\n    def store_file_path(self):\n        return os.path.join(\n            self.translation_project.abs_real_path,\n            *split_pootle_path(self.store.pootle_path)[2:])\n\n    @property\n    def relative_file_path(self):\n        path_parts = split_pootle_path(self.store.pootle_path)\n        path_prefix = [path_parts[1]]\n        if self.project.get_treestyle() != \"gnu\":\n            path_prefix.append(path_parts[0])\n        return os.path.join(*(path_prefix + list(path_parts[2:])))\n\n    @property\n    def unit_class(self):\n        return self.file_class.UnitClass\n\n    @cached_property\n    def file_class(self):\n        # get a plugin adapted file_class\n        fileclass = format_classes.gather().get(\n            str(self.store.filetype.extension))\n        if fileclass:\n            return fileclass\n        if self.store.is_template:\n            # namedtuple is equiv here of object() with name attr\n            return self._getclass(\n                namedtuple(\"instance\", \"name\")(\n                    name=\".\".join(\n                        [os.path.splitext(self.store.name)[0],\n                         str(self.store.filetype.extension)])))\n        return self._getclass(self.store)\n\n    def convert(self, fileclass=None):\n        \"\"\"export to fileclass\"\"\"\n        fileclass = fileclass or self.file_class\n        logging.debug(\n            u\"Converting %s to %s\",\n            self.store.pootle_path,\n            fileclass)\n        output = fileclass()\n        output.settargetlanguage(self.language.code)\n        # FIXME: we should add some headers\n        for unit in self.store.units.iterator():\n            output.addunit(\n                self.unit_sync_class(unit).convert(output.UnitClass))\n        return output\n\n    def _getclass(self, obj):\n        try:\n            return getclass(obj)\n        except ValueError:\n            raise ValueError(\n                \"Unable to find conversion class for Store '%s'\"\n                % self.store.name)\n\n    def get_new_units(self, old_ids, new_ids):\n        return self.store.findid_bulk(\n            [self.dbid_index.get(uid)\n             for uid\n             in new_ids - old_ids])\n\n    def get_units_to_obsolete(self, old_ids, new_ids):\n        for uid in old_ids - new_ids:\n            unit = self.disk_store.findid(uid)\n            if unit and not unit.isobsolete():\n                yield unit\n\n    def obsolete_unit(self, unit, conservative):\n        deleted = not unit.istranslated()\n        obsoleted = (\n            not deleted\n            and not conservative)\n        if obsoleted:\n            unit.makeobsolete()\n            deleted = not unit.isobsolete()\n        if deleted:\n            del unit\n        return obsoleted, deleted\n\n    def update_structure(self, obsolete_units, new_units, conservative):\n        obsolete = 0\n        deleted = 0\n        added = 0\n        for unit in obsolete_units:\n            _obsolete, _deleted = self.obsolete_unit(unit, conservative)\n            if _obsolete:\n                obsolete += 1\n            if _deleted:\n                deleted += 1\n        for unit in new_units:\n            newunit = unit.convert(self.disk_store.UnitClass)\n            self.disk_store.addunit(newunit)\n            added += 1\n        return obsolete, deleted, added\n\n    def create_store_file(self, last_revision, user):\n        logging.debug(u\"Creating file %s\", self.store.pootle_path)\n        store = self.convert()\n        if not os.path.exists(os.path.dirname(self.store_file_path)):\n            os.makedirs(os.path.dirname(self.store_file_path))\n        self.store.file = self.relative_file_path\n        store.savefile(self.store_file_path)\n        log(u\"Created file for %s [revision: %d]\" %\n            (self.store.pootle_path, last_revision))\n        self.update_store_header(user=user)\n        self.store.file.savestore()\n        self.store.file_mtime = self.store.get_file_mtime()\n        self.store.last_sync_revision = last_revision\n        self.store.save()\n\n    def update_newer(self, last_revision):\n        return (\n            not self.store.file.exists()\n            or last_revision > self.store.last_sync_revision\n        )\n\n    @cached_property\n    def dbid_index(self):\n        \"\"\"build a quick mapping index between unit ids and database ids\"\"\"\n        return dict(\n            self.store.unit_set.live().values_list('unitid', 'id'))\n\n    def sync(self, update_structure=False, conservative=True,\n             user=None, only_newer=True):\n        last_revision = self.store.get_max_unit_revision()\n\n        # TODO only_newer -> not force\n        if only_newer and not self.update_newer(last_revision):\n            logging.info(\n                u\"[sync] No updates for %s after [revision: %d]\",\n                self.store.pootle_path, self.store.last_sync_revision)\n            return\n\n        if not self.store.file.exists():\n            self.create_store_file(last_revision, user)\n            return\n\n        if conservative and self.store.is_template:\n            return\n\n        file_changed, changes = self.sync_store(\n            last_revision,\n            update_structure,\n            conservative)\n        self.save_store(\n            last_revision,\n            user,\n            changes,\n            (file_changed or not conservative))\n\n    def sync_store(self, last_revision, update_structure, conservative):\n        logging.info(u\"Syncing %s\", self.store.pootle_path)\n        old_ids = set(self.disk_store.getids())\n        new_ids = set(self.dbid_index.keys())\n        file_changed = False\n        changes = {}\n        if update_structure:\n            obsolete_units = self.get_units_to_obsolete(old_ids, new_ids)\n            new_units = self.get_new_units(old_ids, new_ids)\n            if obsolete_units or new_units:\n                file_changed = True\n                (changes['obsolete'],\n                 changes['deleted'],\n                 changes['added']) = self.update_structure(\n                    obsolete_units,\n                    new_units,\n                    conservative=conservative)\n        changes[\"updated\"] = self.sync_units(\n            self.get_common_units(\n                set(self.dbid_index.get(uid)\n                    for uid\n                    in old_ids & new_ids),\n                last_revision,\n                conservative))\n        return bool(file_changed or any(changes.values())), changes\n\n    def save_store(self, last_revision, user, changes, updated):\n        # TODO conservative -> not overwrite\n        if updated:\n            self.update_store_header(user=user)\n            self.store.file.savestore()\n            self.store.file_mtime = self.store.get_file_mtime()\n            log(u\"[sync] File saved; %s units in %s [revision: %d]\" %\n                (get_change_str(changes),\n                 self.store.pootle_path,\n                 last_revision))\n        else:\n            logging.info(\n                u\"[sync] nothing changed in %s [revision: %d]\",\n                self.store.pootle_path,\n                last_revision)\n        self.store.last_sync_revision = last_revision\n        self.store.save()\n\n    def get_revision_filters(self, last_revision):\n        # Get units modified after last sync and before this sync started\n        filter_by = {\n            'revision__lte': last_revision,\n            'store': self.store}\n        # Sync all units if first sync\n        if self.store.last_sync_revision is not None:\n            filter_by.update({'revision__gt': self.store.last_sync_revision})\n        return filter_by\n\n    def get_modified_units(self, last_revision):\n        return set(\n            Unit.objects.filter(**self.get_revision_filters(last_revision))\n                        .values_list('id', flat=True).distinct()\n            if last_revision > self.store.last_sync_revision\n            else [])\n\n    def get_common_units(self, common_dbids, last_revision, conservative):\n        if conservative:\n            # Sync only modified units\n            common_dbids &= self.get_modified_units(last_revision)\n        return self.store.findid_bulk(list(common_dbids))\n\n    def sync_units(self, units):\n        updated = 0\n        for unit in units:\n            match = self.disk_store.findid(unit.getid())\n            if match is not None:\n                changed = unit.sync(match)\n                if changed:\n                    updated += 1\n        return updated\n\n    def update_store_header(self, **kwargs_):\n        self.disk_store.settargetlanguage(self.language.code)\n        self.disk_store.setsourcelanguage(self.source_language.code)\n", "path": "pootle/apps/pootle_store/syncer.py"}]}