We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
update_tmserver shouldn't use disabled projects
`update_tmserver` command shouldn't use disabled projects unless they are specifically requested.

By default we shouldn't use them.  If the user supplies a command line option or uses a specific project name they we should load that project into the TM.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pootle/apps/pootle_app/management/commands/update_tmserver.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright (C) Pootle contributors.
5 #
6 # This file is a part of the Pootle project. It is distributed under the GPL3
7 # or later license. See the LICENSE file for a copy of the license and the
8 # AUTHORS file for copyright and authorship information.
9 
10 from hashlib import md5
11 from optparse import make_option
12 import os
13 import sys
14 
15 # This must be run before importing Django.
16 os.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'
17 
18 from elasticsearch import helpers, Elasticsearch
19 from translate.storage import factory
20 
21 from django.conf import settings
22 from django.core.management.base import BaseCommand, CommandError
23 
24 from pootle_store.models import Unit
25 
26 
27 BULK_CHUNK_SIZE = 5000
28 
29 
30 class DBParser(object):
31 
32     def __init__(self, *args, **kwargs):
33         self.stdout = kwargs.pop('stdout')
34         self.INDEX_NAME = kwargs.pop('index', None)
35 
36     def get_units(self, *filenames):
37         """Gets the units to import and its total count."""
38         units_qs = Unit.simple_objects \
39             .exclude(target_f__isnull=True) \
40             .exclude(target_f__exact='') \
41             .filter(revision__gt=self.last_indexed_revision) \
42             .select_related(
43                 'submitted_by',
44                 'store',
45                 'store__translation_project__project',
46                 'store__translation_project__language'
47             ).values(
48                 'id',
49                 'revision',
50                 'source_f',
51                 'target_f',
52                 'submitted_by__username',
53                 'submitted_by__full_name',
54                 'submitted_by__email',
55                 'store__translation_project__project__fullname',
56                 'store__pootle_path',
57                 'store__translation_project__language__code'
58             ).order_by()
59 
60         return units_qs.iterator(), units_qs.count()
61 
62     def get_unit_data(self, unit):
63         """Return dict with data to import for a single unit."""
64         fullname = (unit['submitted_by__full_name'] or
65                     unit['submitted_by__username'])
66 
67         email_md5 = None
68         if unit['submitted_by__email']:
69             email_md5 = md5(unit['submitted_by__email']).hexdigest()
70 
71         return {
72             '_index': self.INDEX_NAME,
73             '_type': unit['store__translation_project__language__code'],
74             '_id': unit['id'],
75             'revision': int(unit['revision']),
76             'project': unit['store__translation_project__project__fullname'],
77             'path': unit['store__pootle_path'],
78             'username': unit['submitted_by__username'],
79             'fullname': fullname,
80             'email_md5': email_md5,
81             'source': unit['source_f'],
82             'target': unit['target_f'],
83         }
84 
85 
86 class FileParser(object):
87 
88     def __init__(self, *args, **kwargs):
89         self.stdout = kwargs.pop('stdout')
90         self.INDEX_NAME = kwargs.pop('index', None)
91         self.target_language = kwargs.pop('language', None)
92         self.project = kwargs.pop('project', None)
93 
94     def get_units(self, *filenames):
95         """Gets the units to import and its total count."""
96         units = []
97         all_filenames = set()
98 
99         for filename in filenames:
100             if not os.path.exists(filename):
101                 self.stdout.write("File %s doesn't exist. Skipping it." %
102                                   filename)
103                 continue
104 
105             if os.path.isdir(filename):
106                 for dirpath, dirs, fnames in os.walk(filename):
107                     if (os.path.basename(dirpath) in
108                         ["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]):
109 
110                         continue
111 
112                     for f in fnames:
113                         all_filenames.add(os.path.join(dirpath, f))
114             else:
115                 all_filenames.add(filename)
116 
117         for filename in all_filenames:
118             store = factory.getobject(filename)
119             if not store.gettargetlanguage() and not self.target_language:
120                 raise CommandError("Unable to determine target language for "
121                                    "'%s'. Try again specifying a fallback "
122                                    "target language with --target-language" %
123                                    filename)
124 
125             self.filename = filename
126             units.extend([unit for unit in store.units if unit.istranslated()])
127 
128         return units, len(units)
129 
130     def get_unit_data(self, unit):
131         """Return dict with data to import for a single unit."""
132         target_language = unit.gettargetlanguage()
133         if target_language is None:
134             target_language = self.target_language
135 
136         return {
137             '_index': self.INDEX_NAME,
138             '_type': target_language,
139             '_id': unit.getid(),
140             'revision': 0,
141             'project': self.project,
142             'path': self.filename,
143             'username': None,
144             'fullname': None,
145             'email_md5': None,
146             'source': unit.source,
147             'target': unit.target,
148         }
149 
150 
151 class Command(BaseCommand):
152     help = "Load Translation Memory with translations"
153     args = "[files]"
154     option_list = BaseCommand.option_list + (
155         make_option('--refresh',
156                     action='store_true',
157                     dest='refresh',
158                     default=False,
159                     help='Process all items, not just the new ones, so '
160                          'existing translations are refreshed'),
161         make_option('--rebuild',
162                     action='store_true',
163                     dest='rebuild',
164                     default=False,
165                     help='Drop the entire TM on start and update everything '
166                          'from scratch'),
167         make_option('--dry-run',
168                     action='store_true',
169                     dest='dry_run',
170                     default=False,
171                     help='Report the number of translations to index and quit'),
172         # External TM specific options.
173         make_option('--tm',
174                     action='store',
175                     dest='tm',
176                     default='local',
177                     help="TM to use. TM must exist on settings. TM will be "
178                          "created on the server if it doesn't exist"),
179         make_option('--target-language',
180                     action='store',
181                     dest='target_language',
182                     default='',
183                     help="Target language to fallback to use in case it can't "
184                          "be guessed for any of the input files."),
185         make_option('--display-name',
186                     action='store',
187                     dest='project',
188                     default='',
189                     help='Name used when displaying TM matches for these '
190                          'translations.'),
191     )
192 
193     def _parse_translations(self, *args, **options):
194         units, total = self.parser.get_units(*args)
195 
196         if total == 0:
197             self.stdout.write("No translations to index")
198             sys.exit()
199 
200         self.stdout.write("%s translations to index" % total)
201 
202         if options['dry_run']:
203             sys.exit()
204 
205         self.stdout.write("")
206 
207         for i, unit in enumerate(units, start=1):
208             if (i % 1000 == 0) or (i == total):
209                 percent = "%.1f" % (i * 100.0 / total)
210                 self.stdout.write("%s (%s%%)" % (i, percent), ending='\r')
211                 self.stdout.flush()
212 
213             yield self.parser.get_unit_data(unit)
214 
215         if i != total:
216             self.stdout.write("Expected %d, loaded %d." % (total, i))
217 
218     def _initialize(self, *args, **options):
219         if not getattr(settings, 'POOTLE_TM_SERVER', False):
220             raise CommandError('POOTLE_TM_SERVER setting is missing.')
221 
222         try:
223             self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]
224         except KeyError:
225             raise CommandError("Translation Memory '%s' is not defined in the "
226                                "POOTLE_TM_SERVER setting. Please ensure it "
227                                "exists and double-check you typed it "
228                                "correctly." % options.get('tm'))
229 
230         self.INDEX_NAME = self.tm_settings['INDEX_NAME']
231         self.is_local_tm = options.get('tm') == 'local'
232 
233         self.es = Elasticsearch([{
234                 'host': self.tm_settings['HOST'],
235                 'port': self.tm_settings['PORT'],
236             }],
237             retry_on_timeout=True
238         )
239 
240         # If files to import have been provided.
241         if len(args):
242             if self.is_local_tm:
243                 raise CommandError('You cannot add translations from files to '
244                                    'a local TM.')
245 
246             self.target_language = options.pop('target_language')
247             self.project = options.pop('project')
248 
249             if not self.project:
250                 raise CommandError('You must specify a project with '
251                                    '--project.')
252             self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,
253                                      language=self.target_language,
254                                      project=self.project)
255         elif not self.is_local_tm:
256             raise CommandError('You cannot add translations from database to '
257                                'an external TM.')
258         else:
259             self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)
260 
261     def _set_latest_indexed_revision(self, **options):
262         self.last_indexed_revision = -1
263 
264         if (not options['rebuild'] and
265             not options['refresh'] and
266             self.es.indices.exists(self.INDEX_NAME)):
267 
268             result = self.es.search(
269                 index=self.INDEX_NAME,
270                 body={
271                     'query': {
272                         'match_all': {}
273                     },
274                     'facets': {
275                         'stat1': {
276                             'statistical': {
277                                 'field': 'revision'
278                             }
279                         }
280                     }
281                 }
282             )
283             self.last_indexed_revision = result['facets']['stat1']['max']
284 
285         self.parser.last_indexed_revision = self.last_indexed_revision
286 
287         self.stdout.write("Last indexed revision = %s" %
288                           self.last_indexed_revision)
289 
290     def handle(self, *args, **options):
291         self._initialize(*args, **options)
292 
293         if (options['rebuild'] and
294             not options['dry_run'] and
295             self.es.indices.exists(self.INDEX_NAME)):
296 
297             self.es.indices.delete(index=self.INDEX_NAME)
298 
299         if (not options['dry_run'] and
300             not self.es.indices.exists(self.INDEX_NAME)):
301 
302             self.es.indices.create(index=self.INDEX_NAME)
303 
304         if self.is_local_tm:
305             self._set_latest_indexed_revision(**options)
306 
307         success, _ = helpers.bulk(self.es,
308                                   self._parse_translations(*args, **options))
309
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pootle/apps/pootle_app/management/commands/update_tmserver.py b/pootle/apps/pootle_app/management/commands/update_tmserver.py
--- a/pootle/apps/pootle_app/management/commands/update_tmserver.py
+++ b/pootle/apps/pootle_app/management/commands/update_tmserver.py
@@ -32,6 +32,7 @@
     def __init__(self, *args, **kwargs):
         self.stdout = kwargs.pop('stdout')
         self.INDEX_NAME = kwargs.pop('index', None)
+        self.exclude_disabled_projects = not kwargs.pop('disabled_projects')
 
     def get_units(self, *filenames):
         """Gets the units to import and its total count."""
@@ -44,7 +45,14 @@
                 'store',
                 'store__translation_project__project',
                 'store__translation_project__language'
-            ).values(
+            )
+
+        if self.exclude_disabled_projects:
+            units_qs = units_qs.exclude(
+                store__translation_project__project__disabled=True
+            )
+
+        units_qs = units_qs.values(
                 'id',
                 'revision',
                 'source_f',
@@ -169,6 +177,12 @@
                     dest='dry_run',
                     default=False,
                     help='Report the number of translations to index and quit'),
+        # Local TM specific options.
+        make_option('--include-disabled-projects',
+                    action='store_true',
+                    dest='disabled_projects',
+                    default=False,
+                    help='Add translations from disabled projects'),
         # External TM specific options.
         make_option('--tm',
                     action='store',
@@ -256,7 +270,8 @@
             raise CommandError('You cannot add translations from database to '
                                'an external TM.')
         else:
-            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)
+            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME,
+                                   disabled_projects=options.pop('disabled_projects'))
 
     def _set_latest_indexed_revision(self, **options):
         self.last_indexed_revision = -1

{"golden_diff": "diff --git a/pootle/apps/pootle_app/management/commands/update_tmserver.py b/pootle/apps/pootle_app/management/commands/update_tmserver.py\n--- a/pootle/apps/pootle_app/management/commands/update_tmserver.py\n+++ b/pootle/apps/pootle_app/management/commands/update_tmserver.py\n@@ -32,6 +32,7 @@\n     def __init__(self, *args, **kwargs):\n         self.stdout = kwargs.pop('stdout')\n         self.INDEX_NAME = kwargs.pop('index', None)\n+        self.exclude_disabled_projects = not kwargs.pop('disabled_projects')\n \n     def get_units(self, *filenames):\n         \"\"\"Gets the units to import and its total count.\"\"\"\n@@ -44,7 +45,14 @@\n                 'store',\n                 'store__translation_project__project',\n                 'store__translation_project__language'\n-            ).values(\n+            )\n+\n+        if self.exclude_disabled_projects:\n+            units_qs = units_qs.exclude(\n+                store__translation_project__project__disabled=True\n+            )\n+\n+        units_qs = units_qs.values(\n                 'id',\n                 'revision',\n                 'source_f',\n@@ -169,6 +177,12 @@\n                     dest='dry_run',\n                     default=False,\n                     help='Report the number of translations to index and quit'),\n+        # Local TM specific options.\n+        make_option('--include-disabled-projects',\n+                    action='store_true',\n+                    dest='disabled_projects',\n+                    default=False,\n+                    help='Add translations from disabled projects'),\n         # External TM specific options.\n         make_option('--tm',\n                     action='store',\n@@ -256,7 +270,8 @@\n             raise CommandError('You cannot add translations from database to '\n                                'an external TM.')\n         else:\n-            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)\n+            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME,\n+                                   disabled_projects=options.pop('disabled_projects'))\n \n     def _set_latest_indexed_revision(self, **options):\n         self.last_indexed_revision = -1\n", "issue": "update_tmserver shouldn't use disabled projects\n`update_tmserver` command shouldn't use disabled projects unless they are specifically requested.\n\nBy default we shouldn't use them.  If the user supplies a command line option or uses a specific project name they we should load that project into the TM.\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nfrom hashlib import md5\nfrom optparse import make_option\nimport os\nimport sys\n\n# This must be run before importing Django.\nos.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'\n\nfrom elasticsearch import helpers, Elasticsearch\nfrom translate.storage import factory\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand, CommandError\n\nfrom pootle_store.models import Unit\n\n\nBULK_CHUNK_SIZE = 5000\n\n\nclass DBParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units_qs = Unit.simple_objects \\\n            .exclude(target_f__isnull=True) \\\n            .exclude(target_f__exact='') \\\n            .filter(revision__gt=self.last_indexed_revision) \\\n            .select_related(\n                'submitted_by',\n                'store',\n                'store__translation_project__project',\n                'store__translation_project__language'\n            ).values(\n                'id',\n                'revision',\n                'source_f',\n                'target_f',\n                'submitted_by__username',\n                'submitted_by__full_name',\n                'submitted_by__email',\n                'store__translation_project__project__fullname',\n                'store__pootle_path',\n                'store__translation_project__language__code'\n            ).order_by()\n\n        return units_qs.iterator(), units_qs.count()\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        fullname = (unit['submitted_by__full_name'] or\n                    unit['submitted_by__username'])\n\n        email_md5 = None\n        if unit['submitted_by__email']:\n            email_md5 = md5(unit['submitted_by__email']).hexdigest()\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': unit['store__translation_project__language__code'],\n            '_id': unit['id'],\n            'revision': int(unit['revision']),\n            'project': unit['store__translation_project__project__fullname'],\n            'path': unit['store__pootle_path'],\n            'username': unit['submitted_by__username'],\n            'fullname': fullname,\n            'email_md5': email_md5,\n            'source': unit['source_f'],\n            'target': unit['target_f'],\n        }\n\n\nclass FileParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n        self.target_language = kwargs.pop('language', None)\n        self.project = kwargs.pop('project', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units = []\n        all_filenames = set()\n\n        for filename in filenames:\n            if not os.path.exists(filename):\n                self.stdout.write(\"File %s doesn't exist. Skipping it.\" %\n                                  filename)\n                continue\n\n            if os.path.isdir(filename):\n                for dirpath, dirs, fnames in os.walk(filename):\n                    if (os.path.basename(dirpath) in\n                        [\"CVS\", \".svn\", \"_darcs\", \".git\", \".hg\", \".bzr\"]):\n\n                        continue\n\n                    for f in fnames:\n                        all_filenames.add(os.path.join(dirpath, f))\n            else:\n                all_filenames.add(filename)\n\n        for filename in all_filenames:\n            store = factory.getobject(filename)\n            if not store.gettargetlanguage() and not self.target_language:\n                raise CommandError(\"Unable to determine target language for \"\n                                   \"'%s'. Try again specifying a fallback \"\n                                   \"target language with --target-language\" %\n                                   filename)\n\n            self.filename = filename\n            units.extend([unit for unit in store.units if unit.istranslated()])\n\n        return units, len(units)\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        target_language = unit.gettargetlanguage()\n        if target_language is None:\n            target_language = self.target_language\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': target_language,\n            '_id': unit.getid(),\n            'revision': 0,\n            'project': self.project,\n            'path': self.filename,\n            'username': None,\n            'fullname': None,\n            'email_md5': None,\n            'source': unit.source,\n            'target': unit.target,\n        }\n\n\nclass Command(BaseCommand):\n    help = \"Load Translation Memory with translations\"\n    args = \"[files]\"\n    option_list = BaseCommand.option_list + (\n        make_option('--refresh',\n                    action='store_true',\n                    dest='refresh',\n                    default=False,\n                    help='Process all items, not just the new ones, so '\n                         'existing translations are refreshed'),\n        make_option('--rebuild',\n                    action='store_true',\n                    dest='rebuild',\n                    default=False,\n                    help='Drop the entire TM on start and update everything '\n                         'from scratch'),\n        make_option('--dry-run',\n                    action='store_true',\n                    dest='dry_run',\n                    default=False,\n                    help='Report the number of translations to index and quit'),\n        # External TM specific options.\n        make_option('--tm',\n                    action='store',\n                    dest='tm',\n                    default='local',\n                    help=\"TM to use. TM must exist on settings. TM will be \"\n                         \"created on the server if it doesn't exist\"),\n        make_option('--target-language',\n                    action='store',\n                    dest='target_language',\n                    default='',\n                    help=\"Target language to fallback to use in case it can't \"\n                         \"be guessed for any of the input files.\"),\n        make_option('--display-name',\n                    action='store',\n                    dest='project',\n                    default='',\n                    help='Name used when displaying TM matches for these '\n                         'translations.'),\n    )\n\n    def _parse_translations(self, *args, **options):\n        units, total = self.parser.get_units(*args)\n\n        if total == 0:\n            self.stdout.write(\"No translations to index\")\n            sys.exit()\n\n        self.stdout.write(\"%s translations to index\" % total)\n\n        if options['dry_run']:\n            sys.exit()\n\n        self.stdout.write(\"\")\n\n        for i, unit in enumerate(units, start=1):\n            if (i % 1000 == 0) or (i == total):\n                percent = \"%.1f\" % (i * 100.0 / total)\n                self.stdout.write(\"%s (%s%%)\" % (i, percent), ending='\\r')\n                self.stdout.flush()\n\n            yield self.parser.get_unit_data(unit)\n\n        if i != total:\n            self.stdout.write(\"Expected %d, loaded %d.\" % (total, i))\n\n    def _initialize(self, *args, **options):\n        if not getattr(settings, 'POOTLE_TM_SERVER', False):\n            raise CommandError('POOTLE_TM_SERVER setting is missing.')\n\n        try:\n            self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]\n        except KeyError:\n            raise CommandError(\"Translation Memory '%s' is not defined in the \"\n                               \"POOTLE_TM_SERVER setting. Please ensure it \"\n                               \"exists and double-check you typed it \"\n                               \"correctly.\" % options.get('tm'))\n\n        self.INDEX_NAME = self.tm_settings['INDEX_NAME']\n        self.is_local_tm = options.get('tm') == 'local'\n\n        self.es = Elasticsearch([{\n                'host': self.tm_settings['HOST'],\n                'port': self.tm_settings['PORT'],\n            }],\n            retry_on_timeout=True\n        )\n\n        # If files to import have been provided.\n        if len(args):\n            if self.is_local_tm:\n                raise CommandError('You cannot add translations from files to '\n                                   'a local TM.')\n\n            self.target_language = options.pop('target_language')\n            self.project = options.pop('project')\n\n            if not self.project:\n                raise CommandError('You must specify a project with '\n                                   '--project.')\n            self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,\n                                     language=self.target_language,\n                                     project=self.project)\n        elif not self.is_local_tm:\n            raise CommandError('You cannot add translations from database to '\n                               'an external TM.')\n        else:\n            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)\n\n    def _set_latest_indexed_revision(self, **options):\n        self.last_indexed_revision = -1\n\n        if (not options['rebuild'] and\n            not options['refresh'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            result = self.es.search(\n                index=self.INDEX_NAME,\n                body={\n                    'query': {\n                        'match_all': {}\n                    },\n                    'facets': {\n                        'stat1': {\n                            'statistical': {\n                                'field': 'revision'\n                            }\n                        }\n                    }\n                }\n            )\n            self.last_indexed_revision = result['facets']['stat1']['max']\n\n        self.parser.last_indexed_revision = self.last_indexed_revision\n\n        self.stdout.write(\"Last indexed revision = %s\" %\n                          self.last_indexed_revision)\n\n    def handle(self, *args, **options):\n        self._initialize(*args, **options)\n\n        if (options['rebuild'] and\n            not options['dry_run'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.delete(index=self.INDEX_NAME)\n\n        if (not options['dry_run'] and\n            not self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.create(index=self.INDEX_NAME)\n\n        if self.is_local_tm:\n            self._set_latest_indexed_revision(**options)\n\n        success, _ = helpers.bulk(self.es,\n                                  self._parse_translations(*args, **options))\n", "path": "pootle/apps/pootle_app/management/commands/update_tmserver.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nfrom hashlib import md5\nfrom optparse import make_option\nimport os\nimport sys\n\n# This must be run before importing Django.\nos.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'\n\nfrom elasticsearch import helpers, Elasticsearch\nfrom translate.storage import factory\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand, CommandError\n\nfrom pootle_store.models import Unit\n\n\nBULK_CHUNK_SIZE = 5000\n\n\nclass DBParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n        self.exclude_disabled_projects = not kwargs.pop('disabled_projects')\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units_qs = Unit.simple_objects \\\n            .exclude(target_f__isnull=True) \\\n            .exclude(target_f__exact='') \\\n            .filter(revision__gt=self.last_indexed_revision) \\\n            .select_related(\n                'submitted_by',\n                'store',\n                'store__translation_project__project',\n                'store__translation_project__language'\n            )\n\n        if self.exclude_disabled_projects:\n            units_qs = units_qs.exclude(\n                store__translation_project__project__disabled=True\n            )\n\n        units_qs = units_qs.values(\n                'id',\n                'revision',\n                'source_f',\n                'target_f',\n                'submitted_by__username',\n                'submitted_by__full_name',\n                'submitted_by__email',\n                'store__translation_project__project__fullname',\n                'store__pootle_path',\n                'store__translation_project__language__code'\n            ).order_by()\n\n        return units_qs.iterator(), units_qs.count()\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        fullname = (unit['submitted_by__full_name'] or\n                    unit['submitted_by__username'])\n\n        email_md5 = None\n        if unit['submitted_by__email']:\n            email_md5 = md5(unit['submitted_by__email']).hexdigest()\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': unit['store__translation_project__language__code'],\n            '_id': unit['id'],\n            'revision': int(unit['revision']),\n            'project': unit['store__translation_project__project__fullname'],\n            'path': unit['store__pootle_path'],\n            'username': unit['submitted_by__username'],\n            'fullname': fullname,\n            'email_md5': email_md5,\n            'source': unit['source_f'],\n            'target': unit['target_f'],\n        }\n\n\nclass FileParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n        self.target_language = kwargs.pop('language', None)\n        self.project = kwargs.pop('project', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units = []\n        all_filenames = set()\n\n        for filename in filenames:\n            if not os.path.exists(filename):\n                self.stdout.write(\"File %s doesn't exist. Skipping it.\" %\n                                  filename)\n                continue\n\n            if os.path.isdir(filename):\n                for dirpath, dirs, fnames in os.walk(filename):\n                    if (os.path.basename(dirpath) in\n                        [\"CVS\", \".svn\", \"_darcs\", \".git\", \".hg\", \".bzr\"]):\n\n                        continue\n\n                    for f in fnames:\n                        all_filenames.add(os.path.join(dirpath, f))\n            else:\n                all_filenames.add(filename)\n\n        for filename in all_filenames:\n            store = factory.getobject(filename)\n            if not store.gettargetlanguage() and not self.target_language:\n                raise CommandError(\"Unable to determine target language for \"\n                                   \"'%s'. Try again specifying a fallback \"\n                                   \"target language with --target-language\" %\n                                   filename)\n\n            self.filename = filename\n            units.extend([unit for unit in store.units if unit.istranslated()])\n\n        return units, len(units)\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        target_language = unit.gettargetlanguage()\n        if target_language is None:\n            target_language = self.target_language\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': target_language,\n            '_id': unit.getid(),\n            'revision': 0,\n            'project': self.project,\n            'path': self.filename,\n            'username': None,\n            'fullname': None,\n            'email_md5': None,\n            'source': unit.source,\n            'target': unit.target,\n        }\n\n\nclass Command(BaseCommand):\n    help = \"Load Translation Memory with translations\"\n    args = \"[files]\"\n    option_list = BaseCommand.option_list + (\n        make_option('--refresh',\n                    action='store_true',\n                    dest='refresh',\n                    default=False,\n                    help='Process all items, not just the new ones, so '\n                         'existing translations are refreshed'),\n        make_option('--rebuild',\n                    action='store_true',\n                    dest='rebuild',\n                    default=False,\n                    help='Drop the entire TM on start and update everything '\n                         'from scratch'),\n        make_option('--dry-run',\n                    action='store_true',\n                    dest='dry_run',\n                    default=False,\n                    help='Report the number of translations to index and quit'),\n        # Local TM specific options.\n        make_option('--include-disabled-projects',\n                    action='store_true',\n                    dest='disabled_projects',\n                    default=False,\n                    help='Add translations from disabled projects'),\n        # External TM specific options.\n        make_option('--tm',\n                    action='store',\n                    dest='tm',\n                    default='local',\n                    help=\"TM to use. TM must exist on settings. TM will be \"\n                         \"created on the server if it doesn't exist\"),\n        make_option('--target-language',\n                    action='store',\n                    dest='target_language',\n                    default='',\n                    help=\"Target language to fallback to use in case it can't \"\n                         \"be guessed for any of the input files.\"),\n        make_option('--display-name',\n                    action='store',\n                    dest='project',\n                    default='',\n                    help='Name used when displaying TM matches for these '\n                         'translations.'),\n    )\n\n    def _parse_translations(self, *args, **options):\n        units, total = self.parser.get_units(*args)\n\n        if total == 0:\n            self.stdout.write(\"No translations to index\")\n            sys.exit()\n\n        self.stdout.write(\"%s translations to index\" % total)\n\n        if options['dry_run']:\n            sys.exit()\n\n        self.stdout.write(\"\")\n\n        for i, unit in enumerate(units, start=1):\n            if (i % 1000 == 0) or (i == total):\n                percent = \"%.1f\" % (i * 100.0 / total)\n                self.stdout.write(\"%s (%s%%)\" % (i, percent), ending='\\r')\n                self.stdout.flush()\n\n            yield self.parser.get_unit_data(unit)\n\n        if i != total:\n            self.stdout.write(\"Expected %d, loaded %d.\" % (total, i))\n\n    def _initialize(self, *args, **options):\n        if not getattr(settings, 'POOTLE_TM_SERVER', False):\n            raise CommandError('POOTLE_TM_SERVER setting is missing.')\n\n        try:\n            self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]\n        except KeyError:\n            raise CommandError(\"Translation Memory '%s' is not defined in the \"\n                               \"POOTLE_TM_SERVER setting. Please ensure it \"\n                               \"exists and double-check you typed it \"\n                               \"correctly.\" % options.get('tm'))\n\n        self.INDEX_NAME = self.tm_settings['INDEX_NAME']\n        self.is_local_tm = options.get('tm') == 'local'\n\n        self.es = Elasticsearch([{\n                'host': self.tm_settings['HOST'],\n                'port': self.tm_settings['PORT'],\n            }],\n            retry_on_timeout=True\n        )\n\n        # If files to import have been provided.\n        if len(args):\n            if self.is_local_tm:\n                raise CommandError('You cannot add translations from files to '\n                                   'a local TM.')\n\n            self.target_language = options.pop('target_language')\n            self.project = options.pop('project')\n\n            if not self.project:\n                raise CommandError('You must specify a project with '\n                                   '--project.')\n            self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,\n                                     language=self.target_language,\n                                     project=self.project)\n        elif not self.is_local_tm:\n            raise CommandError('You cannot add translations from database to '\n                               'an external TM.')\n        else:\n            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME,\n                                   disabled_projects=options.pop('disabled_projects'))\n\n    def _set_latest_indexed_revision(self, **options):\n        self.last_indexed_revision = -1\n\n        if (not options['rebuild'] and\n            not options['refresh'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            result = self.es.search(\n                index=self.INDEX_NAME,\n                body={\n                    'query': {\n                        'match_all': {}\n                    },\n                    'facets': {\n                        'stat1': {\n                            'statistical': {\n                                'field': 'revision'\n                            }\n                        }\n                    }\n                }\n            )\n            self.last_indexed_revision = result['facets']['stat1']['max']\n\n        self.parser.last_indexed_revision = self.last_indexed_revision\n\n        self.stdout.write(\"Last indexed revision = %s\" %\n                          self.last_indexed_revision)\n\n    def handle(self, *args, **options):\n        self._initialize(*args, **options)\n\n        if (options['rebuild'] and\n            not options['dry_run'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.delete(index=self.INDEX_NAME)\n\n        if (not options['dry_run'] and\n            not self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.create(index=self.INDEX_NAME)\n\n        if self.is_local_tm:\n            self._set_latest_indexed_revision(**options)\n\n        success, _ = helpers.bulk(self.es,\n                                  self._parse_translations(*args, **options))\n", "path": "pootle/apps/pootle_app/management/commands/update_tmserver.py"}]}