We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Some update_tmserver flags and feedback are confusing
I was trying the `update_tmserver` command and I found some confusing and counter-intuitive points:
1. Missing usage
   
   ``` bash
   $ ./manage.py update_tmserver
   CommandError: Specified Translation Memory is not defined in POOTLE_TM_SERVER.
   ```
   
   At this point the users are lost, as they didn't specify any TM yet it complains about the specified TM not being defined in `POOTLE_TM_SERVER`. At the same time, they have absolutely no clue what `POOTLE_TM_SERVER` is. Since this returns a `CommandError`, users might think they're doing something wrong when typing the command in, however the complain is about settings.
2. `--project` is ambiguous and confusing
   
   ```
   $ ./manage.py update_tmserver --tm=default foo.po
   CommandError: You must specify a project with --project.
   ```
   
   The naming is confusing, as one can think this has to be a project existing in Pootle, which is not the case. If it's a label or display name, let's rather use something along those lines, as `--project` is ambiguous in this context.
3. Incomplete usage message
   
   ```
   $ ./manage.py update_tmserver --help
   Usage: ./manage.py update_tmserver [options]
   ...
   ```
   
   Among other things, nowhere in the CLI is defined that one can pass file paths as arguments to the command.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pootle/apps/pootle_app/management/commands/update_tmserver.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright (C) Pootle contributors.
5 #
6 # This file is a part of the Pootle project. It is distributed under the GPL3
7 # or later license. See the LICENSE file for a copy of the license and the
8 # AUTHORS file for copyright and authorship information.
9 
10 from hashlib import md5
11 from optparse import make_option
12 import os
13 import sys
14 
15 # This must be run before importing Django.
16 os.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'
17 
18 from elasticsearch import helpers, Elasticsearch
19 from translate.storage import factory
20 
21 from django.conf import settings
22 from django.core.management.base import BaseCommand, CommandError
23 
24 from pootle_store.models import Unit
25 
26 
27 BULK_CHUNK_SIZE = 5000
28 
29 
30 class DBParser(object):
31 
32     def __init__(self, *args, **kwargs):
33         self.stdout = kwargs.pop('stdout')
34         self.INDEX_NAME = kwargs.pop('index', None)
35 
36     def get_units(self, *filenames):
37         """Gets the units to import and its total count."""
38         units_qs = Unit.simple_objects \
39             .exclude(target_f__isnull=True) \
40             .exclude(target_f__exact='') \
41             .filter(revision__gt=self.last_indexed_revision) \
42             .select_related(
43                 'submitted_by',
44                 'store',
45                 'store__translation_project__project',
46                 'store__translation_project__language'
47             ).values(
48                 'id',
49                 'revision',
50                 'source_f',
51                 'target_f',
52                 'submitted_by__username',
53                 'submitted_by__full_name',
54                 'submitted_by__email',
55                 'store__translation_project__project__fullname',
56                 'store__pootle_path',
57                 'store__translation_project__language__code'
58             ).order_by()
59 
60         return units_qs.iterator(), units_qs.count()
61 
62     def get_unit_data(self, unit):
63         """Return dict with data to import for a single unit."""
64         fullname = (unit['submitted_by__full_name'] or
65                     unit['submitted_by__username'])
66 
67         email_md5 = None
68         if unit['submitted_by__email']:
69             email_md5 = md5(unit['submitted_by__email']).hexdigest()
70 
71         return {
72             '_index': self.INDEX_NAME,
73             '_type': unit['store__translation_project__language__code'],
74             '_id': unit['id'],
75             'revision': int(unit['revision']),
76             'project': unit['store__translation_project__project__fullname'],
77             'path': unit['store__pootle_path'],
78             'username': unit['submitted_by__username'],
79             'fullname': fullname,
80             'email_md5': email_md5,
81             'source': unit['source_f'],
82             'target': unit['target_f'],
83         }
84 
85 
86 class FileParser(object):
87 
88     def __init__(self, *args, **kwargs):
89         self.stdout = kwargs.pop('stdout')
90         self.INDEX_NAME = kwargs.pop('index', None)
91         self.target_language = kwargs.pop('language', None)
92         self.project = kwargs.pop('project', None)
93 
94     def get_units(self, *filenames):
95         """Gets the units to import and its total count."""
96         units = []
97         all_filenames = set()
98 
99         for filename in filenames:
100             if not os.path.exists(filename):
101                 self.stdout.write("File %s doesn't exist. Skipping it." %
102                                   filename)
103                 continue
104 
105             if os.path.isdir(filename):
106                 for dirpath, dirs, fnames in os.walk(filename):
107                     if (os.path.basename(dirpath) in
108                         ["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]):
109 
110                         continue
111 
112                     for f in fnames:
113                         all_filenames.add(os.path.join(dirpath, f))
114             else:
115                 all_filenames.add(filename)
116 
117         for filename in all_filenames:
118             store = factory.getobject(filename)
119             if not store.gettargetlanguage() and not self.target_language:
120                 raise CommandError("Unable to determine target language for "
121                                    "'%s'. Try again specifying a fallback "
122                                    "target language with --target-language" %
123                                    filename)
124 
125             self.filename = filename
126             units.extend([unit for unit in store.units if unit.istranslated()])
127 
128         return units, len(units)
129 
130     def get_unit_data(self, unit):
131         """Return dict with data to import for a single unit."""
132         target_language = unit.gettargetlanguage()
133         if target_language is None:
134             target_language = self.target_language
135 
136         return {
137             '_index': self.INDEX_NAME,
138             '_type': target_language,
139             '_id': unit.getid(),
140             'revision': 0,
141             'project': self.project,
142             'path': self.filename,
143             'username': None,
144             'fullname': None,
145             'email_md5': None,
146             'source': unit.source,
147             'target': unit.target,
148         }
149 
150 
151 class Command(BaseCommand):
152     help = "Load Translation Memory with translations"
153     option_list = BaseCommand.option_list + (
154         make_option('--refresh',
155                     action='store_true',
156                     dest='refresh',
157                     default=False,
158                     help='Process all items, not just the new ones, so '
159                          'existing translations are refreshed'),
160         make_option('--rebuild',
161                     action='store_true',
162                     dest='rebuild',
163                     default=False,
164                     help='Drop the entire TM on start and update everything '
165                          'from scratch'),
166         make_option('--dry-run',
167                     action='store_true',
168                     dest='dry_run',
169                     default=False,
170                     help='Report the number of translations to index and quit'),
171         # External TM specific options.
172         make_option('--tm',
173                     action='store',
174                     dest='tm',
175                     default='local',
176                     help="TM to use. TM must exist on settings. TM will be "
177                          "created on the server if it doesn't exist"),
178         make_option('--target-language',
179                     action='store',
180                     dest='target_language',
181                     default='',
182                     help="Target language to fallback to use in case it can't "
183                          "be guessed for any of the input files."),
184         make_option('--project',
185                     action='store',
186                     dest='project',
187                     default='',
188                     help='Project to use when displaying TM matches for this '
189                          'translations.'),
190     )
191 
192     def _parse_translations(self, *args, **options):
193         units, total = self.parser.get_units(*args)
194 
195         if total == 0:
196             self.stdout.write("No translations to index")
197             sys.exit()
198 
199         self.stdout.write("%s translations to index" % total)
200 
201         if options['dry_run']:
202             sys.exit()
203 
204         self.stdout.write("")
205 
206         for i, unit in enumerate(units, start=1):
207             if (i % 1000 == 0) or (i == total):
208                 percent = "%.1f" % (i * 100.0 / total)
209                 self.stdout.write("%s (%s%%)" % (i, percent), ending='\r')
210                 self.stdout.flush()
211 
212             yield self.parser.get_unit_data(unit)
213 
214         if i != total:
215             self.stdout.write("Expected %d, loaded %d." % (total, i))
216 
217     def _initialize(self, *args, **options):
218         if not getattr(settings, 'POOTLE_TM_SERVER', False):
219             raise CommandError('POOTLE_TM_SERVER setting is missing.')
220 
221         try:
222             self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]
223         except KeyError:
224             raise CommandError('Specified Translation Memory is not defined '
225                                'in POOTLE_TM_SERVER.')
226 
227         self.INDEX_NAME = self.tm_settings['INDEX_NAME']
228         self.is_local_tm = options.get('tm') == 'local'
229 
230         self.es = Elasticsearch([{
231                 'host': self.tm_settings['HOST'],
232                 'port': self.tm_settings['PORT'],
233             }],
234             retry_on_timeout=True
235         )
236 
237         # If files to import have been provided.
238         if len(args):
239             if self.is_local_tm:
240                 raise CommandError('You cannot add translations from files to '
241                                    'a local TM.')
242 
243             self.target_language = options.pop('target_language')
244             self.project = options.pop('project')
245 
246             if not self.project:
247                 raise CommandError('You must specify a project with '
248                                    '--project.')
249             self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,
250                                      language=self.target_language,
251                                      project=self.project)
252         elif not self.is_local_tm:
253             raise CommandError('You cannot add translations from database to '
254                                'an external TM.')
255         else:
256             self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)
257 
258     def _set_latest_indexed_revision(self, **options):
259         self.last_indexed_revision = -1
260 
261         if (not options['rebuild'] and
262             not options['refresh'] and
263             self.es.indices.exists(self.INDEX_NAME)):
264 
265             result = self.es.search(
266                 index=self.INDEX_NAME,
267                 body={
268                     'query': {
269                         'match_all': {}
270                     },
271                     'facets': {
272                         'stat1': {
273                             'statistical': {
274                                 'field': 'revision'
275                             }
276                         }
277                     }
278                 }
279             )
280             self.last_indexed_revision = result['facets']['stat1']['max']
281 
282         self.parser.last_indexed_revision = self.last_indexed_revision
283 
284         self.stdout.write("Last indexed revision = %s" %
285                           self.last_indexed_revision)
286 
287     def handle(self, *args, **options):
288         self._initialize(*args, **options)
289 
290         if (options['rebuild'] and
291             not options['dry_run'] and
292             self.es.indices.exists(self.INDEX_NAME)):
293 
294             self.es.indices.delete(index=self.INDEX_NAME)
295 
296         if (not options['dry_run'] and
297             not self.es.indices.exists(self.INDEX_NAME)):
298 
299             self.es.indices.create(index=self.INDEX_NAME)
300 
301         if self.is_local_tm:
302             self._set_latest_indexed_revision(**options)
303 
304         success, _ = helpers.bulk(self.es,
305                                   self._parse_translations(*args, **options))
306
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pootle/apps/pootle_app/management/commands/update_tmserver.py b/pootle/apps/pootle_app/management/commands/update_tmserver.py
--- a/pootle/apps/pootle_app/management/commands/update_tmserver.py
+++ b/pootle/apps/pootle_app/management/commands/update_tmserver.py
@@ -150,6 +150,7 @@
 
 class Command(BaseCommand):
     help = "Load Translation Memory with translations"
+    args = "[files]"
     option_list = BaseCommand.option_list + (
         make_option('--refresh',
                     action='store_true',
@@ -181,11 +182,11 @@
                     default='',
                     help="Target language to fallback to use in case it can't "
                          "be guessed for any of the input files."),
-        make_option('--project',
+        make_option('--display-name',
                     action='store',
                     dest='project',
                     default='',
-                    help='Project to use when displaying TM matches for this '
+                    help='Name used when displaying TM matches for these '
                          'translations.'),
     )
 
@@ -221,8 +222,10 @@
         try:
             self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]
         except KeyError:
-            raise CommandError('Specified Translation Memory is not defined '
-                               'in POOTLE_TM_SERVER.')
+            raise CommandError("Translation Memory '%s' is not defined in the "
+                               "POOTLE_TM_SERVER setting. Please ensure it "
+                               "exists and double-check you typed it "
+                               "correctly." % options.get('tm'))
 
         self.INDEX_NAME = self.tm_settings['INDEX_NAME']
         self.is_local_tm = options.get('tm') == 'local'

{"golden_diff": "diff --git a/pootle/apps/pootle_app/management/commands/update_tmserver.py b/pootle/apps/pootle_app/management/commands/update_tmserver.py\n--- a/pootle/apps/pootle_app/management/commands/update_tmserver.py\n+++ b/pootle/apps/pootle_app/management/commands/update_tmserver.py\n@@ -150,6 +150,7 @@\n \n class Command(BaseCommand):\n     help = \"Load Translation Memory with translations\"\n+    args = \"[files]\"\n     option_list = BaseCommand.option_list + (\n         make_option('--refresh',\n                     action='store_true',\n@@ -181,11 +182,11 @@\n                     default='',\n                     help=\"Target language to fallback to use in case it can't \"\n                          \"be guessed for any of the input files.\"),\n-        make_option('--project',\n+        make_option('--display-name',\n                     action='store',\n                     dest='project',\n                     default='',\n-                    help='Project to use when displaying TM matches for this '\n+                    help='Name used when displaying TM matches for these '\n                          'translations.'),\n     )\n \n@@ -221,8 +222,10 @@\n         try:\n             self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]\n         except KeyError:\n-            raise CommandError('Specified Translation Memory is not defined '\n-                               'in POOTLE_TM_SERVER.')\n+            raise CommandError(\"Translation Memory '%s' is not defined in the \"\n+                               \"POOTLE_TM_SERVER setting. Please ensure it \"\n+                               \"exists and double-check you typed it \"\n+                               \"correctly.\" % options.get('tm'))\n \n         self.INDEX_NAME = self.tm_settings['INDEX_NAME']\n         self.is_local_tm = options.get('tm') == 'local'\n", "issue": "Some update_tmserver flags and feedback are confusing\nI was trying the `update_tmserver` command and I found some confusing and counter-intuitive points:\n1. Missing usage\n   \n   ``` bash\n   $ ./manage.py update_tmserver\n   CommandError: Specified Translation Memory is not defined in POOTLE_TM_SERVER.\n   ```\n   \n   At this point the users are lost, as they didn't specify any TM yet it complains about the specified TM not being defined in `POOTLE_TM_SERVER`. At the same time, they have absolutely no clue what `POOTLE_TM_SERVER` is. Since this returns a `CommandError`, users might think they're doing something wrong when typing the command in, however the complain is about settings.\n2. `--project` is ambiguous and confusing\n   \n   ```\n   $ ./manage.py update_tmserver --tm=default foo.po\n   CommandError: You must specify a project with --project.\n   ```\n   \n   The naming is confusing, as one can think this has to be a project existing in Pootle, which is not the case. If it's a label or display name, let's rather use something along those lines, as `--project` is ambiguous in this context.\n3. Incomplete usage message\n   \n   ```\n   $ ./manage.py update_tmserver --help\n   Usage: ./manage.py update_tmserver [options]\n   ...\n   ```\n   \n   Among other things, nowhere in the CLI is defined that one can pass file paths as arguments to the command.\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nfrom hashlib import md5\nfrom optparse import make_option\nimport os\nimport sys\n\n# This must be run before importing Django.\nos.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'\n\nfrom elasticsearch import helpers, Elasticsearch\nfrom translate.storage import factory\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand, CommandError\n\nfrom pootle_store.models import Unit\n\n\nBULK_CHUNK_SIZE = 5000\n\n\nclass DBParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units_qs = Unit.simple_objects \\\n            .exclude(target_f__isnull=True) \\\n            .exclude(target_f__exact='') \\\n            .filter(revision__gt=self.last_indexed_revision) \\\n            .select_related(\n                'submitted_by',\n                'store',\n                'store__translation_project__project',\n                'store__translation_project__language'\n            ).values(\n                'id',\n                'revision',\n                'source_f',\n                'target_f',\n                'submitted_by__username',\n                'submitted_by__full_name',\n                'submitted_by__email',\n                'store__translation_project__project__fullname',\n                'store__pootle_path',\n                'store__translation_project__language__code'\n            ).order_by()\n\n        return units_qs.iterator(), units_qs.count()\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        fullname = (unit['submitted_by__full_name'] or\n                    unit['submitted_by__username'])\n\n        email_md5 = None\n        if unit['submitted_by__email']:\n            email_md5 = md5(unit['submitted_by__email']).hexdigest()\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': unit['store__translation_project__language__code'],\n            '_id': unit['id'],\n            'revision': int(unit['revision']),\n            'project': unit['store__translation_project__project__fullname'],\n            'path': unit['store__pootle_path'],\n            'username': unit['submitted_by__username'],\n            'fullname': fullname,\n            'email_md5': email_md5,\n            'source': unit['source_f'],\n            'target': unit['target_f'],\n        }\n\n\nclass FileParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n        self.target_language = kwargs.pop('language', None)\n        self.project = kwargs.pop('project', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units = []\n        all_filenames = set()\n\n        for filename in filenames:\n            if not os.path.exists(filename):\n                self.stdout.write(\"File %s doesn't exist. Skipping it.\" %\n                                  filename)\n                continue\n\n            if os.path.isdir(filename):\n                for dirpath, dirs, fnames in os.walk(filename):\n                    if (os.path.basename(dirpath) in\n                        [\"CVS\", \".svn\", \"_darcs\", \".git\", \".hg\", \".bzr\"]):\n\n                        continue\n\n                    for f in fnames:\n                        all_filenames.add(os.path.join(dirpath, f))\n            else:\n                all_filenames.add(filename)\n\n        for filename in all_filenames:\n            store = factory.getobject(filename)\n            if not store.gettargetlanguage() and not self.target_language:\n                raise CommandError(\"Unable to determine target language for \"\n                                   \"'%s'. Try again specifying a fallback \"\n                                   \"target language with --target-language\" %\n                                   filename)\n\n            self.filename = filename\n            units.extend([unit for unit in store.units if unit.istranslated()])\n\n        return units, len(units)\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        target_language = unit.gettargetlanguage()\n        if target_language is None:\n            target_language = self.target_language\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': target_language,\n            '_id': unit.getid(),\n            'revision': 0,\n            'project': self.project,\n            'path': self.filename,\n            'username': None,\n            'fullname': None,\n            'email_md5': None,\n            'source': unit.source,\n            'target': unit.target,\n        }\n\n\nclass Command(BaseCommand):\n    help = \"Load Translation Memory with translations\"\n    option_list = BaseCommand.option_list + (\n        make_option('--refresh',\n                    action='store_true',\n                    dest='refresh',\n                    default=False,\n                    help='Process all items, not just the new ones, so '\n                         'existing translations are refreshed'),\n        make_option('--rebuild',\n                    action='store_true',\n                    dest='rebuild',\n                    default=False,\n                    help='Drop the entire TM on start and update everything '\n                         'from scratch'),\n        make_option('--dry-run',\n                    action='store_true',\n                    dest='dry_run',\n                    default=False,\n                    help='Report the number of translations to index and quit'),\n        # External TM specific options.\n        make_option('--tm',\n                    action='store',\n                    dest='tm',\n                    default='local',\n                    help=\"TM to use. TM must exist on settings. TM will be \"\n                         \"created on the server if it doesn't exist\"),\n        make_option('--target-language',\n                    action='store',\n                    dest='target_language',\n                    default='',\n                    help=\"Target language to fallback to use in case it can't \"\n                         \"be guessed for any of the input files.\"),\n        make_option('--project',\n                    action='store',\n                    dest='project',\n                    default='',\n                    help='Project to use when displaying TM matches for this '\n                         'translations.'),\n    )\n\n    def _parse_translations(self, *args, **options):\n        units, total = self.parser.get_units(*args)\n\n        if total == 0:\n            self.stdout.write(\"No translations to index\")\n            sys.exit()\n\n        self.stdout.write(\"%s translations to index\" % total)\n\n        if options['dry_run']:\n            sys.exit()\n\n        self.stdout.write(\"\")\n\n        for i, unit in enumerate(units, start=1):\n            if (i % 1000 == 0) or (i == total):\n                percent = \"%.1f\" % (i * 100.0 / total)\n                self.stdout.write(\"%s (%s%%)\" % (i, percent), ending='\\r')\n                self.stdout.flush()\n\n            yield self.parser.get_unit_data(unit)\n\n        if i != total:\n            self.stdout.write(\"Expected %d, loaded %d.\" % (total, i))\n\n    def _initialize(self, *args, **options):\n        if not getattr(settings, 'POOTLE_TM_SERVER', False):\n            raise CommandError('POOTLE_TM_SERVER setting is missing.')\n\n        try:\n            self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]\n        except KeyError:\n            raise CommandError('Specified Translation Memory is not defined '\n                               'in POOTLE_TM_SERVER.')\n\n        self.INDEX_NAME = self.tm_settings['INDEX_NAME']\n        self.is_local_tm = options.get('tm') == 'local'\n\n        self.es = Elasticsearch([{\n                'host': self.tm_settings['HOST'],\n                'port': self.tm_settings['PORT'],\n            }],\n            retry_on_timeout=True\n        )\n\n        # If files to import have been provided.\n        if len(args):\n            if self.is_local_tm:\n                raise CommandError('You cannot add translations from files to '\n                                   'a local TM.')\n\n            self.target_language = options.pop('target_language')\n            self.project = options.pop('project')\n\n            if not self.project:\n                raise CommandError('You must specify a project with '\n                                   '--project.')\n            self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,\n                                     language=self.target_language,\n                                     project=self.project)\n        elif not self.is_local_tm:\n            raise CommandError('You cannot add translations from database to '\n                               'an external TM.')\n        else:\n            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)\n\n    def _set_latest_indexed_revision(self, **options):\n        self.last_indexed_revision = -1\n\n        if (not options['rebuild'] and\n            not options['refresh'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            result = self.es.search(\n                index=self.INDEX_NAME,\n                body={\n                    'query': {\n                        'match_all': {}\n                    },\n                    'facets': {\n                        'stat1': {\n                            'statistical': {\n                                'field': 'revision'\n                            }\n                        }\n                    }\n                }\n            )\n            self.last_indexed_revision = result['facets']['stat1']['max']\n\n        self.parser.last_indexed_revision = self.last_indexed_revision\n\n        self.stdout.write(\"Last indexed revision = %s\" %\n                          self.last_indexed_revision)\n\n    def handle(self, *args, **options):\n        self._initialize(*args, **options)\n\n        if (options['rebuild'] and\n            not options['dry_run'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.delete(index=self.INDEX_NAME)\n\n        if (not options['dry_run'] and\n            not self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.create(index=self.INDEX_NAME)\n\n        if self.is_local_tm:\n            self._set_latest_indexed_revision(**options)\n\n        success, _ = helpers.bulk(self.es,\n                                  self._parse_translations(*args, **options))\n", "path": "pootle/apps/pootle_app/management/commands/update_tmserver.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) Pootle contributors.\n#\n# This file is a part of the Pootle project. It is distributed under the GPL3\n# or later license. See the LICENSE file for a copy of the license and the\n# AUTHORS file for copyright and authorship information.\n\nfrom hashlib import md5\nfrom optparse import make_option\nimport os\nimport sys\n\n# This must be run before importing Django.\nos.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'\n\nfrom elasticsearch import helpers, Elasticsearch\nfrom translate.storage import factory\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand, CommandError\n\nfrom pootle_store.models import Unit\n\n\nBULK_CHUNK_SIZE = 5000\n\n\nclass DBParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units_qs = Unit.simple_objects \\\n            .exclude(target_f__isnull=True) \\\n            .exclude(target_f__exact='') \\\n            .filter(revision__gt=self.last_indexed_revision) \\\n            .select_related(\n                'submitted_by',\n                'store',\n                'store__translation_project__project',\n                'store__translation_project__language'\n            ).values(\n                'id',\n                'revision',\n                'source_f',\n                'target_f',\n                'submitted_by__username',\n                'submitted_by__full_name',\n                'submitted_by__email',\n                'store__translation_project__project__fullname',\n                'store__pootle_path',\n                'store__translation_project__language__code'\n            ).order_by()\n\n        return units_qs.iterator(), units_qs.count()\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        fullname = (unit['submitted_by__full_name'] or\n                    unit['submitted_by__username'])\n\n        email_md5 = None\n        if unit['submitted_by__email']:\n            email_md5 = md5(unit['submitted_by__email']).hexdigest()\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': unit['store__translation_project__language__code'],\n            '_id': unit['id'],\n            'revision': int(unit['revision']),\n            'project': unit['store__translation_project__project__fullname'],\n            'path': unit['store__pootle_path'],\n            'username': unit['submitted_by__username'],\n            'fullname': fullname,\n            'email_md5': email_md5,\n            'source': unit['source_f'],\n            'target': unit['target_f'],\n        }\n\n\nclass FileParser(object):\n\n    def __init__(self, *args, **kwargs):\n        self.stdout = kwargs.pop('stdout')\n        self.INDEX_NAME = kwargs.pop('index', None)\n        self.target_language = kwargs.pop('language', None)\n        self.project = kwargs.pop('project', None)\n\n    def get_units(self, *filenames):\n        \"\"\"Gets the units to import and its total count.\"\"\"\n        units = []\n        all_filenames = set()\n\n        for filename in filenames:\n            if not os.path.exists(filename):\n                self.stdout.write(\"File %s doesn't exist. Skipping it.\" %\n                                  filename)\n                continue\n\n            if os.path.isdir(filename):\n                for dirpath, dirs, fnames in os.walk(filename):\n                    if (os.path.basename(dirpath) in\n                        [\"CVS\", \".svn\", \"_darcs\", \".git\", \".hg\", \".bzr\"]):\n\n                        continue\n\n                    for f in fnames:\n                        all_filenames.add(os.path.join(dirpath, f))\n            else:\n                all_filenames.add(filename)\n\n        for filename in all_filenames:\n            store = factory.getobject(filename)\n            if not store.gettargetlanguage() and not self.target_language:\n                raise CommandError(\"Unable to determine target language for \"\n                                   \"'%s'. Try again specifying a fallback \"\n                                   \"target language with --target-language\" %\n                                   filename)\n\n            self.filename = filename\n            units.extend([unit for unit in store.units if unit.istranslated()])\n\n        return units, len(units)\n\n    def get_unit_data(self, unit):\n        \"\"\"Return dict with data to import for a single unit.\"\"\"\n        target_language = unit.gettargetlanguage()\n        if target_language is None:\n            target_language = self.target_language\n\n        return {\n            '_index': self.INDEX_NAME,\n            '_type': target_language,\n            '_id': unit.getid(),\n            'revision': 0,\n            'project': self.project,\n            'path': self.filename,\n            'username': None,\n            'fullname': None,\n            'email_md5': None,\n            'source': unit.source,\n            'target': unit.target,\n        }\n\n\nclass Command(BaseCommand):\n    help = \"Load Translation Memory with translations\"\n    args = \"[files]\"\n    option_list = BaseCommand.option_list + (\n        make_option('--refresh',\n                    action='store_true',\n                    dest='refresh',\n                    default=False,\n                    help='Process all items, not just the new ones, so '\n                         'existing translations are refreshed'),\n        make_option('--rebuild',\n                    action='store_true',\n                    dest='rebuild',\n                    default=False,\n                    help='Drop the entire TM on start and update everything '\n                         'from scratch'),\n        make_option('--dry-run',\n                    action='store_true',\n                    dest='dry_run',\n                    default=False,\n                    help='Report the number of translations to index and quit'),\n        # External TM specific options.\n        make_option('--tm',\n                    action='store',\n                    dest='tm',\n                    default='local',\n                    help=\"TM to use. TM must exist on settings. TM will be \"\n                         \"created on the server if it doesn't exist\"),\n        make_option('--target-language',\n                    action='store',\n                    dest='target_language',\n                    default='',\n                    help=\"Target language to fallback to use in case it can't \"\n                         \"be guessed for any of the input files.\"),\n        make_option('--display-name',\n                    action='store',\n                    dest='project',\n                    default='',\n                    help='Name used when displaying TM matches for these '\n                         'translations.'),\n    )\n\n    def _parse_translations(self, *args, **options):\n        units, total = self.parser.get_units(*args)\n\n        if total == 0:\n            self.stdout.write(\"No translations to index\")\n            sys.exit()\n\n        self.stdout.write(\"%s translations to index\" % total)\n\n        if options['dry_run']:\n            sys.exit()\n\n        self.stdout.write(\"\")\n\n        for i, unit in enumerate(units, start=1):\n            if (i % 1000 == 0) or (i == total):\n                percent = \"%.1f\" % (i * 100.0 / total)\n                self.stdout.write(\"%s (%s%%)\" % (i, percent), ending='\\r')\n                self.stdout.flush()\n\n            yield self.parser.get_unit_data(unit)\n\n        if i != total:\n            self.stdout.write(\"Expected %d, loaded %d.\" % (total, i))\n\n    def _initialize(self, *args, **options):\n        if not getattr(settings, 'POOTLE_TM_SERVER', False):\n            raise CommandError('POOTLE_TM_SERVER setting is missing.')\n\n        try:\n            self.tm_settings = settings.POOTLE_TM_SERVER[options.get('tm')]\n        except KeyError:\n            raise CommandError(\"Translation Memory '%s' is not defined in the \"\n                               \"POOTLE_TM_SERVER setting. Please ensure it \"\n                               \"exists and double-check you typed it \"\n                               \"correctly.\" % options.get('tm'))\n\n        self.INDEX_NAME = self.tm_settings['INDEX_NAME']\n        self.is_local_tm = options.get('tm') == 'local'\n\n        self.es = Elasticsearch([{\n                'host': self.tm_settings['HOST'],\n                'port': self.tm_settings['PORT'],\n            }],\n            retry_on_timeout=True\n        )\n\n        # If files to import have been provided.\n        if len(args):\n            if self.is_local_tm:\n                raise CommandError('You cannot add translations from files to '\n                                   'a local TM.')\n\n            self.target_language = options.pop('target_language')\n            self.project = options.pop('project')\n\n            if not self.project:\n                raise CommandError('You must specify a project with '\n                                   '--project.')\n            self.parser = FileParser(stdout=self.stdout, index=self.INDEX_NAME,\n                                     language=self.target_language,\n                                     project=self.project)\n        elif not self.is_local_tm:\n            raise CommandError('You cannot add translations from database to '\n                               'an external TM.')\n        else:\n            self.parser = DBParser(stdout=self.stdout, index=self.INDEX_NAME)\n\n    def _set_latest_indexed_revision(self, **options):\n        self.last_indexed_revision = -1\n\n        if (not options['rebuild'] and\n            not options['refresh'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            result = self.es.search(\n                index=self.INDEX_NAME,\n                body={\n                    'query': {\n                        'match_all': {}\n                    },\n                    'facets': {\n                        'stat1': {\n                            'statistical': {\n                                'field': 'revision'\n                            }\n                        }\n                    }\n                }\n            )\n            self.last_indexed_revision = result['facets']['stat1']['max']\n\n        self.parser.last_indexed_revision = self.last_indexed_revision\n\n        self.stdout.write(\"Last indexed revision = %s\" %\n                          self.last_indexed_revision)\n\n    def handle(self, *args, **options):\n        self._initialize(*args, **options)\n\n        if (options['rebuild'] and\n            not options['dry_run'] and\n            self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.delete(index=self.INDEX_NAME)\n\n        if (not options['dry_run'] and\n            not self.es.indices.exists(self.INDEX_NAME)):\n\n            self.es.indices.create(index=self.INDEX_NAME)\n\n        if self.is_local_tm:\n            self._set_latest_indexed_revision(**options)\n\n        success, _ = helpers.bulk(self.es,\n                                  self._parse_translations(*args, **options))\n", "path": "pootle/apps/pootle_app/management/commands/update_tmserver.py"}]}