We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
management command 'geolocation --stat' failed
really not a blocking bug, but seems very easy to fix:
```
    self.sysout(f'{Town.objects.count()} town(s) in database.')
TypeError: sysout() missing 1 required positional argument: 'visible'
```

there is just a missing argument in self.sysout()

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `creme/geolocation/management/commands/geolocation.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 ################################################################################
4 #    Creme is a free/open-source Customer Relationship Management software
5 #    Copyright (C) 2015-2021  Hybird
6 #
7 #    This program is free software: you can redistribute it and/or modify
8 #    it under the terms of the GNU Affero General Public License as published by
9 #    the Free Software Foundation, either version 3 of the License, or
10 #    (at your option) any later version.
11 #
12 #    This program is distributed in the hope that it will be useful,
13 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 #    GNU Affero General Public License for more details.
16 #
17 #    You should have received a copy of the GNU Affero General Public License
18 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 ################################################################################
20 
21 import csv
22 import io
23 import logging
24 from functools import partial
25 from urllib.request import urlopen
26 from zipfile import ZipFile
27 
28 from django.conf import settings
29 from django.core.management.base import BaseCommand
30 from django.db import transaction
31 from django.template.defaultfilters import slugify
32 
33 from creme.creme_core.utils.chunktools import iter_as_chunk
34 from creme.creme_core.utils.collections import OrderedSet
35 from creme.creme_core.utils.url import parse_path
36 from creme.persons import get_address_model
37 
38 from ...models import GeoAddress, Town
39 
40 logger = logging.getLogger(__name__)
41 
42 
43 class CSVPopulatorError(Exception):
44     pass
45 
46 
47 class CSVPopulator:
48     class ProtocolError(CSVPopulatorError):
49         pass
50 
51     class ReadError(CSVPopulatorError):
52         pass
53 
54     class ParseError(CSVPopulatorError):
55         pass
56 
57     class Context:
58         def __init__(self, defaults):
59             self.line = 1
60             self.defaults = defaults
61 
62     def __init__(self, columns, defaults=None, chunksize=50):
63         """Constructor.
64         @param columns: Names of columns to extract from CSV file.
65                Raises an error if a column is neither in file nor in defaults.
66         @param defaults: dict of default values.
67         @param chunksize: Number of lines in same transaction.
68                By default sqlite supports 999 entries for each transaction,
69                so use 999/fields as max chunksize value.
70         """
71         self.columns = columns
72         self.defaults = defaults or {}
73         self.chunksize = chunksize
74 
75     def _get_source_file(self, url_info):
76         if url_info.scheme in {'file', ''}:
77             self.info(f'Reading database from {url_info.geturl()}...')
78             return open(url_info.path, 'rb')
79         elif url_info.scheme in {'http', 'https'}:
80             self.info(f'Downloading database from {url_info.geturl()}...')
81             return urlopen(url_info.geturl())
82         else:
83             raise self.ProtocolError(
84                 f'Unable to open CSV data from {url_info.geturl()} : '
85                 f'unsupported protocol.'
86             )
87 
88     def _mapper(self, header):
89         columns = self.columns
90         defaults = self.defaults
91 
92         column_keys = OrderedSet(h.lower() for h in columns)  # TODO: OrderedFrozenSet
93         row_keys = frozenset(k.lower() for k in header)
94 
95         missings = []
96         constants = {}
97         indices = [(key, index) for index, key in enumerate(header) if key in column_keys]
98 
99         for key in column_keys:
100             if key not in row_keys:
101                 try:
102                     constants[key] = defaults[key]
103                 except KeyError:
104                     missings.append(key)
105 
106         if missings:
107             raise self.ParseError(
108                 f"Following columns are missing and haven't got any default value : {missings}"
109             )
110 
111         def _aux(row):
112             data = {key: row[index] or defaults.get(key) for key, index in indices}
113             data.update(constants)
114             return data
115 
116         return _aux
117 
118     def create(self, row, context):
119         raise NotImplementedError
120 
121     def save(self, entries, context):
122         raise NotImplementedError
123 
124     def pre(self, rows, context):
125         pass
126 
127     def post(self, entries, context):
128         pass
129 
130     def line_error(self, e, row, context):
131         pass
132 
133     def chunk_error(self, e, rows, context):
134         pass
135 
136     def info(self, message):
137         logger.info(message)
138 
139     def populate(self, source):
140         if isinstance(source, str):
141             try:
142                 url_info = parse_path(source)
143 
144                 with self._get_source_file(url_info) as bytes_input:
145                     if url_info.path.endswith('.zip'):
146                         archive = ZipFile(
147                             bytes_input
148                             if bytes_input.seekable() else
149                             io.BytesIO(bytes_input.read())
150                         )
151 
152                         with archive.open(archive.namelist()[0]) as zipped_bytes_input:
153                             self._populate_from_bytes(zipped_bytes_input)
154                     else:
155                         self._populate_from_bytes(bytes_input)
156             except CSVPopulatorError:
157                 raise
158             except Exception as e:
159                 raise self.ReadError(f'Unable to open CSV data from {source} : {e}') from e
160         elif hasattr(source, '__iter__'):
161             self._populate_from_lines(iter(source))
162         else:
163             raise ValueError('The source must be a path or an iterable.')
164 
165     def _populate_from_bytes(self, bytes_input):
166         with io.TextIOWrapper(bytes_input) as wrapped_bytes_input:
167             self._populate_from_lines(csv.reader(wrapped_bytes_input))
168 
169     def _populate_from_lines(self, lines):
170         mapper = self._mapper(next(lines))
171         context = self.Context(self.defaults)
172 
173         for rows in iter_as_chunk(lines, self.chunksize):
174             entries = []
175 
176             if mapper:
177                 rows = [mapper(row) for row in rows]
178 
179             try:
180                 self.pre(rows, context)
181 
182                 for row in rows:
183                     try:
184                         entries.extend(self.create(row, context))
185                     except Exception as e:
186                         self.line_error(e, row, context)
187 
188                     context.line += 1
189 
190                 self.save(entries, context)
191                 self.post(entries, context)
192             except Exception as e:
193                 self.chunk_error(e, rows, context)
194 
195     def sync(self, model, entries, build_pk):
196         created = []
197         updated = []
198 
199         for t in entries:
200             pk = build_pk(t)
201 
202             if not pk:
203                 created.append(t)
204             else:
205                 t.pk = pk
206                 updated.append(t)
207 
208         with transaction.atomic():
209             model.objects.bulk_create(created)
210 
211             # TODO: bulk_update() ?
212             for entry in updated:
213                 entry.save(force_update=True)
214 
215 
216 class CSVTownPopulator(CSVPopulator):
217     def __init__(self, defaults=None, chunksize=100):
218         super().__init__(['title', 'zipcode', 'latitude', 'longitude', 'country'],
219                          defaults=defaults, chunksize=chunksize,
220                         )
221 
222     def line_error(self, e, row, context):
223         logger.error('    invalid data (line %d) : %s', context.line, e)
224 
225     def chunk_error(self, e, rows, context):
226         logger.error('    invalid data chunk : %s', e)
227 
228     def create(self, row, context):
229         zipcodes = row['zipcode'].split('-')
230 
231         name      = row['title']
232         latitude  = row['latitude']
233         longitude = row['longitude']
234 
235         slug = slugify(name)
236         country = row['country']
237 
238         build_town = partial(Town, country=country)
239 
240         return [build_town(name=name,
241                            slug=slug,
242                            zipcode=zipcode,
243                            latitude=latitude,
244                            longitude=longitude,
245                           ) for zipcode in zipcodes
246                ]
247 
248     def save(self, entries, context):
249         get_existing_pk_by_zipcode = dict(
250             Town.objects.filter(
251                 zipcode__in=(t.zipcode for t in entries),
252                 slug__in=(t.slug for t in entries),
253             ).values_list('zipcode', 'pk')
254         ).get
255 
256         self.sync(
257             Town, entries,
258             lambda town: get_existing_pk_by_zipcode(town.zipcode),
259         )
260 
261 
262 class Command(BaseCommand):
263     def add_arguments(self, parser):
264         add_argument = parser.add_argument
265         add_argument(
266             '-p', '--populate', action='store_true', dest='populate',
267             help='Populate addresses', default=False,
268         )
269         add_argument(
270             '-s', '--stat', action='store_true', dest='stats',
271             help='Display geolocation database stats', default=False,
272         )
273         add_argument(
274             '-i', '--import', action='store_true', dest='import', default=False,
275             help='Import towns configured in GEOLOCATION_TOWNS setting',
276         )
277 
278     def sysout(self, message, visible):
279         if visible:
280             self.stdout.write(message)
281 
282     def syserr(self, message):
283         self.stderr.write(message)
284 
285     def populate_addresses(self, verbosity=0):
286         self.sysout('Populate geolocation information of addresses...', verbosity > 0)
287         GeoAddress.populate_geoaddresses(get_address_model().objects.exclude(zipcode='', city=''))
288 
289     def import_town_database(self, url, defaults):
290         try:
291             CSVTownPopulator(defaults=defaults).populate(url)
292         except Exception as e:
293             self.syserr(str(e))
294 
295     def import_town_all(self, verbosity=0):
296         self.sysout('Importing Towns database...', verbosity > 0)
297 
298         for url, defaults in settings.GEOLOCATION_TOWNS:
299             self.sysout(url, verbosity > 1)
300             self.import_town_database(url, defaults)
301 
302     def print_stats(self):
303         self.sysout(f'{Town.objects.count()} town(s) in database.')
304 
305     def handle(self, *args, **options):
306         populate = options.get('populate')
307         stats = options.get('stats')
308         imports = options.get('import')
309         verbosity = options.get('verbosity')
310 
311         if stats:
312             self.print_stats()
313 
314         if imports:
315             self.import_town_all(verbosity)
316 
317         if populate:
318             self.populate_addresses(verbosity)
319
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/creme/geolocation/management/commands/geolocation.py b/creme/geolocation/management/commands/geolocation.py
--- a/creme/geolocation/management/commands/geolocation.py
+++ b/creme/geolocation/management/commands/geolocation.py
@@ -299,8 +299,8 @@
             self.sysout(url, verbosity > 1)
             self.import_town_database(url, defaults)
 
-    def print_stats(self):
-        self.sysout(f'{Town.objects.count()} town(s) in database.')
+    def print_stats(self, verbosity=0):
+        self.sysout(f'{Town.objects.count()} town(s) in database.', verbosity > 0)
 
     def handle(self, *args, **options):
         populate = options.get('populate')
@@ -309,7 +309,7 @@
         verbosity = options.get('verbosity')
 
         if stats:
-            self.print_stats()
+            self.print_stats(verbosity)
 
         if imports:
             self.import_town_all(verbosity)

{"golden_diff": "diff --git a/creme/geolocation/management/commands/geolocation.py b/creme/geolocation/management/commands/geolocation.py\n--- a/creme/geolocation/management/commands/geolocation.py\n+++ b/creme/geolocation/management/commands/geolocation.py\n@@ -299,8 +299,8 @@\n             self.sysout(url, verbosity > 1)\n             self.import_town_database(url, defaults)\n \n-    def print_stats(self):\n-        self.sysout(f'{Town.objects.count()} town(s) in database.')\n+    def print_stats(self, verbosity=0):\n+        self.sysout(f'{Town.objects.count()} town(s) in database.', verbosity > 0)\n \n     def handle(self, *args, **options):\n         populate = options.get('populate')\n@@ -309,7 +309,7 @@\n         verbosity = options.get('verbosity')\n \n         if stats:\n-            self.print_stats()\n+            self.print_stats(verbosity)\n \n         if imports:\n             self.import_town_all(verbosity)\n", "issue": "management command 'geolocation --stat' failed\nreally not a blocking bug, but seems very easy to fix:\r\n```\r\n    self.sysout(f'{Town.objects.count()} town(s) in database.')\r\nTypeError: sysout() missing 1 required positional argument: 'visible'\r\n```\r\n\r\nthere is just a missing argument in self.sysout()\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\n################################################################################\n#    Creme is a free/open-source Customer Relationship Management software\n#    Copyright (C) 2015-2021  Hybird\n#\n#    This program is free software: you can redistribute it and/or modify\n#    it under the terms of the GNU Affero General Public License as published by\n#    the Free Software Foundation, either version 3 of the License, or\n#    (at your option) any later version.\n#\n#    This program is distributed in the hope that it will be useful,\n#    but WITHOUT ANY WARRANTY; without even the implied warranty of\n#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n#    GNU Affero General Public License for more details.\n#\n#    You should have received a copy of the GNU Affero General Public License\n#    along with this program.  If not, see <http://www.gnu.org/licenses/>.\n################################################################################\n\nimport csv\nimport io\nimport logging\nfrom functools import partial\nfrom urllib.request import urlopen\nfrom zipfile import ZipFile\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand\nfrom django.db import transaction\nfrom django.template.defaultfilters import slugify\n\nfrom creme.creme_core.utils.chunktools import iter_as_chunk\nfrom creme.creme_core.utils.collections import OrderedSet\nfrom creme.creme_core.utils.url import parse_path\nfrom creme.persons import get_address_model\n\nfrom ...models import GeoAddress, Town\n\nlogger = logging.getLogger(__name__)\n\n\nclass CSVPopulatorError(Exception):\n    pass\n\n\nclass CSVPopulator:\n    class ProtocolError(CSVPopulatorError):\n        pass\n\n    class ReadError(CSVPopulatorError):\n        pass\n\n    class ParseError(CSVPopulatorError):\n        pass\n\n    class Context:\n        def __init__(self, defaults):\n            self.line = 1\n            self.defaults = defaults\n\n    def __init__(self, columns, defaults=None, chunksize=50):\n        \"\"\"Constructor.\n        @param columns: Names of columns to extract from CSV file.\n               Raises an error if a column is neither in file nor in defaults.\n        @param defaults: dict of default values.\n        @param chunksize: Number of lines in same transaction.\n               By default sqlite supports 999 entries for each transaction,\n               so use 999/fields as max chunksize value.\n        \"\"\"\n        self.columns = columns\n        self.defaults = defaults or {}\n        self.chunksize = chunksize\n\n    def _get_source_file(self, url_info):\n        if url_info.scheme in {'file', ''}:\n            self.info(f'Reading database from {url_info.geturl()}...')\n            return open(url_info.path, 'rb')\n        elif url_info.scheme in {'http', 'https'}:\n            self.info(f'Downloading database from {url_info.geturl()}...')\n            return urlopen(url_info.geturl())\n        else:\n            raise self.ProtocolError(\n                f'Unable to open CSV data from {url_info.geturl()} : '\n                f'unsupported protocol.'\n            )\n\n    def _mapper(self, header):\n        columns = self.columns\n        defaults = self.defaults\n\n        column_keys = OrderedSet(h.lower() for h in columns)  # TODO: OrderedFrozenSet\n        row_keys = frozenset(k.lower() for k in header)\n\n        missings = []\n        constants = {}\n        indices = [(key, index) for index, key in enumerate(header) if key in column_keys]\n\n        for key in column_keys:\n            if key not in row_keys:\n                try:\n                    constants[key] = defaults[key]\n                except KeyError:\n                    missings.append(key)\n\n        if missings:\n            raise self.ParseError(\n                f\"Following columns are missing and haven't got any default value : {missings}\"\n            )\n\n        def _aux(row):\n            data = {key: row[index] or defaults.get(key) for key, index in indices}\n            data.update(constants)\n            return data\n\n        return _aux\n\n    def create(self, row, context):\n        raise NotImplementedError\n\n    def save(self, entries, context):\n        raise NotImplementedError\n\n    def pre(self, rows, context):\n        pass\n\n    def post(self, entries, context):\n        pass\n\n    def line_error(self, e, row, context):\n        pass\n\n    def chunk_error(self, e, rows, context):\n        pass\n\n    def info(self, message):\n        logger.info(message)\n\n    def populate(self, source):\n        if isinstance(source, str):\n            try:\n                url_info = parse_path(source)\n\n                with self._get_source_file(url_info) as bytes_input:\n                    if url_info.path.endswith('.zip'):\n                        archive = ZipFile(\n                            bytes_input\n                            if bytes_input.seekable() else\n                            io.BytesIO(bytes_input.read())\n                        )\n\n                        with archive.open(archive.namelist()[0]) as zipped_bytes_input:\n                            self._populate_from_bytes(zipped_bytes_input)\n                    else:\n                        self._populate_from_bytes(bytes_input)\n            except CSVPopulatorError:\n                raise\n            except Exception as e:\n                raise self.ReadError(f'Unable to open CSV data from {source} : {e}') from e\n        elif hasattr(source, '__iter__'):\n            self._populate_from_lines(iter(source))\n        else:\n            raise ValueError('The source must be a path or an iterable.')\n\n    def _populate_from_bytes(self, bytes_input):\n        with io.TextIOWrapper(bytes_input) as wrapped_bytes_input:\n            self._populate_from_lines(csv.reader(wrapped_bytes_input))\n\n    def _populate_from_lines(self, lines):\n        mapper = self._mapper(next(lines))\n        context = self.Context(self.defaults)\n\n        for rows in iter_as_chunk(lines, self.chunksize):\n            entries = []\n\n            if mapper:\n                rows = [mapper(row) for row in rows]\n\n            try:\n                self.pre(rows, context)\n\n                for row in rows:\n                    try:\n                        entries.extend(self.create(row, context))\n                    except Exception as e:\n                        self.line_error(e, row, context)\n\n                    context.line += 1\n\n                self.save(entries, context)\n                self.post(entries, context)\n            except Exception as e:\n                self.chunk_error(e, rows, context)\n\n    def sync(self, model, entries, build_pk):\n        created = []\n        updated = []\n\n        for t in entries:\n            pk = build_pk(t)\n\n            if not pk:\n                created.append(t)\n            else:\n                t.pk = pk\n                updated.append(t)\n\n        with transaction.atomic():\n            model.objects.bulk_create(created)\n\n            # TODO: bulk_update() ?\n            for entry in updated:\n                entry.save(force_update=True)\n\n\nclass CSVTownPopulator(CSVPopulator):\n    def __init__(self, defaults=None, chunksize=100):\n        super().__init__(['title', 'zipcode', 'latitude', 'longitude', 'country'],\n                         defaults=defaults, chunksize=chunksize,\n                        )\n\n    def line_error(self, e, row, context):\n        logger.error('    invalid data (line %d) : %s', context.line, e)\n\n    def chunk_error(self, e, rows, context):\n        logger.error('    invalid data chunk : %s', e)\n\n    def create(self, row, context):\n        zipcodes = row['zipcode'].split('-')\n\n        name      = row['title']\n        latitude  = row['latitude']\n        longitude = row['longitude']\n\n        slug = slugify(name)\n        country = row['country']\n\n        build_town = partial(Town, country=country)\n\n        return [build_town(name=name,\n                           slug=slug,\n                           zipcode=zipcode,\n                           latitude=latitude,\n                           longitude=longitude,\n                          ) for zipcode in zipcodes\n               ]\n\n    def save(self, entries, context):\n        get_existing_pk_by_zipcode = dict(\n            Town.objects.filter(\n                zipcode__in=(t.zipcode for t in entries),\n                slug__in=(t.slug for t in entries),\n            ).values_list('zipcode', 'pk')\n        ).get\n\n        self.sync(\n            Town, entries,\n            lambda town: get_existing_pk_by_zipcode(town.zipcode),\n        )\n\n\nclass Command(BaseCommand):\n    def add_arguments(self, parser):\n        add_argument = parser.add_argument\n        add_argument(\n            '-p', '--populate', action='store_true', dest='populate',\n            help='Populate addresses', default=False,\n        )\n        add_argument(\n            '-s', '--stat', action='store_true', dest='stats',\n            help='Display geolocation database stats', default=False,\n        )\n        add_argument(\n            '-i', '--import', action='store_true', dest='import', default=False,\n            help='Import towns configured in GEOLOCATION_TOWNS setting',\n        )\n\n    def sysout(self, message, visible):\n        if visible:\n            self.stdout.write(message)\n\n    def syserr(self, message):\n        self.stderr.write(message)\n\n    def populate_addresses(self, verbosity=0):\n        self.sysout('Populate geolocation information of addresses...', verbosity > 0)\n        GeoAddress.populate_geoaddresses(get_address_model().objects.exclude(zipcode='', city=''))\n\n    def import_town_database(self, url, defaults):\n        try:\n            CSVTownPopulator(defaults=defaults).populate(url)\n        except Exception as e:\n            self.syserr(str(e))\n\n    def import_town_all(self, verbosity=0):\n        self.sysout('Importing Towns database...', verbosity > 0)\n\n        for url, defaults in settings.GEOLOCATION_TOWNS:\n            self.sysout(url, verbosity > 1)\n            self.import_town_database(url, defaults)\n\n    def print_stats(self):\n        self.sysout(f'{Town.objects.count()} town(s) in database.')\n\n    def handle(self, *args, **options):\n        populate = options.get('populate')\n        stats = options.get('stats')\n        imports = options.get('import')\n        verbosity = options.get('verbosity')\n\n        if stats:\n            self.print_stats()\n\n        if imports:\n            self.import_town_all(verbosity)\n\n        if populate:\n            self.populate_addresses(verbosity)\n", "path": "creme/geolocation/management/commands/geolocation.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\n################################################################################\n#    Creme is a free/open-source Customer Relationship Management software\n#    Copyright (C) 2015-2021  Hybird\n#\n#    This program is free software: you can redistribute it and/or modify\n#    it under the terms of the GNU Affero General Public License as published by\n#    the Free Software Foundation, either version 3 of the License, or\n#    (at your option) any later version.\n#\n#    This program is distributed in the hope that it will be useful,\n#    but WITHOUT ANY WARRANTY; without even the implied warranty of\n#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n#    GNU Affero General Public License for more details.\n#\n#    You should have received a copy of the GNU Affero General Public License\n#    along with this program.  If not, see <http://www.gnu.org/licenses/>.\n################################################################################\n\nimport csv\nimport io\nimport logging\nfrom functools import partial\nfrom urllib.request import urlopen\nfrom zipfile import ZipFile\n\nfrom django.conf import settings\nfrom django.core.management.base import BaseCommand\nfrom django.db import transaction\nfrom django.template.defaultfilters import slugify\n\nfrom creme.creme_core.utils.chunktools import iter_as_chunk\nfrom creme.creme_core.utils.collections import OrderedSet\nfrom creme.creme_core.utils.url import parse_path\nfrom creme.persons import get_address_model\n\nfrom ...models import GeoAddress, Town\n\nlogger = logging.getLogger(__name__)\n\n\nclass CSVPopulatorError(Exception):\n    pass\n\n\nclass CSVPopulator:\n    class ProtocolError(CSVPopulatorError):\n        pass\n\n    class ReadError(CSVPopulatorError):\n        pass\n\n    class ParseError(CSVPopulatorError):\n        pass\n\n    class Context:\n        def __init__(self, defaults):\n            self.line = 1\n            self.defaults = defaults\n\n    def __init__(self, columns, defaults=None, chunksize=50):\n        \"\"\"Constructor.\n        @param columns: Names of columns to extract from CSV file.\n               Raises an error if a column is neither in file nor in defaults.\n        @param defaults: dict of default values.\n        @param chunksize: Number of lines in same transaction.\n               By default sqlite supports 999 entries for each transaction,\n               so use 999/fields as max chunksize value.\n        \"\"\"\n        self.columns = columns\n        self.defaults = defaults or {}\n        self.chunksize = chunksize\n\n    def _get_source_file(self, url_info):\n        if url_info.scheme in {'file', ''}:\n            self.info(f'Reading database from {url_info.geturl()}...')\n            return open(url_info.path, 'rb')\n        elif url_info.scheme in {'http', 'https'}:\n            self.info(f'Downloading database from {url_info.geturl()}...')\n            return urlopen(url_info.geturl())\n        else:\n            raise self.ProtocolError(\n                f'Unable to open CSV data from {url_info.geturl()} : '\n                f'unsupported protocol.'\n            )\n\n    def _mapper(self, header):\n        columns = self.columns\n        defaults = self.defaults\n\n        column_keys = OrderedSet(h.lower() for h in columns)  # TODO: OrderedFrozenSet\n        row_keys = frozenset(k.lower() for k in header)\n\n        missings = []\n        constants = {}\n        indices = [(key, index) for index, key in enumerate(header) if key in column_keys]\n\n        for key in column_keys:\n            if key not in row_keys:\n                try:\n                    constants[key] = defaults[key]\n                except KeyError:\n                    missings.append(key)\n\n        if missings:\n            raise self.ParseError(\n                f\"Following columns are missing and haven't got any default value : {missings}\"\n            )\n\n        def _aux(row):\n            data = {key: row[index] or defaults.get(key) for key, index in indices}\n            data.update(constants)\n            return data\n\n        return _aux\n\n    def create(self, row, context):\n        raise NotImplementedError\n\n    def save(self, entries, context):\n        raise NotImplementedError\n\n    def pre(self, rows, context):\n        pass\n\n    def post(self, entries, context):\n        pass\n\n    def line_error(self, e, row, context):\n        pass\n\n    def chunk_error(self, e, rows, context):\n        pass\n\n    def info(self, message):\n        logger.info(message)\n\n    def populate(self, source):\n        if isinstance(source, str):\n            try:\n                url_info = parse_path(source)\n\n                with self._get_source_file(url_info) as bytes_input:\n                    if url_info.path.endswith('.zip'):\n                        archive = ZipFile(\n                            bytes_input\n                            if bytes_input.seekable() else\n                            io.BytesIO(bytes_input.read())\n                        )\n\n                        with archive.open(archive.namelist()[0]) as zipped_bytes_input:\n                            self._populate_from_bytes(zipped_bytes_input)\n                    else:\n                        self._populate_from_bytes(bytes_input)\n            except CSVPopulatorError:\n                raise\n            except Exception as e:\n                raise self.ReadError(f'Unable to open CSV data from {source} : {e}') from e\n        elif hasattr(source, '__iter__'):\n            self._populate_from_lines(iter(source))\n        else:\n            raise ValueError('The source must be a path or an iterable.')\n\n    def _populate_from_bytes(self, bytes_input):\n        with io.TextIOWrapper(bytes_input) as wrapped_bytes_input:\n            self._populate_from_lines(csv.reader(wrapped_bytes_input))\n\n    def _populate_from_lines(self, lines):\n        mapper = self._mapper(next(lines))\n        context = self.Context(self.defaults)\n\n        for rows in iter_as_chunk(lines, self.chunksize):\n            entries = []\n\n            if mapper:\n                rows = [mapper(row) for row in rows]\n\n            try:\n                self.pre(rows, context)\n\n                for row in rows:\n                    try:\n                        entries.extend(self.create(row, context))\n                    except Exception as e:\n                        self.line_error(e, row, context)\n\n                    context.line += 1\n\n                self.save(entries, context)\n                self.post(entries, context)\n            except Exception as e:\n                self.chunk_error(e, rows, context)\n\n    def sync(self, model, entries, build_pk):\n        created = []\n        updated = []\n\n        for t in entries:\n            pk = build_pk(t)\n\n            if not pk:\n                created.append(t)\n            else:\n                t.pk = pk\n                updated.append(t)\n\n        with transaction.atomic():\n            model.objects.bulk_create(created)\n\n            # TODO: bulk_update() ?\n            for entry in updated:\n                entry.save(force_update=True)\n\n\nclass CSVTownPopulator(CSVPopulator):\n    def __init__(self, defaults=None, chunksize=100):\n        super().__init__(['title', 'zipcode', 'latitude', 'longitude', 'country'],\n                         defaults=defaults, chunksize=chunksize,\n                        )\n\n    def line_error(self, e, row, context):\n        logger.error('    invalid data (line %d) : %s', context.line, e)\n\n    def chunk_error(self, e, rows, context):\n        logger.error('    invalid data chunk : %s', e)\n\n    def create(self, row, context):\n        zipcodes = row['zipcode'].split('-')\n\n        name      = row['title']\n        latitude  = row['latitude']\n        longitude = row['longitude']\n\n        slug = slugify(name)\n        country = row['country']\n\n        build_town = partial(Town, country=country)\n\n        return [build_town(name=name,\n                           slug=slug,\n                           zipcode=zipcode,\n                           latitude=latitude,\n                           longitude=longitude,\n                          ) for zipcode in zipcodes\n               ]\n\n    def save(self, entries, context):\n        get_existing_pk_by_zipcode = dict(\n            Town.objects.filter(\n                zipcode__in=(t.zipcode for t in entries),\n                slug__in=(t.slug for t in entries),\n            ).values_list('zipcode', 'pk')\n        ).get\n\n        self.sync(\n            Town, entries,\n            lambda town: get_existing_pk_by_zipcode(town.zipcode),\n        )\n\n\nclass Command(BaseCommand):\n    def add_arguments(self, parser):\n        add_argument = parser.add_argument\n        add_argument(\n            '-p', '--populate', action='store_true', dest='populate',\n            help='Populate addresses', default=False,\n        )\n        add_argument(\n            '-s', '--stat', action='store_true', dest='stats',\n            help='Display geolocation database stats', default=False,\n        )\n        add_argument(\n            '-i', '--import', action='store_true', dest='import', default=False,\n            help='Import towns configured in GEOLOCATION_TOWNS setting',\n        )\n\n    def sysout(self, message, visible):\n        if visible:\n            self.stdout.write(message)\n\n    def syserr(self, message):\n        self.stderr.write(message)\n\n    def populate_addresses(self, verbosity=0):\n        self.sysout('Populate geolocation information of addresses...', verbosity > 0)\n        GeoAddress.populate_geoaddresses(get_address_model().objects.exclude(zipcode='', city=''))\n\n    def import_town_database(self, url, defaults):\n        try:\n            CSVTownPopulator(defaults=defaults).populate(url)\n        except Exception as e:\n            self.syserr(str(e))\n\n    def import_town_all(self, verbosity=0):\n        self.sysout('Importing Towns database...', verbosity > 0)\n\n        for url, defaults in settings.GEOLOCATION_TOWNS:\n            self.sysout(url, verbosity > 1)\n            self.import_town_database(url, defaults)\n\n    def print_stats(self, verbosity=0):\n        self.sysout(f'{Town.objects.count()} town(s) in database.', verbosity > 0)\n\n    def handle(self, *args, **options):\n        populate = options.get('populate')\n        stats = options.get('stats')\n        imports = options.get('import')\n        verbosity = options.get('verbosity')\n\n        if stats:\n            self.print_stats(verbosity)\n\n        if imports:\n            self.import_town_all(verbosity)\n\n        if populate:\n            self.populate_addresses(verbosity)\n", "path": "creme/geolocation/management/commands/geolocation.py"}]}