We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Wrong junction naming for non-urban accidents
You can see that for example every accident on road 20 (Ayalon) which is not a specific intersection of 2 roads would have the name `מחלף וולפסון`. 
This happens due to the .csv file reading which is not including the `KM` parameter in its calculation.

Our `non-urban-intersection` takes its junction based on `KVISH1` + `KVISH2`.
When the accident is not an intersection: `KVISH2 = 0`.
Logically, there are multiple combinations for our `KVISH1` with `KVISH2=0`.
We should select the correct one using `KM` parameter.

This will also shorten `process.py` population time in ~50% as it doesn't assign a junction to **every accident** in Israel as used to be.
Non-urban accidents over 10 KM's away from an intersection will not be given any junction name.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `process.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 from __future__ import print_function
3 import glob
4 import os
5 import argparse
6 import json
7 from flask.ext.sqlalchemy import SQLAlchemy
8 import field_names
9 from models import Marker
10 import models
11 from utilities import ProgressSpinner, ItmToWGS84, init_flask, CsvReader
12 import itertools
13 import localization
14 import re
15 from datetime import datetime
16 
17 directories_not_processes = {}
18 
19 progress_wheel = ProgressSpinner()
20 content_encoding = 'cp1255'
21 accident_type_regex = re.compile("Accidents Type (?P<type>\d)")
22 
23 ACCIDENTS = 'accidents'
24 CITIES = 'cities'
25 STREETS = 'streets'
26 ROADS = "roads"
27 URBAN_INTERSECTION = 'urban_intersection'
28 NON_URBAN_INTERSECTION = 'non_urban_intersection'
29 DICTIONARY = "dictionary"
30 
31 lms_files = {ACCIDENTS: "AccData.csv",
32              URBAN_INTERSECTION: "IntersectUrban.csv",
33              NON_URBAN_INTERSECTION: "IntersectNonUrban.csv",
34              STREETS: "DicStreets.csv",
35              DICTIONARY: "Dictionary.csv",
36 }
37 
38 coordinates_converter = ItmToWGS84()
39 app = init_flask(__name__)
40 db = SQLAlchemy(app)
41 
42 
43 def get_street(settlement_sign, street_sign, streets):
44     """
45     extracts the street name using the settlement id and street id
46     """
47     if settlement_sign not in streets:
48         # Changed to return blank string instead of None for correct presentation (Omer)
49         return u""
50     street_name = [x[field_names.street_name].decode(content_encoding) for x in streets[settlement_sign] if
51                    x[field_names.street_sign] == street_sign]
52     # there should be only one street name, or none if it wasn't found.
53     return street_name[0] if len(street_name) == 1 else u""
54 
55 
56 def get_address(accident, streets):
57     """
58     extracts the address of the main street.
59     tries to build the full address: <street_name> <street_number>, <settlement>,
60     but might return a partial one if unsuccessful.
61     """
62     street = get_street(accident[field_names.settlement_sign], accident[field_names.street1], streets)
63     if not street:
64         return u""
65 
66     # the home field is invalid if it's empty or if it contains 9999
67     home = accident[field_names.home] if accident[field_names.home] != 9999 else None
68     settlement = localization.get_city_name(accident[field_names.settlement_sign])
69 
70     if not home and not settlement:
71         return street
72     if not home and settlement:
73         return u"{}, {}".format(street, settlement)
74     if home and not settlement:
75         return u"{} {}".format(street, home)
76 
77     return u"{} {}, {}".format(street, home, settlement)
78 
79 
80 def get_streets(accident, streets):
81     """
82     extracts the streets the accident occurred in.
83     every accident has a main street and a secondary street.
84     :return: a tuple containing both streets.
85     """
86     main_street = get_address(accident, streets)
87     secondary_street = get_street(accident[field_names.settlement_sign], accident[field_names.street2], streets)
88     return main_street, secondary_street
89 
90 
91 def get_junction(accident, roads):
92     """
93     extracts the junction from an accident
94     :return: returns the junction or None if it wasn't found
95     """
96     key = accident[field_names.road1], accident[field_names.road2]
97     junction = roads.get(key, None)
98     return junction.decode(content_encoding) if junction else u""
99 
100 
101 def parse_date(accident):
102     """
103     parses an accident's date
104     """
105     year = accident[field_names.accident_year]
106     month = accident[field_names.accident_month]
107     day = accident[field_names.accident_day]
108     hour = accident[field_names.accident_hour] % 24
109     accident_date = datetime(year, month, day, hour, 0, 0)
110     return accident_date
111 
112 
113 def load_extra_data(accident, streets, roads):
114     """
115     loads more data about the accident
116     :return: a dictionary containing all the extra fields and their values
117     :rtype: dict
118     """
119     extra_fields = {}
120     # if the accident occurred in an urban setting
121     if bool(accident[field_names.urban_intersection]):
122         main_street, secondary_street = get_streets(accident, streets)
123         if main_street:
124             extra_fields[field_names.street1] = main_street
125         if secondary_street:
126             extra_fields[field_names.street2] = secondary_street
127 
128     # if the accident occurred in a non urban setting (highway, etc')
129     if bool(accident[field_names.non_urban_intersection]):
130         junction = get_junction(accident, roads)
131         if junction:
132             extra_fields[field_names.junction_name] = junction
133 
134     # localize static accident values
135     for field in localization.get_supported_tables():
136         if accident[field]:
137             # if we have a localized field for that particular field, save the field value
138             # it will be fetched we deserialized
139             if localization.get_field(field, accident[field]):
140                 extra_fields[field] = accident[field]
141 
142     return extra_fields
143 
144 
145 def get_data_value(value):
146     """
147     :returns: value for parameters which are not mandatory in an accident data
148     OR zero if the parameter value does not exist
149     """
150     return int(value) if value else 0
151 
152 
153 def import_accidents(provider_code, accidents, streets, roads):
154     print("reading accidents from file %s" % (accidents.name(),))
155     for accident in accidents:
156         if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:
157             raise ValueError("x and y coordinates are missing from the accidents file!")
158         if not accident[field_names.x_coordinate] or not accident[field_names.y_coordinate]:
159             continue
160         lng, lat = coordinates_converter.convert(accident[field_names.x_coordinate], accident[field_names.y_coordinate])
161         main_street, secondary_street = get_streets(accident, streets)
162 
163         marker = {
164             "id":int("{0}{1}".format(provider_code, accident[field_names.id])),
165             "title":"Accident",
166             "description":json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding),
167             "address":get_address(accident, streets),
168             "latitude":lat,
169             "longitude":lng,
170             "subtype":int(accident[field_names.accident_type]),
171             "severity":int(accident[field_names.accident_severity]),
172             "created":parse_date(accident),
173             "locationAccuracy":int(accident[field_names.igun]),
174             "roadType": int(accident[field_names.road_type]),
175             # subtype
176             "roadShape": int(accident[field_names.road_shape]),
177             # severity
178             "dayType": int(accident[field_names.day_type]),
179             # locationAccuracy
180             "unit": int(accident[field_names.unit]),
181             "mainStreet": main_street,
182             "secondaryStreet": secondary_street,
183             "junction": get_junction(accident, roads),
184             "one_lane": get_data_value(accident[field_names.one_lane]),
185             "multi_lane": get_data_value(accident[field_names.multi_lane]),
186             "speed_limit": get_data_value(accident[field_names.speed_limit]),
187             "intactness": get_data_value(accident[field_names.intactness]),
188             "road_width": get_data_value(accident[field_names.road_width]),
189             "road_sign": get_data_value(accident[field_names.road_sign]),
190             "road_light": get_data_value(accident[field_names.road_light]),
191             "road_control": get_data_value(accident[field_names.road_control]),
192             "weather": get_data_value(accident[field_names.weather]),
193             "road_surface": get_data_value(accident[field_names.road_surface]),
194             "road_object": get_data_value(accident[field_names.road_object]),
195             "object_distance": get_data_value(accident[field_names.object_distance]),
196             "didnt_cross": get_data_value(accident[field_names.didnt_cross]),
197             "cross_mode": get_data_value(accident[field_names.cross_mode]),
198             "cross_location": get_data_value(accident[field_names.cross_location]),
199             "cross_direction": get_data_value(accident[field_names.cross_direction]),
200         }
201 
202         yield marker
203     accidents.close()
204 
205 
206 def get_files(directory):
207     for name, filename in lms_files.iteritems():
208 
209         if name not in [STREETS, NON_URBAN_INTERSECTION, ACCIDENTS]:
210             continue
211 
212         files = filter(lambda path: filename.lower() in path.lower(), os.listdir(directory))
213         amount = len(files)
214         if amount == 0:
215             raise ValueError("file not found in directory: " + filename)
216         if amount > 1:
217             raise ValueError("there are too many matches: " + filename)
218 
219         csv = CsvReader(os.path.join(directory, files[0]))
220 
221         if name == STREETS:
222             streets_map = {}
223             for settlement in itertools.groupby(csv, lambda street: street.get(field_names.settlement, "OTHER")):
224                 key, val = tuple(settlement)
225 
226                 streets_map[key] = [{field_names.street_sign: x[field_names.street_sign],
227                                      field_names.street_name: x[field_names.street_name]} for x in val if
228                                     field_names.street_name in x and field_names.street_sign in x]
229             csv.close()
230             yield name, streets_map
231         elif name == NON_URBAN_INTERSECTION:
232             roads = {(x[field_names.road1], x[field_names.road2]): x[field_names.junction_name] for x in csv if
233                      field_names.road1 in x and field_names.road2 in x}
234             csv.close()
235             yield ROADS, roads
236         elif name == ACCIDENTS:
237             yield name, csv
238 
239 
240 def import_to_datastore(directory, provider_code, batch_size):
241     """
242     goes through all the files in a given directory, parses and commits them
243     """
244     try:
245         files_from_lms = dict(get_files(directory))
246         if len(files_from_lms) == 0:
247             return
248         print("importing data from directory: {}".format(directory))
249         now = datetime.now()
250         accidents = list(import_accidents(provider_code=provider_code, **files_from_lms))
251         db.session.execute(Marker.__table__.insert(), accidents)
252         db.session.commit()
253         took = int((datetime.now() - now).total_seconds())
254         print("imported {0} items from directory: {1} in {2} seconds".format(len(accidents), directory, took))
255     except Exception as e:
256         directories_not_processes[directory] = e.message
257 
258 
259 def get_provider_code(directory_name=None):
260     if directory_name:
261         match = accident_type_regex.match(directory_name)
262         if match:
263             return int(match.groupdict()['type'])
264 
265     ans = ""
266     while not ans.isdigit():
267         ans = raw_input("directory provider code is invalid, please enter a valid code: ")
268         if ans.isdigit():
269             return int(ans)
270 
271 
272 def main():
273     parser = argparse.ArgumentParser()
274     parser.add_argument('--path', type=str, default="static/data/lms")
275     parser.add_argument('--batch_size', type=int, default=100)
276     parser.add_argument('--delete_all', dest='delete_all', action='store_true', default=True)
277     parser.add_argument('--provider_code', type=int)
278     args = parser.parse_args()
279     # wipe all the Markers first
280     if args.delete_all:
281         print("deleting the entire db!")
282         db.session.query(Marker).delete()
283 
284         db.session.commit()
285 
286     for directory in glob.glob("{0}/*/*".format(args.path)):
287         parent_directory = os.path.basename(os.path.dirname(os.path.join(os.pardir, directory)))
288         provider_code = args.provider_code if args.provider_code else get_provider_code(parent_directory)
289         import_to_datastore(directory, provider_code, args.batch_size)
290 
291     failed = ["{0}: {1}".format(directory, fail_reason) for directory, fail_reason in
292               directories_not_processes.iteritems()]
293     print("finished processing all directories, except: %s" % "\n".join(failed))
294 
295 
296 if __name__ == "__main__":
297     main()
298
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/process.py b/process.py
--- a/process.py
+++ b/process.py
@@ -91,11 +91,43 @@
 def get_junction(accident, roads):
     """
     extracts the junction from an accident
+    omerxx: added "km" parameter to the calculation to only show the right junction,
+    every non-urban accident shows nearest junction with distance and direction
     :return: returns the junction or None if it wasn't found
     """
-    key = accident[field_names.road1], accident[field_names.road2]
-    junction = roads.get(key, None)
-    return junction.decode(content_encoding) if junction else u""
+    if accident["KM"] is not None and accident[field_names.non_urban_intersection] is None:
+        min_dist = 100000
+        key = (), ()
+        junc_km = 0
+        for option in roads:
+            if accident[field_names.road1] == option[0] and abs(accident["KM"]-option[2]) < min_dist:
+                min_dist = abs(accident["KM"]-option[2])
+                key = accident[field_names.road1], option[1], option[2]
+                junc_km = option[2]
+        junction = roads.get(key, None)
+        if junction:
+            if accident["KM"] - junc_km > 0:
+                direction = u"צפונית" if accident[field_names.road1] % 2 == 0 else u"מזרחית"
+            else:
+                direction = u"דרומית" if accident[field_names.road1] % 2 == 0 else u"מערבית"
+            if abs(float(accident["KM"] - junc_km)/10) >= 1:
+                string = str(abs(float(accident["KM"])-junc_km)/10) + u" ק״מ " + direction + u" ל" + \
+                    junction.decode(content_encoding)
+            elif 0 < abs(float(accident["KM"] - junc_km)/10) < 1:
+                string = str(int((abs(float(accident["KM"])-junc_km)/10)*1000)) + u" מטרים " + direction + u" ל" + \
+                    junction.decode(content_encoding)
+            else:
+                string = junction.decode(content_encoding)
+            return string
+        else:
+            return u""
+
+    elif accident[field_names.non_urban_intersection] is not None:
+        key = accident[field_names.road1], accident[field_names.road2], accident["KM"]
+        junction = roads.get(key, None)
+        return junction.decode(content_encoding) if junction else u""
+    else:
+        return u""
 
 
 def parse_date(accident):
@@ -229,7 +261,7 @@
             csv.close()
             yield name, streets_map
         elif name == NON_URBAN_INTERSECTION:
-            roads = {(x[field_names.road1], x[field_names.road2]): x[field_names.junction_name] for x in csv if
+            roads = {(x[field_names.road1], x[field_names.road2], x["KM"]): x[field_names.junction_name] for x in csv if
                      field_names.road1 in x and field_names.road2 in x}
             csv.close()
             yield ROADS, roads

{"golden_diff": "diff --git a/process.py b/process.py\n--- a/process.py\n+++ b/process.py\n@@ -91,11 +91,43 @@\n def get_junction(accident, roads):\n     \"\"\"\n     extracts the junction from an accident\n+    omerxx: added \"km\" parameter to the calculation to only show the right junction,\n+    every non-urban accident shows nearest junction with distance and direction\n     :return: returns the junction or None if it wasn't found\n     \"\"\"\n-    key = accident[field_names.road1], accident[field_names.road2]\n-    junction = roads.get(key, None)\n-    return junction.decode(content_encoding) if junction else u\"\"\n+    if accident[\"KM\"] is not None and accident[field_names.non_urban_intersection] is None:\n+        min_dist = 100000\n+        key = (), ()\n+        junc_km = 0\n+        for option in roads:\n+            if accident[field_names.road1] == option[0] and abs(accident[\"KM\"]-option[2]) < min_dist:\n+                min_dist = abs(accident[\"KM\"]-option[2])\n+                key = accident[field_names.road1], option[1], option[2]\n+                junc_km = option[2]\n+        junction = roads.get(key, None)\n+        if junction:\n+            if accident[\"KM\"] - junc_km > 0:\n+                direction = u\"\u05e6\u05e4\u05d5\u05e0\u05d9\u05ea\" if accident[field_names.road1] % 2 == 0 else u\"\u05de\u05d6\u05e8\u05d7\u05d9\u05ea\"\n+            else:\n+                direction = u\"\u05d3\u05e8\u05d5\u05de\u05d9\u05ea\" if accident[field_names.road1] % 2 == 0 else u\"\u05de\u05e2\u05e8\u05d1\u05d9\u05ea\"\n+            if abs(float(accident[\"KM\"] - junc_km)/10) >= 1:\n+                string = str(abs(float(accident[\"KM\"])-junc_km)/10) + u\" \u05e7\u05f4\u05de \" + direction + u\" \u05dc\" + \\\n+                    junction.decode(content_encoding)\n+            elif 0 < abs(float(accident[\"KM\"] - junc_km)/10) < 1:\n+                string = str(int((abs(float(accident[\"KM\"])-junc_km)/10)*1000)) + u\" \u05de\u05d8\u05e8\u05d9\u05dd \" + direction + u\" \u05dc\" + \\\n+                    junction.decode(content_encoding)\n+            else:\n+                string = junction.decode(content_encoding)\n+            return string\n+        else:\n+            return u\"\"\n+\n+    elif accident[field_names.non_urban_intersection] is not None:\n+        key = accident[field_names.road1], accident[field_names.road2], accident[\"KM\"]\n+        junction = roads.get(key, None)\n+        return junction.decode(content_encoding) if junction else u\"\"\n+    else:\n+        return u\"\"\n \n \n def parse_date(accident):\n@@ -229,7 +261,7 @@\n             csv.close()\n             yield name, streets_map\n         elif name == NON_URBAN_INTERSECTION:\n-            roads = {(x[field_names.road1], x[field_names.road2]): x[field_names.junction_name] for x in csv if\n+            roads = {(x[field_names.road1], x[field_names.road2], x[\"KM\"]): x[field_names.junction_name] for x in csv if\n                      field_names.road1 in x and field_names.road2 in x}\n             csv.close()\n             yield ROADS, roads\n", "issue": "Wrong junction naming for non-urban accidents\nYou can see that for example every accident on road 20 (Ayalon) which is not a specific intersection of 2 roads would have the name `\u05de\u05d7\u05dc\u05e3 \u05d5\u05d5\u05dc\u05e4\u05e1\u05d5\u05df`. \nThis happens due to the .csv file reading which is not including the `KM` parameter in its calculation.\n\nOur `non-urban-intersection` takes its junction based on `KVISH1` + `KVISH2`.\nWhen the accident is not an intersection: `KVISH2 = 0`.\nLogically, there are multiple combinations for our `KVISH1` with `KVISH2=0`.\nWe should select the correct one using `KM` parameter.\n\nThis will also shorten `process.py` population time in ~50% as it doesn't assign a junction to **every accident** in Israel as used to be.\nNon-urban accidents over 10 KM's away from an intersection will not be given any junction name.\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\nfrom __future__ import print_function\nimport glob\nimport os\nimport argparse\nimport json\nfrom flask.ext.sqlalchemy import SQLAlchemy\nimport field_names\nfrom models import Marker\nimport models\nfrom utilities import ProgressSpinner, ItmToWGS84, init_flask, CsvReader\nimport itertools\nimport localization\nimport re\nfrom datetime import datetime\n\ndirectories_not_processes = {}\n\nprogress_wheel = ProgressSpinner()\ncontent_encoding = 'cp1255'\naccident_type_regex = re.compile(\"Accidents Type (?P<type>\\d)\")\n\nACCIDENTS = 'accidents'\nCITIES = 'cities'\nSTREETS = 'streets'\nROADS = \"roads\"\nURBAN_INTERSECTION = 'urban_intersection'\nNON_URBAN_INTERSECTION = 'non_urban_intersection'\nDICTIONARY = \"dictionary\"\n\nlms_files = {ACCIDENTS: \"AccData.csv\",\n             URBAN_INTERSECTION: \"IntersectUrban.csv\",\n             NON_URBAN_INTERSECTION: \"IntersectNonUrban.csv\",\n             STREETS: \"DicStreets.csv\",\n             DICTIONARY: \"Dictionary.csv\",\n}\n\ncoordinates_converter = ItmToWGS84()\napp = init_flask(__name__)\ndb = SQLAlchemy(app)\n\n\ndef get_street(settlement_sign, street_sign, streets):\n    \"\"\"\n    extracts the street name using the settlement id and street id\n    \"\"\"\n    if settlement_sign not in streets:\n        # Changed to return blank string instead of None for correct presentation (Omer)\n        return u\"\"\n    street_name = [x[field_names.street_name].decode(content_encoding) for x in streets[settlement_sign] if\n                   x[field_names.street_sign] == street_sign]\n    # there should be only one street name, or none if it wasn't found.\n    return street_name[0] if len(street_name) == 1 else u\"\"\n\n\ndef get_address(accident, streets):\n    \"\"\"\n    extracts the address of the main street.\n    tries to build the full address: <street_name> <street_number>, <settlement>,\n    but might return a partial one if unsuccessful.\n    \"\"\"\n    street = get_street(accident[field_names.settlement_sign], accident[field_names.street1], streets)\n    if not street:\n        return u\"\"\n\n    # the home field is invalid if it's empty or if it contains 9999\n    home = accident[field_names.home] if accident[field_names.home] != 9999 else None\n    settlement = localization.get_city_name(accident[field_names.settlement_sign])\n\n    if not home and not settlement:\n        return street\n    if not home and settlement:\n        return u\"{}, {}\".format(street, settlement)\n    if home and not settlement:\n        return u\"{} {}\".format(street, home)\n\n    return u\"{} {}, {}\".format(street, home, settlement)\n\n\ndef get_streets(accident, streets):\n    \"\"\"\n    extracts the streets the accident occurred in.\n    every accident has a main street and a secondary street.\n    :return: a tuple containing both streets.\n    \"\"\"\n    main_street = get_address(accident, streets)\n    secondary_street = get_street(accident[field_names.settlement_sign], accident[field_names.street2], streets)\n    return main_street, secondary_street\n\n\ndef get_junction(accident, roads):\n    \"\"\"\n    extracts the junction from an accident\n    :return: returns the junction or None if it wasn't found\n    \"\"\"\n    key = accident[field_names.road1], accident[field_names.road2]\n    junction = roads.get(key, None)\n    return junction.decode(content_encoding) if junction else u\"\"\n\n\ndef parse_date(accident):\n    \"\"\"\n    parses an accident's date\n    \"\"\"\n    year = accident[field_names.accident_year]\n    month = accident[field_names.accident_month]\n    day = accident[field_names.accident_day]\n    hour = accident[field_names.accident_hour] % 24\n    accident_date = datetime(year, month, day, hour, 0, 0)\n    return accident_date\n\n\ndef load_extra_data(accident, streets, roads):\n    \"\"\"\n    loads more data about the accident\n    :return: a dictionary containing all the extra fields and their values\n    :rtype: dict\n    \"\"\"\n    extra_fields = {}\n    # if the accident occurred in an urban setting\n    if bool(accident[field_names.urban_intersection]):\n        main_street, secondary_street = get_streets(accident, streets)\n        if main_street:\n            extra_fields[field_names.street1] = main_street\n        if secondary_street:\n            extra_fields[field_names.street2] = secondary_street\n\n    # if the accident occurred in a non urban setting (highway, etc')\n    if bool(accident[field_names.non_urban_intersection]):\n        junction = get_junction(accident, roads)\n        if junction:\n            extra_fields[field_names.junction_name] = junction\n\n    # localize static accident values\n    for field in localization.get_supported_tables():\n        if accident[field]:\n            # if we have a localized field for that particular field, save the field value\n            # it will be fetched we deserialized\n            if localization.get_field(field, accident[field]):\n                extra_fields[field] = accident[field]\n\n    return extra_fields\n\n\ndef get_data_value(value):\n    \"\"\"\n    :returns: value for parameters which are not mandatory in an accident data\n    OR zero if the parameter value does not exist\n    \"\"\"\n    return int(value) if value else 0\n\n\ndef import_accidents(provider_code, accidents, streets, roads):\n    print(\"reading accidents from file %s\" % (accidents.name(),))\n    for accident in accidents:\n        if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:\n            raise ValueError(\"x and y coordinates are missing from the accidents file!\")\n        if not accident[field_names.x_coordinate] or not accident[field_names.y_coordinate]:\n            continue\n        lng, lat = coordinates_converter.convert(accident[field_names.x_coordinate], accident[field_names.y_coordinate])\n        main_street, secondary_street = get_streets(accident, streets)\n\n        marker = {\n            \"id\":int(\"{0}{1}\".format(provider_code, accident[field_names.id])),\n            \"title\":\"Accident\",\n            \"description\":json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding),\n            \"address\":get_address(accident, streets),\n            \"latitude\":lat,\n            \"longitude\":lng,\n            \"subtype\":int(accident[field_names.accident_type]),\n            \"severity\":int(accident[field_names.accident_severity]),\n            \"created\":parse_date(accident),\n            \"locationAccuracy\":int(accident[field_names.igun]),\n            \"roadType\": int(accident[field_names.road_type]),\n            # subtype\n            \"roadShape\": int(accident[field_names.road_shape]),\n            # severity\n            \"dayType\": int(accident[field_names.day_type]),\n            # locationAccuracy\n            \"unit\": int(accident[field_names.unit]),\n            \"mainStreet\": main_street,\n            \"secondaryStreet\": secondary_street,\n            \"junction\": get_junction(accident, roads),\n            \"one_lane\": get_data_value(accident[field_names.one_lane]),\n            \"multi_lane\": get_data_value(accident[field_names.multi_lane]),\n            \"speed_limit\": get_data_value(accident[field_names.speed_limit]),\n            \"intactness\": get_data_value(accident[field_names.intactness]),\n            \"road_width\": get_data_value(accident[field_names.road_width]),\n            \"road_sign\": get_data_value(accident[field_names.road_sign]),\n            \"road_light\": get_data_value(accident[field_names.road_light]),\n            \"road_control\": get_data_value(accident[field_names.road_control]),\n            \"weather\": get_data_value(accident[field_names.weather]),\n            \"road_surface\": get_data_value(accident[field_names.road_surface]),\n            \"road_object\": get_data_value(accident[field_names.road_object]),\n            \"object_distance\": get_data_value(accident[field_names.object_distance]),\n            \"didnt_cross\": get_data_value(accident[field_names.didnt_cross]),\n            \"cross_mode\": get_data_value(accident[field_names.cross_mode]),\n            \"cross_location\": get_data_value(accident[field_names.cross_location]),\n            \"cross_direction\": get_data_value(accident[field_names.cross_direction]),\n        }\n\n        yield marker\n    accidents.close()\n\n\ndef get_files(directory):\n    for name, filename in lms_files.iteritems():\n\n        if name not in [STREETS, NON_URBAN_INTERSECTION, ACCIDENTS]:\n            continue\n\n        files = filter(lambda path: filename.lower() in path.lower(), os.listdir(directory))\n        amount = len(files)\n        if amount == 0:\n            raise ValueError(\"file not found in directory: \" + filename)\n        if amount > 1:\n            raise ValueError(\"there are too many matches: \" + filename)\n\n        csv = CsvReader(os.path.join(directory, files[0]))\n\n        if name == STREETS:\n            streets_map = {}\n            for settlement in itertools.groupby(csv, lambda street: street.get(field_names.settlement, \"OTHER\")):\n                key, val = tuple(settlement)\n\n                streets_map[key] = [{field_names.street_sign: x[field_names.street_sign],\n                                     field_names.street_name: x[field_names.street_name]} for x in val if\n                                    field_names.street_name in x and field_names.street_sign in x]\n            csv.close()\n            yield name, streets_map\n        elif name == NON_URBAN_INTERSECTION:\n            roads = {(x[field_names.road1], x[field_names.road2]): x[field_names.junction_name] for x in csv if\n                     field_names.road1 in x and field_names.road2 in x}\n            csv.close()\n            yield ROADS, roads\n        elif name == ACCIDENTS:\n            yield name, csv\n\n\ndef import_to_datastore(directory, provider_code, batch_size):\n    \"\"\"\n    goes through all the files in a given directory, parses and commits them\n    \"\"\"\n    try:\n        files_from_lms = dict(get_files(directory))\n        if len(files_from_lms) == 0:\n            return\n        print(\"importing data from directory: {}\".format(directory))\n        now = datetime.now()\n        accidents = list(import_accidents(provider_code=provider_code, **files_from_lms))\n        db.session.execute(Marker.__table__.insert(), accidents)\n        db.session.commit()\n        took = int((datetime.now() - now).total_seconds())\n        print(\"imported {0} items from directory: {1} in {2} seconds\".format(len(accidents), directory, took))\n    except Exception as e:\n        directories_not_processes[directory] = e.message\n\n\ndef get_provider_code(directory_name=None):\n    if directory_name:\n        match = accident_type_regex.match(directory_name)\n        if match:\n            return int(match.groupdict()['type'])\n\n    ans = \"\"\n    while not ans.isdigit():\n        ans = raw_input(\"directory provider code is invalid, please enter a valid code: \")\n        if ans.isdigit():\n            return int(ans)\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--path', type=str, default=\"static/data/lms\")\n    parser.add_argument('--batch_size', type=int, default=100)\n    parser.add_argument('--delete_all', dest='delete_all', action='store_true', default=True)\n    parser.add_argument('--provider_code', type=int)\n    args = parser.parse_args()\n    # wipe all the Markers first\n    if args.delete_all:\n        print(\"deleting the entire db!\")\n        db.session.query(Marker).delete()\n\n        db.session.commit()\n\n    for directory in glob.glob(\"{0}/*/*\".format(args.path)):\n        parent_directory = os.path.basename(os.path.dirname(os.path.join(os.pardir, directory)))\n        provider_code = args.provider_code if args.provider_code else get_provider_code(parent_directory)\n        import_to_datastore(directory, provider_code, args.batch_size)\n\n    failed = [\"{0}: {1}\".format(directory, fail_reason) for directory, fail_reason in\n              directories_not_processes.iteritems()]\n    print(\"finished processing all directories, except: %s\" % \"\\n\".join(failed))\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "process.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\nfrom __future__ import print_function\nimport glob\nimport os\nimport argparse\nimport json\nfrom flask.ext.sqlalchemy import SQLAlchemy\nimport field_names\nfrom models import Marker\nimport models\nfrom utilities import ProgressSpinner, ItmToWGS84, init_flask, CsvReader\nimport itertools\nimport localization\nimport re\nfrom datetime import datetime\n\ndirectories_not_processes = {}\n\nprogress_wheel = ProgressSpinner()\ncontent_encoding = 'cp1255'\naccident_type_regex = re.compile(\"Accidents Type (?P<type>\\d)\")\n\nACCIDENTS = 'accidents'\nCITIES = 'cities'\nSTREETS = 'streets'\nROADS = \"roads\"\nURBAN_INTERSECTION = 'urban_intersection'\nNON_URBAN_INTERSECTION = 'non_urban_intersection'\nDICTIONARY = \"dictionary\"\n\nlms_files = {ACCIDENTS: \"AccData.csv\",\n             URBAN_INTERSECTION: \"IntersectUrban.csv\",\n             NON_URBAN_INTERSECTION: \"IntersectNonUrban.csv\",\n             STREETS: \"DicStreets.csv\",\n             DICTIONARY: \"Dictionary.csv\",\n}\n\ncoordinates_converter = ItmToWGS84()\napp = init_flask(__name__)\ndb = SQLAlchemy(app)\n\n\ndef get_street(settlement_sign, street_sign, streets):\n    \"\"\"\n    extracts the street name using the settlement id and street id\n    \"\"\"\n    if settlement_sign not in streets:\n        # Changed to return blank string instead of None for correct presentation (Omer)\n        return u\"\"\n    street_name = [x[field_names.street_name].decode(content_encoding) for x in streets[settlement_sign] if\n                   x[field_names.street_sign] == street_sign]\n    # there should be only one street name, or none if it wasn't found.\n    return street_name[0] if len(street_name) == 1 else u\"\"\n\n\ndef get_address(accident, streets):\n    \"\"\"\n    extracts the address of the main street.\n    tries to build the full address: <street_name> <street_number>, <settlement>,\n    but might return a partial one if unsuccessful.\n    \"\"\"\n    street = get_street(accident[field_names.settlement_sign], accident[field_names.street1], streets)\n    if not street:\n        return u\"\"\n\n    # the home field is invalid if it's empty or if it contains 9999\n    home = accident[field_names.home] if accident[field_names.home] != 9999 else None\n    settlement = localization.get_city_name(accident[field_names.settlement_sign])\n\n    if not home and not settlement:\n        return street\n    if not home and settlement:\n        return u\"{}, {}\".format(street, settlement)\n    if home and not settlement:\n        return u\"{} {}\".format(street, home)\n\n    return u\"{} {}, {}\".format(street, home, settlement)\n\n\ndef get_streets(accident, streets):\n    \"\"\"\n    extracts the streets the accident occurred in.\n    every accident has a main street and a secondary street.\n    :return: a tuple containing both streets.\n    \"\"\"\n    main_street = get_address(accident, streets)\n    secondary_street = get_street(accident[field_names.settlement_sign], accident[field_names.street2], streets)\n    return main_street, secondary_street\n\n\ndef get_junction(accident, roads):\n    \"\"\"\n    extracts the junction from an accident\n    omerxx: added \"km\" parameter to the calculation to only show the right junction,\n    every non-urban accident shows nearest junction with distance and direction\n    :return: returns the junction or None if it wasn't found\n    \"\"\"\n    if accident[\"KM\"] is not None and accident[field_names.non_urban_intersection] is None:\n        min_dist = 100000\n        key = (), ()\n        junc_km = 0\n        for option in roads:\n            if accident[field_names.road1] == option[0] and abs(accident[\"KM\"]-option[2]) < min_dist:\n                min_dist = abs(accident[\"KM\"]-option[2])\n                key = accident[field_names.road1], option[1], option[2]\n                junc_km = option[2]\n        junction = roads.get(key, None)\n        if junction:\n            if accident[\"KM\"] - junc_km > 0:\n                direction = u\"\u05e6\u05e4\u05d5\u05e0\u05d9\u05ea\" if accident[field_names.road1] % 2 == 0 else u\"\u05de\u05d6\u05e8\u05d7\u05d9\u05ea\"\n            else:\n                direction = u\"\u05d3\u05e8\u05d5\u05de\u05d9\u05ea\" if accident[field_names.road1] % 2 == 0 else u\"\u05de\u05e2\u05e8\u05d1\u05d9\u05ea\"\n            if abs(float(accident[\"KM\"] - junc_km)/10) >= 1:\n                string = str(abs(float(accident[\"KM\"])-junc_km)/10) + u\" \u05e7\u05f4\u05de \" + direction + u\" \u05dc\" + \\\n                    junction.decode(content_encoding)\n            elif 0 < abs(float(accident[\"KM\"] - junc_km)/10) < 1:\n                string = str(int((abs(float(accident[\"KM\"])-junc_km)/10)*1000)) + u\" \u05de\u05d8\u05e8\u05d9\u05dd \" + direction + u\" \u05dc\" + \\\n                    junction.decode(content_encoding)\n            else:\n                string = junction.decode(content_encoding)\n            return string\n        else:\n            return u\"\"\n\n    elif accident[field_names.non_urban_intersection] is not None:\n        key = accident[field_names.road1], accident[field_names.road2], accident[\"KM\"]\n        junction = roads.get(key, None)\n        return junction.decode(content_encoding) if junction else u\"\"\n    else:\n        return u\"\"\n\n\ndef parse_date(accident):\n    \"\"\"\n    parses an accident's date\n    \"\"\"\n    year = accident[field_names.accident_year]\n    month = accident[field_names.accident_month]\n    day = accident[field_names.accident_day]\n    hour = accident[field_names.accident_hour] % 24\n    accident_date = datetime(year, month, day, hour, 0, 0)\n    return accident_date\n\n\ndef load_extra_data(accident, streets, roads):\n    \"\"\"\n    loads more data about the accident\n    :return: a dictionary containing all the extra fields and their values\n    :rtype: dict\n    \"\"\"\n    extra_fields = {}\n    # if the accident occurred in an urban setting\n    if bool(accident[field_names.urban_intersection]):\n        main_street, secondary_street = get_streets(accident, streets)\n        if main_street:\n            extra_fields[field_names.street1] = main_street\n        if secondary_street:\n            extra_fields[field_names.street2] = secondary_street\n\n    # if the accident occurred in a non urban setting (highway, etc')\n    if bool(accident[field_names.non_urban_intersection]):\n        junction = get_junction(accident, roads)\n        if junction:\n            extra_fields[field_names.junction_name] = junction\n\n    # localize static accident values\n    for field in localization.get_supported_tables():\n        if accident[field]:\n            # if we have a localized field for that particular field, save the field value\n            # it will be fetched we deserialized\n            if localization.get_field(field, accident[field]):\n                extra_fields[field] = accident[field]\n\n    return extra_fields\n\n\ndef get_data_value(value):\n    \"\"\"\n    :returns: value for parameters which are not mandatory in an accident data\n    OR zero if the parameter value does not exist\n    \"\"\"\n    return int(value) if value else 0\n\n\ndef import_accidents(provider_code, accidents, streets, roads):\n    print(\"reading accidents from file %s\" % (accidents.name(),))\n    for accident in accidents:\n        if field_names.x_coordinate not in accident or field_names.y_coordinate not in accident:\n            raise ValueError(\"x and y coordinates are missing from the accidents file!\")\n        if not accident[field_names.x_coordinate] or not accident[field_names.y_coordinate]:\n            continue\n        lng, lat = coordinates_converter.convert(accident[field_names.x_coordinate], accident[field_names.y_coordinate])\n        main_street, secondary_street = get_streets(accident, streets)\n\n        marker = {\n            \"id\":int(\"{0}{1}\".format(provider_code, accident[field_names.id])),\n            \"title\":\"Accident\",\n            \"description\":json.dumps(load_extra_data(accident, streets, roads), encoding=models.db_encoding),\n            \"address\":get_address(accident, streets),\n            \"latitude\":lat,\n            \"longitude\":lng,\n            \"subtype\":int(accident[field_names.accident_type]),\n            \"severity\":int(accident[field_names.accident_severity]),\n            \"created\":parse_date(accident),\n            \"locationAccuracy\":int(accident[field_names.igun]),\n            \"roadType\": int(accident[field_names.road_type]),\n            # subtype\n            \"roadShape\": int(accident[field_names.road_shape]),\n            # severity\n            \"dayType\": int(accident[field_names.day_type]),\n            # locationAccuracy\n            \"unit\": int(accident[field_names.unit]),\n            \"mainStreet\": main_street,\n            \"secondaryStreet\": secondary_street,\n            \"junction\": get_junction(accident, roads),\n            \"one_lane\": get_data_value(accident[field_names.one_lane]),\n            \"multi_lane\": get_data_value(accident[field_names.multi_lane]),\n            \"speed_limit\": get_data_value(accident[field_names.speed_limit]),\n            \"intactness\": get_data_value(accident[field_names.intactness]),\n            \"road_width\": get_data_value(accident[field_names.road_width]),\n            \"road_sign\": get_data_value(accident[field_names.road_sign]),\n            \"road_light\": get_data_value(accident[field_names.road_light]),\n            \"road_control\": get_data_value(accident[field_names.road_control]),\n            \"weather\": get_data_value(accident[field_names.weather]),\n            \"road_surface\": get_data_value(accident[field_names.road_surface]),\n            \"road_object\": get_data_value(accident[field_names.road_object]),\n            \"object_distance\": get_data_value(accident[field_names.object_distance]),\n            \"didnt_cross\": get_data_value(accident[field_names.didnt_cross]),\n            \"cross_mode\": get_data_value(accident[field_names.cross_mode]),\n            \"cross_location\": get_data_value(accident[field_names.cross_location]),\n            \"cross_direction\": get_data_value(accident[field_names.cross_direction]),\n        }\n\n        yield marker\n    accidents.close()\n\n\ndef get_files(directory):\n    for name, filename in lms_files.iteritems():\n\n        if name not in [STREETS, NON_URBAN_INTERSECTION, ACCIDENTS]:\n            continue\n\n        files = filter(lambda path: filename.lower() in path.lower(), os.listdir(directory))\n        amount = len(files)\n        if amount == 0:\n            raise ValueError(\"file not found in directory: \" + filename)\n        if amount > 1:\n            raise ValueError(\"there are too many matches: \" + filename)\n\n        csv = CsvReader(os.path.join(directory, files[0]))\n\n        if name == STREETS:\n            streets_map = {}\n            for settlement in itertools.groupby(csv, lambda street: street.get(field_names.settlement, \"OTHER\")):\n                key, val = tuple(settlement)\n\n                streets_map[key] = [{field_names.street_sign: x[field_names.street_sign],\n                                     field_names.street_name: x[field_names.street_name]} for x in val if\n                                    field_names.street_name in x and field_names.street_sign in x]\n            csv.close()\n            yield name, streets_map\n        elif name == NON_URBAN_INTERSECTION:\n            roads = {(x[field_names.road1], x[field_names.road2], x[\"KM\"]): x[field_names.junction_name] for x in csv if\n                     field_names.road1 in x and field_names.road2 in x}\n            csv.close()\n            yield ROADS, roads\n        elif name == ACCIDENTS:\n            yield name, csv\n\n\ndef import_to_datastore(directory, provider_code, batch_size):\n    \"\"\"\n    goes through all the files in a given directory, parses and commits them\n    \"\"\"\n    try:\n        files_from_lms = dict(get_files(directory))\n        if len(files_from_lms) == 0:\n            return\n        print(\"importing data from directory: {}\".format(directory))\n        now = datetime.now()\n        accidents = list(import_accidents(provider_code=provider_code, **files_from_lms))\n        db.session.execute(Marker.__table__.insert(), accidents)\n        db.session.commit()\n        took = int((datetime.now() - now).total_seconds())\n        print(\"imported {0} items from directory: {1} in {2} seconds\".format(len(accidents), directory, took))\n    except Exception as e:\n        directories_not_processes[directory] = e.message\n\n\ndef get_provider_code(directory_name=None):\n    if directory_name:\n        match = accident_type_regex.match(directory_name)\n        if match:\n            return int(match.groupdict()['type'])\n\n    ans = \"\"\n    while not ans.isdigit():\n        ans = raw_input(\"directory provider code is invalid, please enter a valid code: \")\n        if ans.isdigit():\n            return int(ans)\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--path', type=str, default=\"static/data/lms\")\n    parser.add_argument('--batch_size', type=int, default=100)\n    parser.add_argument('--delete_all', dest='delete_all', action='store_true', default=True)\n    parser.add_argument('--provider_code', type=int)\n    args = parser.parse_args()\n    # wipe all the Markers first\n    if args.delete_all:\n        print(\"deleting the entire db!\")\n        db.session.query(Marker).delete()\n\n        db.session.commit()\n\n    for directory in glob.glob(\"{0}/*/*\".format(args.path)):\n        parent_directory = os.path.basename(os.path.dirname(os.path.join(os.pardir, directory)))\n        provider_code = args.provider_code if args.provider_code else get_provider_code(parent_directory)\n        import_to_datastore(directory, provider_code, args.batch_size)\n\n    failed = [\"{0}: {1}\".format(directory, fail_reason) for directory, fail_reason in\n              directories_not_processes.iteritems()]\n    print(\"finished processing all directories, except: %s\" % \"\\n\".join(failed))\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "process.py"}]}