We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
GB-NIR invalid data in database
The latest observation seems to be problematic:

![image](https://user-images.githubusercontent.com/1655848/42372212-eecab938-8111-11e8-808c-779694b74962.png)

therefore it is not shown on the map. However it is inserted in the database.
We should add proper validations to make sure coal/gas are present and that load > 0

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/GB_NIR.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 from collections import defaultdict
4 from datetime import datetime
5 from io import StringIO
6 from operator import itemgetter
7 
8 import logging
9 import pandas as pd
10 import requests
11 from bs4 import BeautifulSoup
12 from dateutil import parser, tz
13 
14 from .lib.validation import validate
15 
16 thermal_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/FuelMix.aspx'
17 wind_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/SystemOutput.aspx'
18 exchange_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/MoyleTie.aspx'
19 # Positive values represent imports to Northern Ireland.
20 # Negative value represent exports from Northern Ireland.
21 
22 
23 def get_data(url, session=None):
24     """
25     Requests data from a specified url in CSV format.
26     Returns a response.text object.
27     """
28 
29     s = session or requests.Session()
30 
31     headers = {
32         'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',
33         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
34     }
35 
36     pagereq = requests.get(url, headers=headers)
37     soup = BeautifulSoup(pagereq.text, 'html.parser')
38 
39     # Find and define parameters needed to send a POST request for the actual data.
40     viewstategenerator = soup.find("input", attrs={'id': '__VIEWSTATEGENERATOR'})['value']
41     viewstate = soup.find("input", attrs={'id': '__VIEWSTATE'})['value']
42     eventvalidation = soup.find("input", attrs={'id': '__EVENTVALIDATION'})['value']
43 
44     # Set date for post request.
45     current_date = datetime.now().date()
46     month = current_date.month
47     day = current_date.day
48     year = current_date.year
49 
50     FromDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],"01%s-%s-%s-0-0-0-0"]' % (year, month, day, '', year, month, day)
51     ToDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],"01%s-%s-%s-0-0-0-0"]' % (year, month, day, '', year, month, day)
52     btnDownloadCSV = 'Download+CSV'
53     ig_def_dp_cal_clientState = '|0|15,2017,09,2017,%s,%s||[[null,[],null],[{%s},[]],"11,2017,09,2017,%s,%s"]' % (month, day, '', month, day)
54     IG_CSS_LINKS_ = 'ig_res/default/ig_monthcalendar.css|ig_res/default/ig_texteditor.css|ig_res/default/ig_shared.css'
55 
56     postdata = {'__VIEWSTATE': viewstate,
57                 '__VIEWSTATEGENERATOR': viewstategenerator,
58                 '__EVENTVALIDATION': eventvalidation,
59                 'FromDatePicker_clientState': FromDatePicker_clientState,
60                 'ToDatePicker_clientState': ToDatePicker_clientState,
61                 'btnDownloadCSV': btnDownloadCSV,
62                 '_ig_def_dp_cal_clientState': ig_def_dp_cal_clientState,
63                 '_IG_CSS_LINKS_': IG_CSS_LINKS_
64                }
65 
66     postheaders = {
67                    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',
68                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
69                    'Content-Type': 'application/x-www-form-urlencoded'
70                   }
71 
72     datareq = s.post(url, headers=postheaders, data=postdata)
73 
74     return datareq.text
75 
76 
77 def add_default_tz(timestamp):
78     """
79     Adds Northern Ireland timezone to datetime object if tz = None.
80     """
81 
82     NIR = tz.gettz('Europe/Belfast')
83     modified_timestamp = timestamp.replace(tzinfo=timestamp.tzinfo or NIR)
84 
85     return modified_timestamp
86 
87 
88 def create_thermal_df(text_data):
89     """
90     Turns thermal csv data into a usable dataframe.
91     """
92 
93     cols_to_use = [0, 1, 2, 3, 4, 5]
94     df_thermal = pd.read_csv(StringIO(text_data),
95                              usecols=cols_to_use)
96     df_thermal.fillna(0.0, inplace=True)
97 
98     return df_thermal
99 
100 
101 def create_wind_df(text_data):
102     """
103     Turns wind csv data into a usable dataframe.
104     """
105 
106     cols_to_use = [0, 1]
107     df_wind = pd.read_csv(StringIO(text_data),
108                           usecols=cols_to_use)
109     df_wind.fillna(0.0, inplace=True)
110 
111     return df_wind
112 
113 
114 def create_exchange_df(text_data):
115     """
116     Turns exchange csv data into a usable dataframe.
117     """
118 
119     df_exchange = pd.read_csv(StringIO(text_data))
120     df_exchange.fillna(0.0, inplace=True)
121 
122     return df_exchange
123 
124 
125 def thermal_processor(df):
126     """
127     Creates quarter hour datapoints for thermal production.
128     Returns a list.
129     """
130 
131     datapoints = []
132     for index, row in df.iterrows():
133         snapshot = {}
134         snapshot['datetime'] = row['TimeStamp']
135         snapshot['gas'] = row['Gas_MW']
136         snapshot['coal'] = row['Coal_MW']
137         snapshot['oil'] = row['Distillate_MW'] + row['Diesel_MW']
138         datapoints.append(snapshot)
139 
140     return datapoints
141 
142 
143 def wind_processor(df):
144     """
145     Creates quarter hour datapoints for wind production.
146     Returns a list.
147     """
148 
149     datapoints = []
150     for index, row in df.iterrows():
151         snapshot = {}
152         snapshot['datetime'] = row['TimeStamp']
153         snapshot['wind'] = row['Total_Wind_Generated_MW']
154         if snapshot['wind'] > -20:
155             snapshot['wind'] = max(snapshot['wind'], 0)
156         datapoints.append(snapshot)
157 
158     return datapoints
159 
160 
161 def moyle_processor(df):
162     """
163     Creates quarter hour datapoints for GB exchange.
164     Returns a list.
165     """
166 
167     datapoints = []
168     for index, row in df.iterrows():
169         snapshot = {}
170         snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],
171                                                            dayfirst=True))
172         snapshot['netFlow'] = row['Total_Moyle_Load_MW']
173         snapshot['source'] = 'soni.ltd.uk'
174         snapshot['sortedZoneKeys'] = 'GB->GB-NIR'
175         datapoints.append(snapshot)
176 
177     return datapoints
178 
179 
180 def IE_processor(df):
181     """
182     Creates quarter hour datapoints for IE exchange.
183     Returns a list.
184     """
185 
186     datapoints = []
187     for index, row in df.iterrows():
188         snapshot = {}
189         snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],
190                                                            dayfirst=True))
191         netFlow = (row['Total_Str_Let_Load_MW'] +
192                    row['Total_Enn_Cor_Load_MW'] +
193                    row['Total_Tan_Lou_Load_MW'])
194         snapshot['netFlow'] = -1 * (netFlow)
195         snapshot['source'] = 'soni.ltd.uk'
196         snapshot['sortedZoneKeys'] = 'GB-NIR->IE'
197         datapoints.append(snapshot)
198 
199     return datapoints
200 
201 
202 def merge_production(thermal_data, wind_data):
203     """
204     Joins thermal and wind production data on shared datetime key.
205     Returns a list.
206     """
207 
208     total_production = thermal_data + wind_data
209 
210     # Join thermal and wind dicts on 'datetime' key.
211     d = defaultdict(dict)
212     for elem in total_production:
213         d[elem['datetime']].update(elem)
214 
215     joined_data = sorted(d.values(), key=itemgetter("datetime"))
216 
217     for datapoint in joined_data:
218         datapoint['datetime'] = add_default_tz(parser.parse(datapoint['datetime'], dayfirst=True))
219 
220     return joined_data
221 
222 
223 def fetch_production(zone_key='GB-NIR', session=None, target_datetime=None,
224                      logger=logging.getLogger(__name__)):
225     """
226     Requests the last known production mix (in MW) of a given country
227         Arguments:
228         zone_key (optional) -- used in case a parser is able to fetch multiple countries
229         session (optional)      -- request session passed in order to re-use an existing session
230         Return:
231         A dictionary in the form:
232         {
233           'zoneKey': 'FR',
234           'datetime': '2017-01-01T00:00:00Z',
235           'production': {
236               'biomass': 0.0,
237               'coal': 0.0,
238               'gas': 0.0,
239               'hydro': 0.0,
240               'nuclear': null,
241               'oil': 0.0,
242               'solar': 0.0,
243               'wind': 0.0,
244               'geothermal': 0.0,
245               'unknown': 0.0
246           },
247           'storage': {
248               'hydro': -10.0,
249           },
250           'source': 'mysource.com'
251         }
252     """
253     if target_datetime:
254         raise NotImplementedError('This parser is not yet able to parse past dates')
255 
256     thermal_data = get_data(thermal_url)
257     wind_data = get_data(wind_url)
258     thermal_df = create_thermal_df(thermal_data)
259     wind_df = create_wind_df(wind_data)
260     thermal = thermal_processor(thermal_df)
261     wind = wind_processor(wind_df)
262     merge = merge_production(thermal, wind)
263 
264     production_mix_by_quarter_hour = []
265 
266     for datapoint in merge:
267         production_mix = {
268             'zoneKey': zone_key,
269             'datetime': datapoint.get('datetime', 0.0),
270             'production': {
271                 'coal': datapoint.get('coal', 0.0),
272                 'gas': datapoint.get('gas', 0.0),
273                 'oil': datapoint.get('oil', 0.0),
274                 'solar': None,
275                 'wind': datapoint.get('wind', 0.0)
276             },
277             'source': 'soni.ltd.uk'
278         }
279         production_mix_by_quarter_hour.append(
280             validate(production_mix, logger=logger, required=['gas', 'coal']))
281 
282     return production_mix_by_quarter_hour
283 
284 
285 def fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):
286     """Requests the last known power exchange (in MW) between two countries
287     Arguments:
288     zone_key (optional) -- used in case a parser is able to fetch multiple countries
289     session (optional)      -- request session passed in order to re-use an existing session
290     Return:
291     A dictionary in the form:
292     {
293       'sortedZoneKeys': 'DK->NO',
294       'datetime': '2017-01-01T00:00:00Z',
295       'netFlow': 0.0,
296       'source': 'mysource.com'
297     }
298     """
299     if target_datetime:
300         raise NotImplementedError('This parser is not yet able to parse past dates')
301 
302     exchange_data = get_data(exchange_url)
303     exchange_dataframe = create_exchange_df(exchange_data)
304     if '->'.join(sorted([zone_key1, zone_key2])) == 'GB->GB-NIR':
305         moyle = moyle_processor(exchange_dataframe)
306         return moyle
307     elif '->'.join(sorted([zone_key1, zone_key2])) == 'GB-NIR->IE':
308         IE = IE_processor(exchange_dataframe)
309         return IE
310     else:
311         raise NotImplementedError('This exchange pair is not implemented')
312 
313 
314 if __name__ == '__main__':
315     """Main method, never used by the Electricity Map backend, but handy for testing."""
316 
317     print('fetch_production() ->')
318     print(fetch_production())
319     print('fetch_exchange(GB-NIR, GB) ->')
320     print(fetch_exchange('GB-NIR', 'GB'))
321     print('fetch_exchange(GB-NIR, IE) ->')
322     print(fetch_exchange('GB-NIR', 'IE'))
323
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/GB_NIR.py b/parsers/GB_NIR.py
--- a/parsers/GB_NIR.py
+++ b/parsers/GB_NIR.py
@@ -277,7 +277,7 @@
             'source': 'soni.ltd.uk'
         }
         production_mix_by_quarter_hour.append(
-            validate(production_mix, logger=logger, required=['gas', 'coal']))
+            validate(production_mix, logger=logger, required=['gas', 'coal'], floor=1.0))
 
     return production_mix_by_quarter_hour

{"golden_diff": "diff --git a/parsers/GB_NIR.py b/parsers/GB_NIR.py\n--- a/parsers/GB_NIR.py\n+++ b/parsers/GB_NIR.py\n@@ -277,7 +277,7 @@\n             'source': 'soni.ltd.uk'\n         }\n         production_mix_by_quarter_hour.append(\n-            validate(production_mix, logger=logger, required=['gas', 'coal']))\n+            validate(production_mix, logger=logger, required=['gas', 'coal'], floor=1.0))\n \n     return production_mix_by_quarter_hour\n", "issue": "GB-NIR invalid data in database\nThe latest observation seems to be problematic:\r\n\r\n![image](https://user-images.githubusercontent.com/1655848/42372212-eecab938-8111-11e8-808c-779694b74962.png)\r\n\r\ntherefore it is not shown on the map. However it is inserted in the database.\r\nWe should add proper validations to make sure coal/gas are present and that load > 0\n", "before_files": [{"content": "#!/usr/bin/env python3\n\nfrom collections import defaultdict\nfrom datetime import datetime\nfrom io import StringIO\nfrom operator import itemgetter\n\nimport logging\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup\nfrom dateutil import parser, tz\n\nfrom .lib.validation import validate\n\nthermal_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/FuelMix.aspx'\nwind_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/SystemOutput.aspx'\nexchange_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/MoyleTie.aspx'\n# Positive values represent imports to Northern Ireland.\n# Negative value represent exports from Northern Ireland.\n\n\ndef get_data(url, session=None):\n    \"\"\"\n    Requests data from a specified url in CSV format.\n    Returns a response.text object.\n    \"\"\"\n\n    s = session or requests.Session()\n\n    headers = {\n        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',\n        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'\n    }\n\n    pagereq = requests.get(url, headers=headers)\n    soup = BeautifulSoup(pagereq.text, 'html.parser')\n\n    # Find and define parameters needed to send a POST request for the actual data.\n    viewstategenerator = soup.find(\"input\", attrs={'id': '__VIEWSTATEGENERATOR'})['value']\n    viewstate = soup.find(\"input\", attrs={'id': '__VIEWSTATE'})['value']\n    eventvalidation = soup.find(\"input\", attrs={'id': '__EVENTVALIDATION'})['value']\n\n    # Set date for post request.\n    current_date = datetime.now().date()\n    month = current_date.month\n    day = current_date.day\n    year = current_date.year\n\n    FromDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],\"01%s-%s-%s-0-0-0-0\"]' % (year, month, day, '', year, month, day)\n    ToDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],\"01%s-%s-%s-0-0-0-0\"]' % (year, month, day, '', year, month, day)\n    btnDownloadCSV = 'Download+CSV'\n    ig_def_dp_cal_clientState = '|0|15,2017,09,2017,%s,%s||[[null,[],null],[{%s},[]],\"11,2017,09,2017,%s,%s\"]' % (month, day, '', month, day)\n    IG_CSS_LINKS_ = 'ig_res/default/ig_monthcalendar.css|ig_res/default/ig_texteditor.css|ig_res/default/ig_shared.css'\n\n    postdata = {'__VIEWSTATE': viewstate,\n                '__VIEWSTATEGENERATOR': viewstategenerator,\n                '__EVENTVALIDATION': eventvalidation,\n                'FromDatePicker_clientState': FromDatePicker_clientState,\n                'ToDatePicker_clientState': ToDatePicker_clientState,\n                'btnDownloadCSV': btnDownloadCSV,\n                '_ig_def_dp_cal_clientState': ig_def_dp_cal_clientState,\n                '_IG_CSS_LINKS_': IG_CSS_LINKS_\n               }\n\n    postheaders = {\n                   'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',\n                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n                   'Content-Type': 'application/x-www-form-urlencoded'\n                  }\n\n    datareq = s.post(url, headers=postheaders, data=postdata)\n\n    return datareq.text\n\n\ndef add_default_tz(timestamp):\n    \"\"\"\n    Adds Northern Ireland timezone to datetime object if tz = None.\n    \"\"\"\n\n    NIR = tz.gettz('Europe/Belfast')\n    modified_timestamp = timestamp.replace(tzinfo=timestamp.tzinfo or NIR)\n\n    return modified_timestamp\n\n\ndef create_thermal_df(text_data):\n    \"\"\"\n    Turns thermal csv data into a usable dataframe.\n    \"\"\"\n\n    cols_to_use = [0, 1, 2, 3, 4, 5]\n    df_thermal = pd.read_csv(StringIO(text_data),\n                             usecols=cols_to_use)\n    df_thermal.fillna(0.0, inplace=True)\n\n    return df_thermal\n\n\ndef create_wind_df(text_data):\n    \"\"\"\n    Turns wind csv data into a usable dataframe.\n    \"\"\"\n\n    cols_to_use = [0, 1]\n    df_wind = pd.read_csv(StringIO(text_data),\n                          usecols=cols_to_use)\n    df_wind.fillna(0.0, inplace=True)\n\n    return df_wind\n\n\ndef create_exchange_df(text_data):\n    \"\"\"\n    Turns exchange csv data into a usable dataframe.\n    \"\"\"\n\n    df_exchange = pd.read_csv(StringIO(text_data))\n    df_exchange.fillna(0.0, inplace=True)\n\n    return df_exchange\n\n\ndef thermal_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for thermal production.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = row['TimeStamp']\n        snapshot['gas'] = row['Gas_MW']\n        snapshot['coal'] = row['Coal_MW']\n        snapshot['oil'] = row['Distillate_MW'] + row['Diesel_MW']\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef wind_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for wind production.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = row['TimeStamp']\n        snapshot['wind'] = row['Total_Wind_Generated_MW']\n        if snapshot['wind'] > -20:\n            snapshot['wind'] = max(snapshot['wind'], 0)\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef moyle_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for GB exchange.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],\n                                                           dayfirst=True))\n        snapshot['netFlow'] = row['Total_Moyle_Load_MW']\n        snapshot['source'] = 'soni.ltd.uk'\n        snapshot['sortedZoneKeys'] = 'GB->GB-NIR'\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef IE_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for IE exchange.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],\n                                                           dayfirst=True))\n        netFlow = (row['Total_Str_Let_Load_MW'] +\n                   row['Total_Enn_Cor_Load_MW'] +\n                   row['Total_Tan_Lou_Load_MW'])\n        snapshot['netFlow'] = -1 * (netFlow)\n        snapshot['source'] = 'soni.ltd.uk'\n        snapshot['sortedZoneKeys'] = 'GB-NIR->IE'\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef merge_production(thermal_data, wind_data):\n    \"\"\"\n    Joins thermal and wind production data on shared datetime key.\n    Returns a list.\n    \"\"\"\n\n    total_production = thermal_data + wind_data\n\n    # Join thermal and wind dicts on 'datetime' key.\n    d = defaultdict(dict)\n    for elem in total_production:\n        d[elem['datetime']].update(elem)\n\n    joined_data = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    for datapoint in joined_data:\n        datapoint['datetime'] = add_default_tz(parser.parse(datapoint['datetime'], dayfirst=True))\n\n    return joined_data\n\n\ndef fetch_production(zone_key='GB-NIR', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n        Arguments:\n        zone_key (optional) -- used in case a parser is able to fetch multiple countries\n        session (optional)      -- request session passed in order to re-use an existing session\n        Return:\n        A dictionary in the form:\n        {\n          'zoneKey': 'FR',\n          'datetime': '2017-01-01T00:00:00Z',\n          'production': {\n              'biomass': 0.0,\n              'coal': 0.0,\n              'gas': 0.0,\n              'hydro': 0.0,\n              'nuclear': null,\n              'oil': 0.0,\n              'solar': 0.0,\n              'wind': 0.0,\n              'geothermal': 0.0,\n              'unknown': 0.0\n          },\n          'storage': {\n              'hydro': -10.0,\n          },\n          'source': 'mysource.com'\n        }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    thermal_data = get_data(thermal_url)\n    wind_data = get_data(wind_url)\n    thermal_df = create_thermal_df(thermal_data)\n    wind_df = create_wind_df(wind_data)\n    thermal = thermal_processor(thermal_df)\n    wind = wind_processor(wind_df)\n    merge = merge_production(thermal, wind)\n\n    production_mix_by_quarter_hour = []\n\n    for datapoint in merge:\n        production_mix = {\n            'zoneKey': zone_key,\n            'datetime': datapoint.get('datetime', 0.0),\n            'production': {\n                'coal': datapoint.get('coal', 0.0),\n                'gas': datapoint.get('gas', 0.0),\n                'oil': datapoint.get('oil', 0.0),\n                'solar': None,\n                'wind': datapoint.get('wind', 0.0)\n            },\n            'source': 'soni.ltd.uk'\n        }\n        production_mix_by_quarter_hour.append(\n            validate(production_mix, logger=logger, required=['gas', 'coal']))\n\n    return production_mix_by_quarter_hour\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two countries\n    Arguments:\n    zone_key (optional) -- used in case a parser is able to fetch multiple countries\n    session (optional)      -- request session passed in order to re-use an existing session\n    Return:\n    A dictionary in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    exchange_data = get_data(exchange_url)\n    exchange_dataframe = create_exchange_df(exchange_data)\n    if '->'.join(sorted([zone_key1, zone_key2])) == 'GB->GB-NIR':\n        moyle = moyle_processor(exchange_dataframe)\n        return moyle\n    elif '->'.join(sorted([zone_key1, zone_key2])) == 'GB-NIR->IE':\n        IE = IE_processor(exchange_dataframe)\n        return IE\n    else:\n        raise NotImplementedError('This exchange pair is not implemented')\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n    print('fetch_exchange(GB-NIR, GB) ->')\n    print(fetch_exchange('GB-NIR', 'GB'))\n    print('fetch_exchange(GB-NIR, IE) ->')\n    print(fetch_exchange('GB-NIR', 'IE'))\n", "path": "parsers/GB_NIR.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\nfrom collections import defaultdict\nfrom datetime import datetime\nfrom io import StringIO\nfrom operator import itemgetter\n\nimport logging\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup\nfrom dateutil import parser, tz\n\nfrom .lib.validation import validate\n\nthermal_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/FuelMix.aspx'\nwind_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/SystemOutput.aspx'\nexchange_url = 'http://ws.soni.ltd.uk/DownloadCentre/aspx/MoyleTie.aspx'\n# Positive values represent imports to Northern Ireland.\n# Negative value represent exports from Northern Ireland.\n\n\ndef get_data(url, session=None):\n    \"\"\"\n    Requests data from a specified url in CSV format.\n    Returns a response.text object.\n    \"\"\"\n\n    s = session or requests.Session()\n\n    headers = {\n        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',\n        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'\n    }\n\n    pagereq = requests.get(url, headers=headers)\n    soup = BeautifulSoup(pagereq.text, 'html.parser')\n\n    # Find and define parameters needed to send a POST request for the actual data.\n    viewstategenerator = soup.find(\"input\", attrs={'id': '__VIEWSTATEGENERATOR'})['value']\n    viewstate = soup.find(\"input\", attrs={'id': '__VIEWSTATE'})['value']\n    eventvalidation = soup.find(\"input\", attrs={'id': '__EVENTVALIDATION'})['value']\n\n    # Set date for post request.\n    current_date = datetime.now().date()\n    month = current_date.month\n    day = current_date.day\n    year = current_date.year\n\n    FromDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],\"01%s-%s-%s-0-0-0-0\"]' % (year, month, day, '', year, month, day)\n    ToDatePicker_clientState = '|0|01%s-%s-%s-0-0-0-0||[[[[]],[],[]],[{%s},[]],\"01%s-%s-%s-0-0-0-0\"]' % (year, month, day, '', year, month, day)\n    btnDownloadCSV = 'Download+CSV'\n    ig_def_dp_cal_clientState = '|0|15,2017,09,2017,%s,%s||[[null,[],null],[{%s},[]],\"11,2017,09,2017,%s,%s\"]' % (month, day, '', month, day)\n    IG_CSS_LINKS_ = 'ig_res/default/ig_monthcalendar.css|ig_res/default/ig_texteditor.css|ig_res/default/ig_shared.css'\n\n    postdata = {'__VIEWSTATE': viewstate,\n                '__VIEWSTATEGENERATOR': viewstategenerator,\n                '__EVENTVALIDATION': eventvalidation,\n                'FromDatePicker_clientState': FromDatePicker_clientState,\n                'ToDatePicker_clientState': ToDatePicker_clientState,\n                'btnDownloadCSV': btnDownloadCSV,\n                '_ig_def_dp_cal_clientState': ig_def_dp_cal_clientState,\n                '_IG_CSS_LINKS_': IG_CSS_LINKS_\n               }\n\n    postheaders = {\n                   'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0',\n                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n                   'Content-Type': 'application/x-www-form-urlencoded'\n                  }\n\n    datareq = s.post(url, headers=postheaders, data=postdata)\n\n    return datareq.text\n\n\ndef add_default_tz(timestamp):\n    \"\"\"\n    Adds Northern Ireland timezone to datetime object if tz = None.\n    \"\"\"\n\n    NIR = tz.gettz('Europe/Belfast')\n    modified_timestamp = timestamp.replace(tzinfo=timestamp.tzinfo or NIR)\n\n    return modified_timestamp\n\n\ndef create_thermal_df(text_data):\n    \"\"\"\n    Turns thermal csv data into a usable dataframe.\n    \"\"\"\n\n    cols_to_use = [0, 1, 2, 3, 4, 5]\n    df_thermal = pd.read_csv(StringIO(text_data),\n                             usecols=cols_to_use)\n    df_thermal.fillna(0.0, inplace=True)\n\n    return df_thermal\n\n\ndef create_wind_df(text_data):\n    \"\"\"\n    Turns wind csv data into a usable dataframe.\n    \"\"\"\n\n    cols_to_use = [0, 1]\n    df_wind = pd.read_csv(StringIO(text_data),\n                          usecols=cols_to_use)\n    df_wind.fillna(0.0, inplace=True)\n\n    return df_wind\n\n\ndef create_exchange_df(text_data):\n    \"\"\"\n    Turns exchange csv data into a usable dataframe.\n    \"\"\"\n\n    df_exchange = pd.read_csv(StringIO(text_data))\n    df_exchange.fillna(0.0, inplace=True)\n\n    return df_exchange\n\n\ndef thermal_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for thermal production.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = row['TimeStamp']\n        snapshot['gas'] = row['Gas_MW']\n        snapshot['coal'] = row['Coal_MW']\n        snapshot['oil'] = row['Distillate_MW'] + row['Diesel_MW']\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef wind_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for wind production.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = row['TimeStamp']\n        snapshot['wind'] = row['Total_Wind_Generated_MW']\n        if snapshot['wind'] > -20:\n            snapshot['wind'] = max(snapshot['wind'], 0)\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef moyle_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for GB exchange.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],\n                                                           dayfirst=True))\n        snapshot['netFlow'] = row['Total_Moyle_Load_MW']\n        snapshot['source'] = 'soni.ltd.uk'\n        snapshot['sortedZoneKeys'] = 'GB->GB-NIR'\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef IE_processor(df):\n    \"\"\"\n    Creates quarter hour datapoints for IE exchange.\n    Returns a list.\n    \"\"\"\n\n    datapoints = []\n    for index, row in df.iterrows():\n        snapshot = {}\n        snapshot['datetime'] = add_default_tz(parser.parse(row['TimeStamp'],\n                                                           dayfirst=True))\n        netFlow = (row['Total_Str_Let_Load_MW'] +\n                   row['Total_Enn_Cor_Load_MW'] +\n                   row['Total_Tan_Lou_Load_MW'])\n        snapshot['netFlow'] = -1 * (netFlow)\n        snapshot['source'] = 'soni.ltd.uk'\n        snapshot['sortedZoneKeys'] = 'GB-NIR->IE'\n        datapoints.append(snapshot)\n\n    return datapoints\n\n\ndef merge_production(thermal_data, wind_data):\n    \"\"\"\n    Joins thermal and wind production data on shared datetime key.\n    Returns a list.\n    \"\"\"\n\n    total_production = thermal_data + wind_data\n\n    # Join thermal and wind dicts on 'datetime' key.\n    d = defaultdict(dict)\n    for elem in total_production:\n        d[elem['datetime']].update(elem)\n\n    joined_data = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    for datapoint in joined_data:\n        datapoint['datetime'] = add_default_tz(parser.parse(datapoint['datetime'], dayfirst=True))\n\n    return joined_data\n\n\ndef fetch_production(zone_key='GB-NIR', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n        Arguments:\n        zone_key (optional) -- used in case a parser is able to fetch multiple countries\n        session (optional)      -- request session passed in order to re-use an existing session\n        Return:\n        A dictionary in the form:\n        {\n          'zoneKey': 'FR',\n          'datetime': '2017-01-01T00:00:00Z',\n          'production': {\n              'biomass': 0.0,\n              'coal': 0.0,\n              'gas': 0.0,\n              'hydro': 0.0,\n              'nuclear': null,\n              'oil': 0.0,\n              'solar': 0.0,\n              'wind': 0.0,\n              'geothermal': 0.0,\n              'unknown': 0.0\n          },\n          'storage': {\n              'hydro': -10.0,\n          },\n          'source': 'mysource.com'\n        }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    thermal_data = get_data(thermal_url)\n    wind_data = get_data(wind_url)\n    thermal_df = create_thermal_df(thermal_data)\n    wind_df = create_wind_df(wind_data)\n    thermal = thermal_processor(thermal_df)\n    wind = wind_processor(wind_df)\n    merge = merge_production(thermal, wind)\n\n    production_mix_by_quarter_hour = []\n\n    for datapoint in merge:\n        production_mix = {\n            'zoneKey': zone_key,\n            'datetime': datapoint.get('datetime', 0.0),\n            'production': {\n                'coal': datapoint.get('coal', 0.0),\n                'gas': datapoint.get('gas', 0.0),\n                'oil': datapoint.get('oil', 0.0),\n                'solar': None,\n                'wind': datapoint.get('wind', 0.0)\n            },\n            'source': 'soni.ltd.uk'\n        }\n        production_mix_by_quarter_hour.append(\n            validate(production_mix, logger=logger, required=['gas', 'coal'], floor=1.0))\n\n    return production_mix_by_quarter_hour\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two countries\n    Arguments:\n    zone_key (optional) -- used in case a parser is able to fetch multiple countries\n    session (optional)      -- request session passed in order to re-use an existing session\n    Return:\n    A dictionary in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    exchange_data = get_data(exchange_url)\n    exchange_dataframe = create_exchange_df(exchange_data)\n    if '->'.join(sorted([zone_key1, zone_key2])) == 'GB->GB-NIR':\n        moyle = moyle_processor(exchange_dataframe)\n        return moyle\n    elif '->'.join(sorted([zone_key1, zone_key2])) == 'GB-NIR->IE':\n        IE = IE_processor(exchange_dataframe)\n        return IE\n    else:\n        raise NotImplementedError('This exchange pair is not implemented')\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n    print('fetch_exchange(GB-NIR, GB) ->')\n    print(fetch_exchange('GB-NIR', 'GB'))\n    print('fetch_exchange(GB-NIR, IE) ->')\n    print(fetch_exchange('GB-NIR', 'IE'))\n", "path": "parsers/GB_NIR.py"}]}