We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Midnight glitch in UK-GB data
Reported wind generation drops to zero every day at midnight. 

<img width="361" alt="Screenshot 2019-05-04 at 13 22 17" src="https://user-images.githubusercontent.com/13368547/57179023-fd811b00-6e70-11e9-99f7-814c0cfcd3cd.png">

Possibly related to #1779?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/ELEXON.py`
Content:
```
1 #!/usr/bin/env python3
2 # coding=utf-8
3 
4 """
5 Parser that uses the ELEXON API to return the following data types.
6 
7 Production
8 Exchanges
9 
10 Documentation:
11 https://www.elexon.co.uk/wp-content/uploads/2017/06/
12 bmrs_api_data_push_user_guide_v1.1.pdf
13 """
14 
15 import os
16 import arrow
17 import logging
18 import requests
19 import datetime as dt
20 import pandas as pd
21 from io import StringIO
22 
23 from .lib.validation import validate
24 
25 ELEXON_ENDPOINT = 'https://api.bmreports.com/BMRS/{}/v1'
26 
27 REPORT_META = {
28     'B1620': {
29         'expected_fields': 13,
30         'skiprows': 5
31     },
32     'FUELINST': {
33         'expected_fields': 19,
34         'skiprows': 1
35     },
36     'INTERFUELHH': {
37         'expected_fields': 8,
38         'skiprows': 0
39     }
40 }
41 
42 # 'hydro' key is for hydro production
43 # 'hydro storage' key is for hydro storage
44 RESOURCE_TYPE_TO_FUEL = {
45     'Biomass': 'biomass',
46     'Fossil Gas': 'gas',
47     'Fossil Hard coal': 'coal',
48     'Fossil Oil': 'oil',
49     'Hydro Pumped Storage': 'hydro storage',
50     'Hydro Run-of-river and poundage': 'hydro',
51     'Nuclear': 'nuclear',
52     'Solar': 'solar',
53     'Wind Onshore': 'wind',
54     'Wind Offshore': 'wind',
55     'Other': 'unknown'
56 }
57 
58 EXCHANGES = {
59     'FR->GB': 3,
60     'GB-NIR->IE': 4,
61     'GB->NL': 5,
62     'GB->IE': 6,
63     'BE->GB': 7
64 }
65 
66 FETCH_WIND_FROM_FUELINST = True
67 
68 
69 def query_ELEXON(report, session, params):
70     if 'ELEXON_TOKEN' not in os.environ:
71         raise Exception('No ELEXON_TOKEN found! Please add it to secrets.env!')
72     params['APIKey'] = os.environ['ELEXON_TOKEN']
73     return session.get(ELEXON_ENDPOINT.format(report), params=params)
74 
75 
76 def query_exchange(session, target_datetime=None):
77     if target_datetime is None:
78         target_datetime = dt.date.today()
79 
80     from_date = (target_datetime - dt.timedelta(days=1)).strftime('%Y-%m-%d')
81     to_date = target_datetime.strftime('%Y-%m-%d')
82 
83     params = {
84         'FromDate': from_date,
85         'ToDate': to_date,
86         'ServiceType': 'csv'
87     }
88     response = query_ELEXON('INTERFUELHH', session, params)
89     return response.text
90 
91 
92 def query_production(session, target_datetime=None):
93     if target_datetime is None:
94         target_datetime = dt.datetime.now()
95 
96     # we can only fetch one date at a time.
97     # if target_datetime is first 30 minutes of the day fetch the day before.
98     # otherwise fetch the day of target_datetime.
99     if target_datetime.time() <= dt.time(0, 30):
100         settlement_date = target_datetime.date() - dt.timedelta(1)
101     else:
102         settlement_date = target_datetime.date()
103 
104     params = {
105         'SettlementDate': settlement_date.strftime('%Y-%m-%d'),
106         'Period': '*',
107         'ServiceType': 'csv'
108     }
109     response = query_ELEXON('B1620', session, params)
110     return response.text
111 
112 
113 def parse_exchange(zone_key1, zone_key2, csv_text, target_datetime=None,
114                    logger=logging.getLogger(__name__)):
115     if not csv_text:
116         return None
117 
118     report = REPORT_META['INTERFUELHH']
119 
120     sorted_zone_keys = sorted([zone_key1, zone_key2])
121     exchange = '->'.join(sorted_zone_keys)
122     data_points = list()
123     lines = csv_text.split('\n')
124 
125     # check field count in report is as expected
126     field_count = len(lines[1].split(','))
127     if field_count != report['expected_fields']:
128         raise ValueError(
129             'Expected {} fields in INTERFUELHH report, got {}'.format(
130                 report['expected_fields'], field_count))
131 
132     for line in lines[1:-1]:
133         fields = line.split(',')
134 
135         # settlement date / period combinations are always local time
136         date = dt.datetime.strptime(fields[1], '%Y%m%d').date()
137         settlement_period = int(fields[2])
138         datetime = datetime_from_date_sp(date, settlement_period)
139 
140         data = {
141             'sortedZoneKeys': exchange,
142             'datetime': datetime,
143             'source': 'bmreports.com'
144         }
145 
146         # positive value implies import to GB
147         multiplier = -1 if 'GB' in sorted_zone_keys[0] else 1
148         data['netFlow'] = float(fields[EXCHANGES[exchange]]) * multiplier
149         data_points.append(data)
150 
151     return data_points
152 
153 
154 def parse_production(csv_text, target_datetime=None,
155                      logger=logging.getLogger(__name__)):
156     if not csv_text:
157         return None
158 
159     report = REPORT_META['B1620']
160 
161     # create DataFrame from slice of CSV rows
162     df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'] - 1)
163 
164     # check field count in report is as expected
165     field_count = len(df.columns)
166     if field_count != report['expected_fields']:
167         raise ValueError(
168             'Expected {} fields in B1620 report, got {}'.format(
169                 report['expected_fields'], len(df.columns)))
170 
171     # filter out undesired columns
172     df = df.iloc[:-1, [7, 8, 9, 4]]
173 
174     df['Settlement Date'] = df['Settlement Date'].apply(
175         lambda x: dt.datetime.strptime(x, '%Y-%m-%d'))
176     df['Settlement Period'] = df['Settlement Period'].astype(int)
177     df['datetime'] = df.apply(lambda x: datetime_from_date_sp(
178         x['Settlement Date'], x['Settlement Period']), axis=1)
179 
180     # map from report fuel names to electricitymap fuel names
181     fuel_column = 'Power System Resource  Type'
182     df[fuel_column] = df[fuel_column].apply(lambda x: RESOURCE_TYPE_TO_FUEL[x])
183 
184     # loop through unique datetimes and create each data point
185     data_points = list()
186     for time in pd.unique(df['datetime']):
187         time_df = df[df['datetime'] == time]
188 
189         data_point = {
190             'zoneKey': 'GB',
191             'datetime': time.to_pydatetime(),
192             'source': 'bmreports.com',
193             'production': dict(),
194             'storage': dict()
195         }
196 
197         for row in time_df.iterrows():
198             fields = row[1].to_dict()
199             fuel = fields[fuel_column]
200             quantity = fields['Quantity']
201 
202             # check if storage value and if so correct key
203             if 'storage' in fuel:
204                 fuel_key = fuel.replace('storage', '').strip()
205                 # ELEXON storage is negative when storing and positive when
206                 # discharging (the opposite to electricitymap)
207                 data_point['storage'][fuel_key] = quantity * -1
208             else:
209                 # if/else structure allows summation of multiple quantities
210                 # e.g. 'Wind Onshore' and 'Wind Offshore' both have the
211                 # key 'wind' here.
212                 if fuel in data_point['production'].keys():
213                     data_point['production'][fuel] += quantity
214                 else:
215                     data_point['production'][fuel] = quantity
216 
217         data_points.append(data_point)
218 
219     return data_points
220 
221 
222 def datetime_from_date_sp(date, sp):
223     datetime = arrow.get(date).shift(minutes=30 * (sp - 1))
224     return datetime.replace(tzinfo='Europe/London').datetime
225 
226 
227 def _fetch_wind(target_datetime=None):
228     if target_datetime is None:
229         target_datetime = dt.datetime.now()
230 
231     # line up with B1620 (main production report) search range
232     d = target_datetime.date()
233     offset = 1 if target_datetime.time() <= dt.time(0, 30) else 0
234     start = dt.datetime.combine(d - dt.timedelta(days=offset), dt.time(0))
235     end = dt.datetime.combine(d + dt.timedelta(days=1), dt.time(0))
236 
237     session = requests.session()
238     params = {
239         'FromDateTime': start.strftime('%Y-%m-%d %H:%M:%S'),
240         'ToDateTime': end.strftime('%Y-%m-%d %H:%M:%S'),
241         'ServiceType': 'csv'
242     }
243     response = query_ELEXON('FUELINST', session, params)
244     csv_text = response.text
245 
246     report = REPORT_META['FUELINST']
247     df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'],
248                      skipfooter=1, header=None)
249 
250     field_count = len(df.columns)
251     if field_count != report['expected_fields']:
252         raise ValueError(
253             'Expected {} fields in FUELINST report, got {}'.format(
254                 report['expected_fields'], len(df.columns)))
255 
256     df = df.iloc[:, [1, 2, 3, 8]]
257     df.columns = ['Settlement Date', 'Settlement Period', 'published', 'Wind']
258     df['Settlement Date'] = df['Settlement Date'].apply(
259         lambda x: dt.datetime.strptime(str(x), '%Y%m%d'))
260     df['Settlement Period'] = df['Settlement Period'].astype(int)
261     df['datetime'] = df.apply(lambda x: datetime_from_date_sp(
262         x['Settlement Date'], x['Settlement Period']), axis=1)
263 
264     df['published'] = df['published'].apply(
265         lambda x: dt.datetime.strptime(str(x), '%Y%m%d%H%M%S'))
266     # get the most recently published value for each datetime
267     idx = df.groupby('datetime')['published'].transform(max) == df['published']
268     df = df[idx]
269 
270     return df[['datetime', 'Wind']]
271 
272 
273 def fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,
274                    logger=logging.getLogger(__name__)):
275     session = session or requests.session()
276     response = query_exchange(session, target_datetime)
277     data = parse_exchange(zone_key1, zone_key2, response, target_datetime,
278                           logger)
279     return data
280 
281 
282 def fetch_production(zone_key='GB', session=None, target_datetime=None,
283                      logger=logging.getLogger(__name__)):
284     session = session or requests.session()
285     response = query_production(session, target_datetime)
286     data = parse_production(response, target_datetime, logger)
287 
288     # At times B1620 has had poor quality data for wind so fetch from FUELINST
289     if FETCH_WIND_FROM_FUELINST:
290         wind = _fetch_wind(target_datetime)
291         for entry in data:
292             datetime = entry['datetime']
293             wind_row = wind[wind['datetime'] == datetime]
294             if len(wind_row):
295                 entry['production']['wind'] = wind_row.iloc[0]['Wind']
296             else:
297                 entry['production']['wind'] = None
298 
299     required = ['coal', 'gas', 'nuclear']
300     expected_range = {
301         'coal': (0, 10000),
302         'gas': (100, 30000),
303         'nuclear': (100, 20000)
304     }
305     data = [x for x in data
306             if validate(
307                 x, logger, required=required, expected_range=expected_range)]
308 
309     return data
310 
311 
312 if __name__ == '__main__':
313     """Main method, never used by the Electricity Map backend, but handy 
314     for testing."""
315 
316     print('fetch_production() ->')
317     print(fetch_production())
318 
319     print('fetch_exchange(FR, GB) ->')
320     print(fetch_exchange('FR', 'GB'))
321 
322     print('fetch_exchange(GB, IE) ->')
323     print(fetch_exchange('GB', 'IE'))
324 
325     print('fetch_exchange(GB, NL) ->')
326     print(fetch_exchange('GB', 'NL'))
327
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/ELEXON.py b/parsers/ELEXON.py
--- a/parsers/ELEXON.py
+++ b/parsers/ELEXON.py
@@ -230,8 +230,7 @@
 
     # line up with B1620 (main production report) search range
     d = target_datetime.date()
-    offset = 1 if target_datetime.time() <= dt.time(0, 30) else 0
-    start = dt.datetime.combine(d - dt.timedelta(days=offset), dt.time(0))
+    start = d - dt.timedelta(hours=24)
     end = dt.datetime.combine(d + dt.timedelta(days=1), dt.time(0))
 
     session = requests.session()

{"golden_diff": "diff --git a/parsers/ELEXON.py b/parsers/ELEXON.py\n--- a/parsers/ELEXON.py\n+++ b/parsers/ELEXON.py\n@@ -230,8 +230,7 @@\n \n     # line up with B1620 (main production report) search range\n     d = target_datetime.date()\n-    offset = 1 if target_datetime.time() <= dt.time(0, 30) else 0\n-    start = dt.datetime.combine(d - dt.timedelta(days=offset), dt.time(0))\n+    start = d - dt.timedelta(hours=24)\n     end = dt.datetime.combine(d + dt.timedelta(days=1), dt.time(0))\n \n     session = requests.session()\n", "issue": "Midnight glitch in UK-GB data\nReported wind generation drops to zero every day at midnight. \r\n\r\n<img width=\"361\" alt=\"Screenshot 2019-05-04 at 13 22 17\" src=\"https://user-images.githubusercontent.com/13368547/57179023-fd811b00-6e70-11e9-99f7-814c0cfcd3cd.png\">\r\n\r\nPossibly related to #1779?\n", "before_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\n\n\"\"\"\nParser that uses the ELEXON API to return the following data types.\n\nProduction\nExchanges\n\nDocumentation:\nhttps://www.elexon.co.uk/wp-content/uploads/2017/06/\nbmrs_api_data_push_user_guide_v1.1.pdf\n\"\"\"\n\nimport os\nimport arrow\nimport logging\nimport requests\nimport datetime as dt\nimport pandas as pd\nfrom io import StringIO\n\nfrom .lib.validation import validate\n\nELEXON_ENDPOINT = 'https://api.bmreports.com/BMRS/{}/v1'\n\nREPORT_META = {\n    'B1620': {\n        'expected_fields': 13,\n        'skiprows': 5\n    },\n    'FUELINST': {\n        'expected_fields': 19,\n        'skiprows': 1\n    },\n    'INTERFUELHH': {\n        'expected_fields': 8,\n        'skiprows': 0\n    }\n}\n\n# 'hydro' key is for hydro production\n# 'hydro storage' key is for hydro storage\nRESOURCE_TYPE_TO_FUEL = {\n    'Biomass': 'biomass',\n    'Fossil Gas': 'gas',\n    'Fossil Hard coal': 'coal',\n    'Fossil Oil': 'oil',\n    'Hydro Pumped Storage': 'hydro storage',\n    'Hydro Run-of-river and poundage': 'hydro',\n    'Nuclear': 'nuclear',\n    'Solar': 'solar',\n    'Wind Onshore': 'wind',\n    'Wind Offshore': 'wind',\n    'Other': 'unknown'\n}\n\nEXCHANGES = {\n    'FR->GB': 3,\n    'GB-NIR->IE': 4,\n    'GB->NL': 5,\n    'GB->IE': 6,\n    'BE->GB': 7\n}\n\nFETCH_WIND_FROM_FUELINST = True\n\n\ndef query_ELEXON(report, session, params):\n    if 'ELEXON_TOKEN' not in os.environ:\n        raise Exception('No ELEXON_TOKEN found! Please add it to secrets.env!')\n    params['APIKey'] = os.environ['ELEXON_TOKEN']\n    return session.get(ELEXON_ENDPOINT.format(report), params=params)\n\n\ndef query_exchange(session, target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.date.today()\n\n    from_date = (target_datetime - dt.timedelta(days=1)).strftime('%Y-%m-%d')\n    to_date = target_datetime.strftime('%Y-%m-%d')\n\n    params = {\n        'FromDate': from_date,\n        'ToDate': to_date,\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('INTERFUELHH', session, params)\n    return response.text\n\n\ndef query_production(session, target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.datetime.now()\n\n    # we can only fetch one date at a time.\n    # if target_datetime is first 30 minutes of the day fetch the day before.\n    # otherwise fetch the day of target_datetime.\n    if target_datetime.time() <= dt.time(0, 30):\n        settlement_date = target_datetime.date() - dt.timedelta(1)\n    else:\n        settlement_date = target_datetime.date()\n\n    params = {\n        'SettlementDate': settlement_date.strftime('%Y-%m-%d'),\n        'Period': '*',\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('B1620', session, params)\n    return response.text\n\n\ndef parse_exchange(zone_key1, zone_key2, csv_text, target_datetime=None,\n                   logger=logging.getLogger(__name__)):\n    if not csv_text:\n        return None\n\n    report = REPORT_META['INTERFUELHH']\n\n    sorted_zone_keys = sorted([zone_key1, zone_key2])\n    exchange = '->'.join(sorted_zone_keys)\n    data_points = list()\n    lines = csv_text.split('\\n')\n\n    # check field count in report is as expected\n    field_count = len(lines[1].split(','))\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in INTERFUELHH report, got {}'.format(\n                report['expected_fields'], field_count))\n\n    for line in lines[1:-1]:\n        fields = line.split(',')\n\n        # settlement date / period combinations are always local time\n        date = dt.datetime.strptime(fields[1], '%Y%m%d').date()\n        settlement_period = int(fields[2])\n        datetime = datetime_from_date_sp(date, settlement_period)\n\n        data = {\n            'sortedZoneKeys': exchange,\n            'datetime': datetime,\n            'source': 'bmreports.com'\n        }\n\n        # positive value implies import to GB\n        multiplier = -1 if 'GB' in sorted_zone_keys[0] else 1\n        data['netFlow'] = float(fields[EXCHANGES[exchange]]) * multiplier\n        data_points.append(data)\n\n    return data_points\n\n\ndef parse_production(csv_text, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    if not csv_text:\n        return None\n\n    report = REPORT_META['B1620']\n\n    # create DataFrame from slice of CSV rows\n    df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'] - 1)\n\n    # check field count in report is as expected\n    field_count = len(df.columns)\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in B1620 report, got {}'.format(\n                report['expected_fields'], len(df.columns)))\n\n    # filter out undesired columns\n    df = df.iloc[:-1, [7, 8, 9, 4]]\n\n    df['Settlement Date'] = df['Settlement Date'].apply(\n        lambda x: dt.datetime.strptime(x, '%Y-%m-%d'))\n    df['Settlement Period'] = df['Settlement Period'].astype(int)\n    df['datetime'] = df.apply(lambda x: datetime_from_date_sp(\n        x['Settlement Date'], x['Settlement Period']), axis=1)\n\n    # map from report fuel names to electricitymap fuel names\n    fuel_column = 'Power System Resource  Type'\n    df[fuel_column] = df[fuel_column].apply(lambda x: RESOURCE_TYPE_TO_FUEL[x])\n\n    # loop through unique datetimes and create each data point\n    data_points = list()\n    for time in pd.unique(df['datetime']):\n        time_df = df[df['datetime'] == time]\n\n        data_point = {\n            'zoneKey': 'GB',\n            'datetime': time.to_pydatetime(),\n            'source': 'bmreports.com',\n            'production': dict(),\n            'storage': dict()\n        }\n\n        for row in time_df.iterrows():\n            fields = row[1].to_dict()\n            fuel = fields[fuel_column]\n            quantity = fields['Quantity']\n\n            # check if storage value and if so correct key\n            if 'storage' in fuel:\n                fuel_key = fuel.replace('storage', '').strip()\n                # ELEXON storage is negative when storing and positive when\n                # discharging (the opposite to electricitymap)\n                data_point['storage'][fuel_key] = quantity * -1\n            else:\n                # if/else structure allows summation of multiple quantities\n                # e.g. 'Wind Onshore' and 'Wind Offshore' both have the\n                # key 'wind' here.\n                if fuel in data_point['production'].keys():\n                    data_point['production'][fuel] += quantity\n                else:\n                    data_point['production'][fuel] = quantity\n\n        data_points.append(data_point)\n\n    return data_points\n\n\ndef datetime_from_date_sp(date, sp):\n    datetime = arrow.get(date).shift(minutes=30 * (sp - 1))\n    return datetime.replace(tzinfo='Europe/London').datetime\n\n\ndef _fetch_wind(target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.datetime.now()\n\n    # line up with B1620 (main production report) search range\n    d = target_datetime.date()\n    offset = 1 if target_datetime.time() <= dt.time(0, 30) else 0\n    start = dt.datetime.combine(d - dt.timedelta(days=offset), dt.time(0))\n    end = dt.datetime.combine(d + dt.timedelta(days=1), dt.time(0))\n\n    session = requests.session()\n    params = {\n        'FromDateTime': start.strftime('%Y-%m-%d %H:%M:%S'),\n        'ToDateTime': end.strftime('%Y-%m-%d %H:%M:%S'),\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('FUELINST', session, params)\n    csv_text = response.text\n\n    report = REPORT_META['FUELINST']\n    df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'],\n                     skipfooter=1, header=None)\n\n    field_count = len(df.columns)\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in FUELINST report, got {}'.format(\n                report['expected_fields'], len(df.columns)))\n\n    df = df.iloc[:, [1, 2, 3, 8]]\n    df.columns = ['Settlement Date', 'Settlement Period', 'published', 'Wind']\n    df['Settlement Date'] = df['Settlement Date'].apply(\n        lambda x: dt.datetime.strptime(str(x), '%Y%m%d'))\n    df['Settlement Period'] = df['Settlement Period'].astype(int)\n    df['datetime'] = df.apply(lambda x: datetime_from_date_sp(\n        x['Settlement Date'], x['Settlement Period']), axis=1)\n\n    df['published'] = df['published'].apply(\n        lambda x: dt.datetime.strptime(str(x), '%Y%m%d%H%M%S'))\n    # get the most recently published value for each datetime\n    idx = df.groupby('datetime')['published'].transform(max) == df['published']\n    df = df[idx]\n\n    return df[['datetime', 'Wind']]\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,\n                   logger=logging.getLogger(__name__)):\n    session = session or requests.session()\n    response = query_exchange(session, target_datetime)\n    data = parse_exchange(zone_key1, zone_key2, response, target_datetime,\n                          logger)\n    return data\n\n\ndef fetch_production(zone_key='GB', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    session = session or requests.session()\n    response = query_production(session, target_datetime)\n    data = parse_production(response, target_datetime, logger)\n\n    # At times B1620 has had poor quality data for wind so fetch from FUELINST\n    if FETCH_WIND_FROM_FUELINST:\n        wind = _fetch_wind(target_datetime)\n        for entry in data:\n            datetime = entry['datetime']\n            wind_row = wind[wind['datetime'] == datetime]\n            if len(wind_row):\n                entry['production']['wind'] = wind_row.iloc[0]['Wind']\n            else:\n                entry['production']['wind'] = None\n\n    required = ['coal', 'gas', 'nuclear']\n    expected_range = {\n        'coal': (0, 10000),\n        'gas': (100, 30000),\n        'nuclear': (100, 20000)\n    }\n    data = [x for x in data\n            if validate(\n                x, logger, required=required, expected_range=expected_range)]\n\n    return data\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy \n    for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n\n    print('fetch_exchange(FR, GB) ->')\n    print(fetch_exchange('FR', 'GB'))\n\n    print('fetch_exchange(GB, IE) ->')\n    print(fetch_exchange('GB', 'IE'))\n\n    print('fetch_exchange(GB, NL) ->')\n    print(fetch_exchange('GB', 'NL'))\n", "path": "parsers/ELEXON.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\n\n\"\"\"\nParser that uses the ELEXON API to return the following data types.\n\nProduction\nExchanges\n\nDocumentation:\nhttps://www.elexon.co.uk/wp-content/uploads/2017/06/\nbmrs_api_data_push_user_guide_v1.1.pdf\n\"\"\"\n\nimport os\nimport arrow\nimport logging\nimport requests\nimport datetime as dt\nimport pandas as pd\nfrom io import StringIO\n\nfrom .lib.validation import validate\n\nELEXON_ENDPOINT = 'https://api.bmreports.com/BMRS/{}/v1'\n\nREPORT_META = {\n    'B1620': {\n        'expected_fields': 13,\n        'skiprows': 5\n    },\n    'FUELINST': {\n        'expected_fields': 19,\n        'skiprows': 1\n    },\n    'INTERFUELHH': {\n        'expected_fields': 8,\n        'skiprows': 0\n    }\n}\n\n# 'hydro' key is for hydro production\n# 'hydro storage' key is for hydro storage\nRESOURCE_TYPE_TO_FUEL = {\n    'Biomass': 'biomass',\n    'Fossil Gas': 'gas',\n    'Fossil Hard coal': 'coal',\n    'Fossil Oil': 'oil',\n    'Hydro Pumped Storage': 'hydro storage',\n    'Hydro Run-of-river and poundage': 'hydro',\n    'Nuclear': 'nuclear',\n    'Solar': 'solar',\n    'Wind Onshore': 'wind',\n    'Wind Offshore': 'wind',\n    'Other': 'unknown'\n}\n\nEXCHANGES = {\n    'FR->GB': 3,\n    'GB-NIR->IE': 4,\n    'GB->NL': 5,\n    'GB->IE': 6,\n    'BE->GB': 7\n}\n\nFETCH_WIND_FROM_FUELINST = True\n\n\ndef query_ELEXON(report, session, params):\n    if 'ELEXON_TOKEN' not in os.environ:\n        raise Exception('No ELEXON_TOKEN found! Please add it to secrets.env!')\n    params['APIKey'] = os.environ['ELEXON_TOKEN']\n    return session.get(ELEXON_ENDPOINT.format(report), params=params)\n\n\ndef query_exchange(session, target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.date.today()\n\n    from_date = (target_datetime - dt.timedelta(days=1)).strftime('%Y-%m-%d')\n    to_date = target_datetime.strftime('%Y-%m-%d')\n\n    params = {\n        'FromDate': from_date,\n        'ToDate': to_date,\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('INTERFUELHH', session, params)\n    return response.text\n\n\ndef query_production(session, target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.datetime.now()\n\n    # we can only fetch one date at a time.\n    # if target_datetime is first 30 minutes of the day fetch the day before.\n    # otherwise fetch the day of target_datetime.\n    if target_datetime.time() <= dt.time(0, 30):\n        settlement_date = target_datetime.date() - dt.timedelta(1)\n    else:\n        settlement_date = target_datetime.date()\n\n    params = {\n        'SettlementDate': settlement_date.strftime('%Y-%m-%d'),\n        'Period': '*',\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('B1620', session, params)\n    return response.text\n\n\ndef parse_exchange(zone_key1, zone_key2, csv_text, target_datetime=None,\n                   logger=logging.getLogger(__name__)):\n    if not csv_text:\n        return None\n\n    report = REPORT_META['INTERFUELHH']\n\n    sorted_zone_keys = sorted([zone_key1, zone_key2])\n    exchange = '->'.join(sorted_zone_keys)\n    data_points = list()\n    lines = csv_text.split('\\n')\n\n    # check field count in report is as expected\n    field_count = len(lines[1].split(','))\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in INTERFUELHH report, got {}'.format(\n                report['expected_fields'], field_count))\n\n    for line in lines[1:-1]:\n        fields = line.split(',')\n\n        # settlement date / period combinations are always local time\n        date = dt.datetime.strptime(fields[1], '%Y%m%d').date()\n        settlement_period = int(fields[2])\n        datetime = datetime_from_date_sp(date, settlement_period)\n\n        data = {\n            'sortedZoneKeys': exchange,\n            'datetime': datetime,\n            'source': 'bmreports.com'\n        }\n\n        # positive value implies import to GB\n        multiplier = -1 if 'GB' in sorted_zone_keys[0] else 1\n        data['netFlow'] = float(fields[EXCHANGES[exchange]]) * multiplier\n        data_points.append(data)\n\n    return data_points\n\n\ndef parse_production(csv_text, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    if not csv_text:\n        return None\n\n    report = REPORT_META['B1620']\n\n    # create DataFrame from slice of CSV rows\n    df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'] - 1)\n\n    # check field count in report is as expected\n    field_count = len(df.columns)\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in B1620 report, got {}'.format(\n                report['expected_fields'], len(df.columns)))\n\n    # filter out undesired columns\n    df = df.iloc[:-1, [7, 8, 9, 4]]\n\n    df['Settlement Date'] = df['Settlement Date'].apply(\n        lambda x: dt.datetime.strptime(x, '%Y-%m-%d'))\n    df['Settlement Period'] = df['Settlement Period'].astype(int)\n    df['datetime'] = df.apply(lambda x: datetime_from_date_sp(\n        x['Settlement Date'], x['Settlement Period']), axis=1)\n\n    # map from report fuel names to electricitymap fuel names\n    fuel_column = 'Power System Resource  Type'\n    df[fuel_column] = df[fuel_column].apply(lambda x: RESOURCE_TYPE_TO_FUEL[x])\n\n    # loop through unique datetimes and create each data point\n    data_points = list()\n    for time in pd.unique(df['datetime']):\n        time_df = df[df['datetime'] == time]\n\n        data_point = {\n            'zoneKey': 'GB',\n            'datetime': time.to_pydatetime(),\n            'source': 'bmreports.com',\n            'production': dict(),\n            'storage': dict()\n        }\n\n        for row in time_df.iterrows():\n            fields = row[1].to_dict()\n            fuel = fields[fuel_column]\n            quantity = fields['Quantity']\n\n            # check if storage value and if so correct key\n            if 'storage' in fuel:\n                fuel_key = fuel.replace('storage', '').strip()\n                # ELEXON storage is negative when storing and positive when\n                # discharging (the opposite to electricitymap)\n                data_point['storage'][fuel_key] = quantity * -1\n            else:\n                # if/else structure allows summation of multiple quantities\n                # e.g. 'Wind Onshore' and 'Wind Offshore' both have the\n                # key 'wind' here.\n                if fuel in data_point['production'].keys():\n                    data_point['production'][fuel] += quantity\n                else:\n                    data_point['production'][fuel] = quantity\n\n        data_points.append(data_point)\n\n    return data_points\n\n\ndef datetime_from_date_sp(date, sp):\n    datetime = arrow.get(date).shift(minutes=30 * (sp - 1))\n    return datetime.replace(tzinfo='Europe/London').datetime\n\n\ndef _fetch_wind(target_datetime=None):\n    if target_datetime is None:\n        target_datetime = dt.datetime.now()\n\n    # line up with B1620 (main production report) search range\n    d = target_datetime.date()\n    start = d - dt.timedelta(hours=24)\n    end = dt.datetime.combine(d + dt.timedelta(days=1), dt.time(0))\n\n    session = requests.session()\n    params = {\n        'FromDateTime': start.strftime('%Y-%m-%d %H:%M:%S'),\n        'ToDateTime': end.strftime('%Y-%m-%d %H:%M:%S'),\n        'ServiceType': 'csv'\n    }\n    response = query_ELEXON('FUELINST', session, params)\n    csv_text = response.text\n\n    report = REPORT_META['FUELINST']\n    df = pd.read_csv(StringIO(csv_text), skiprows=report['skiprows'],\n                     skipfooter=1, header=None)\n\n    field_count = len(df.columns)\n    if field_count != report['expected_fields']:\n        raise ValueError(\n            'Expected {} fields in FUELINST report, got {}'.format(\n                report['expected_fields'], len(df.columns)))\n\n    df = df.iloc[:, [1, 2, 3, 8]]\n    df.columns = ['Settlement Date', 'Settlement Period', 'published', 'Wind']\n    df['Settlement Date'] = df['Settlement Date'].apply(\n        lambda x: dt.datetime.strptime(str(x), '%Y%m%d'))\n    df['Settlement Period'] = df['Settlement Period'].astype(int)\n    df['datetime'] = df.apply(lambda x: datetime_from_date_sp(\n        x['Settlement Date'], x['Settlement Period']), axis=1)\n\n    df['published'] = df['published'].apply(\n        lambda x: dt.datetime.strptime(str(x), '%Y%m%d%H%M%S'))\n    # get the most recently published value for each datetime\n    idx = df.groupby('datetime')['published'].transform(max) == df['published']\n    df = df[idx]\n\n    return df[['datetime', 'Wind']]\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None,\n                   logger=logging.getLogger(__name__)):\n    session = session or requests.session()\n    response = query_exchange(session, target_datetime)\n    data = parse_exchange(zone_key1, zone_key2, response, target_datetime,\n                          logger)\n    return data\n\n\ndef fetch_production(zone_key='GB', session=None, target_datetime=None,\n                     logger=logging.getLogger(__name__)):\n    session = session or requests.session()\n    response = query_production(session, target_datetime)\n    data = parse_production(response, target_datetime, logger)\n\n    # At times B1620 has had poor quality data for wind so fetch from FUELINST\n    if FETCH_WIND_FROM_FUELINST:\n        wind = _fetch_wind(target_datetime)\n        for entry in data:\n            datetime = entry['datetime']\n            wind_row = wind[wind['datetime'] == datetime]\n            if len(wind_row):\n                entry['production']['wind'] = wind_row.iloc[0]['Wind']\n            else:\n                entry['production']['wind'] = None\n\n    required = ['coal', 'gas', 'nuclear']\n    expected_range = {\n        'coal': (0, 10000),\n        'gas': (100, 30000),\n        'nuclear': (100, 20000)\n    }\n    data = [x for x in data\n            if validate(\n                x, logger, required=required, expected_range=expected_range)]\n\n    return data\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy \n    for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n\n    print('fetch_exchange(FR, GB) ->')\n    print(fetch_exchange('FR', 'GB'))\n\n    print('fetch_exchange(GB, IE) ->')\n    print(fetch_exchange('GB', 'IE'))\n\n    print('fetch_exchange(GB, NL) ->')\n    print(fetch_exchange('GB', 'NL'))\n", "path": "parsers/ELEXON.py"}]}