We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
WARNING - thermal_production: PALENQUE is missing from the DO plant mapping


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/DO.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 import arrow
4 from bs4 import BeautifulSoup
5 from collections import defaultdict
6 from math import isnan
7 import numpy as np
8 from operator import itemgetter
9 import pandas as pd
10 import requests
11 
12 try:
13     unicode         # Python 2
14 except NameError:
15     unicode = str   # Python 3
16 
17 try:
18     xrange          # Python 2
19 except NameError:
20     xrange = range  # Python 3
21 
22 
23 # This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.
24 # The data is in MWh but since it is updated hourly we can view it as MW.
25 # Solar generation has no data available currently but multiple projects are planned/under construction.
26 
27 url = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'
28 
29 total_mapping = {
30                 u'Total T\xe9rmico': 'Thermal',
31                 u'Total E\xf3lico': 'Wind',
32                 u'Total Hidroel\xe9ctrica': 'Hydro',
33                 'Total Generado': 'Generated'
34                 }
35 
36 # Power plant types
37 # http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/
38 # Reporte_diario_de_generacion_31_enero_2017_merged2.pdf
39 
40 thermal_plants = {
41                  u'AES ANDRES': 'gas',
42                  u'BARAHONA CARBON': 'coal',
43                  u'BERSAL': 'unknown',
44                  u'CEPP 1': 'oil',
45                  u'CEPP 2': 'oil',
46                  u'CESPM 1': 'oil',
47                  u'CESPM 2': 'oil',
48                  u'CESPM 3': 'oil',
49                  u'ESTRELLA DEL MAR 2 CFO': 'oil',
50                  u'ESTRELLA DEL MAR 2 CGN': 'gas',
51                  u'ESTRELLA DEL MAR 2 SFO': 'oil',
52                  u'ESTRELLA DEL MAR 2 SGN': 'gas',
53                  u'HAINA TG': 'oil',
54                  u'INCA KM22': 'oil',
55                  u'ITABO 1': 'coal',
56                  u'ITABO 2': 'coal',
57                  u'LA VEGA': 'oil',
58                  u'LOS MINA 5': 'gas',
59                  u'LOS MINA 6': 'gas',
60                  u'LOS MINA 7': 'gas',
61                  u'LOS OR\xcdGENES POWER PLANT FUEL OIL': 'oil',
62                  u'LOS OR\xcdGENES POWER PLANT GAS NATURAL': 'gas',
63                  u'METALDOM': 'oil',
64                  u'MONTE PLATA SOLAR': 'solar',
65                  u'MONTE RIO': 'oil',
66                  u'PALAMARA': 'oil',
67                  u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',
68                  u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',
69                  u'PIMENTEL 1': 'oil',
70                  u'PIMENTEL 2': 'oil',
71                  u'PIMENTEL 3': 'oil',
72                  u'QUISQUEYA 1': 'gas',
73                  u'QUISQUEYA 2': 'gas',
74                  u'RIO SAN JUAN': 'oil',
75                  u'SAN FELIPE': 'oil',
76                  u'SAN FELIPE CC': 'gas',
77                  u'SAN FELIPE VAP': 'oil',
78                  u'SAN LORENZO 1': 'gas',
79                  u'SAN PEDRO BIO-ENERGY': 'biomass',
80                  u'SAN PEDRO VAPOR': 'oil',
81                  u'SULTANA DEL ESTE': 'oil'
82                  }
83 
84 
85 def get_data(session=None):
86     """
87     Makes a request to source url.
88     Finds main table and creates a list of all table elements in unicode string format.
89     Returns a list.
90     """
91 
92     data = []
93     s = session or requests.Session()
94     data_req = s.get(url)
95     soup = BeautifulSoup(data_req.content, 'lxml')
96 
97     tbs = soup.find("table", id="PostdespachoUnidadesTermicasGrid_DXMainTable")
98     rows = tbs.find_all("td")
99 
100     for row in rows:
101         num = row.getText().strip()
102         data.append(unicode(num))
103 
104     return data
105 
106 
107 def floater(item):
108     """
109     Attempts to convert any item given to a float.  Returns item if it fails.
110     """
111 
112     try:
113         return float(item)
114     except ValueError:
115         return item
116 
117 
118 def chunker(big_lst):
119     """
120     Breaks a big list into a list of lists.  Removes any list with no data then turns remaining
121     lists into key: value pairs with first element from the list being the key.
122     Returns a dictionary.
123     """
124 
125     chunks = [big_lst[x:x + 27] for x in xrange(0, len(big_lst), 27)]
126 
127     # Remove the list if it contains no data.
128     for chunk in chunks:
129         if any(chunk):
130             continue
131         else:
132             chunks.remove(chunk)
133 
134     chunked_list = {words[0]: words[1:] for words in chunks}
135 
136     return chunked_list
137 
138 
139 def data_formatter(data):
140     """
141     Takes data and finds relevant sections.  Formats and breaks data into usable parts.
142     Returns a nested dictionary.
143     """
144 
145     find_thermal_index = data.index(u'GRUPO: T\xe9rmica')
146     find_totals_index = data.index(u'Total T\xe9rmico')
147     find_totals_end = data.index(u'Total Programado')
148 
149     ufthermal = data[find_thermal_index + 3:find_totals_index - 59]
150     total_data = data[find_totals_index:find_totals_end]
151 
152     # Remove all company names.
153     for val in ufthermal:
154         if ':' in val:
155             i = ufthermal.index(val)
156             del ufthermal[i:i + 3]
157 
158     formatted_thermal = chunker([floater(item) for item in ufthermal])
159     mapped_totals = [total_mapping.get(x, x) for x in total_data]
160     formatted_totals = chunker([floater(item) for item in mapped_totals])
161 
162     return {'totals': formatted_totals, 'thermal': formatted_thermal}
163 
164 
165 def data_parser(formatted_data):
166     """
167     Converts formatted data into a pandas dataframe.  Removes any empty rows.
168     Returns a DataFrame.
169     """
170 
171     hours = list(range(1, 24)) + [0] + [25, 26]
172     dft = pd.DataFrame(formatted_data, index=hours)
173 
174     dft = dft.drop(dft.index[[-1, -2]])
175     dft = dft.replace(u'', np.nan)
176     dft = dft.dropna(how='all')
177 
178     return dft
179 
180 
181 def thermal_production(df, logger):
182     """
183     Takes DataFrame and finds thermal generation for each hour.
184     Removes any non generating plants then maps plants to type.
185     Sums type instances and returns a dictionary.
186     """
187 
188     therms = []
189     unmapped = set()
190     for hour in df.index.values:
191         dt = hour
192         currentt = df.loc[[hour]]
193 
194         # Create current plant output.
195         tp = {}
196         for item in list(df):
197             v = currentt.iloc[0][item]
198             tp[item] = v
199 
200         current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}
201 
202         for plant in current_plants.keys():
203             if plant not in thermal_plants.keys():
204                 unmapped.add(plant)
205 
206         mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.items()]
207 
208         thermalDict = defaultdict(lambda: 0.0)
209 
210         # Sum values for duplicate keys.
211         for key, val in mapped_plants:
212             thermalDict[key] += val
213 
214         thermalDict['datetime'] = dt
215         thermalDict = dict(thermalDict)
216         therms.append(thermalDict)
217 
218     for plant in unmapped:
219         logger.warning(
220             '{} is missing from the DO plant mapping!'.format(plant),
221             extra={'key': 'DO'})
222 
223     return therms
224 
225 
226 def total_production(df):
227     """
228     Takes DataFrame and finds generation totals for each hour.
229     Returns a dictionary.
230     """
231 
232     vals = []
233     # The Dominican Republic does not observe daylight savings time.
234     for hour in df.index.values:
235         dt = hour
236         current = df.loc[[hour]]
237         hydro = current.iloc[0]['Hydro']
238         wind = current.iloc[0]['Wind']
239         if wind > -10:
240             wind = max(wind, 0)
241 
242         # Wind and hydro totals do not always update exactly on the new hour.
243         # In this case we set them to None because they are unknown rather than zero.
244         if isnan(wind):
245             wind = None
246         if isnan(hydro):
247             hydro = None
248 
249         prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}
250         vals.append(prod)
251 
252     return vals
253 
254 
255 def merge_production(thermal, total):
256     """
257     Takes thermal generation and total generation and merges them using 'datetime' key.
258     Returns a defaultdict.
259     """
260 
261     d = defaultdict(dict)
262     for each in (thermal, total):
263         for elem in each:
264             d[elem['datetime']].update(elem)
265 
266     final = sorted(d.values(), key=itemgetter("datetime"))
267 
268     def get_datetime(hour):
269         at = arrow.now('America/Dominica').floor('day')
270         dt = (at.shift(hours=int(hour) - 1)).datetime
271         return dt
272 
273     for item in final:
274         i = item['datetime']
275         j = get_datetime(i)
276         item['datetime'] = j
277 
278     return final
279 
280 
281 def fetch_production(zone_key='DO', session=None, target_datetime=None, logger=None):
282     """
283     Requests the last known production mix (in MW) of a given country
284     Arguments:
285     zone_key (optional) -- used in case a parser is able to fetch multiple countries
286     Return:
287     A dictionary in the form:
288     {
289       'zoneKey': 'FR',
290       'datetime': '2017-01-01T00:00:00Z',
291       'production': {
292           'biomass': 0.0,
293           'coal': 0.0,
294           'gas': 0.0,
295           'hydro': 0.0,
296           'nuclear': null,
297           'oil': 0.0,
298           'solar': 0.0,
299           'wind': 0.0,
300           'geothermal': 0.0,
301           'unknown': 0.0
302       },
303       'storage': {
304           'hydro': -10.0,
305       },
306       'source': 'mysource.com'
307     }
308     """
309     if target_datetime:
310         raise NotImplementedError('This parser is not yet able to parse past dates')
311 
312     dat = data_formatter(get_data(session=None))
313     tot = data_parser(dat['totals'])
314     th = data_parser(dat['thermal'])
315     thermal = thermal_production(th, logger)
316     total = total_production(tot)
317     merge = merge_production(thermal, total)
318 
319     production_mix_by_hour = []
320     for hour in merge:
321         production_mix = {
322           'zoneKey': zone_key,
323           'datetime': hour['datetime'],
324           'production': {
325               'biomass': hour.get('biomass', 0.0),
326               'coal': hour.get('coal', 0.0),
327               'gas': hour.get('gas', 0.0),
328               'hydro': hour.get('hydro', 0.0),
329               'nuclear': 0.0,
330               'oil': hour.get('oil', 0.0),
331               'solar': None,
332               'wind': hour.get('wind', 0.0),
333               'geothermal': 0.0,
334               'unknown': hour.get('unknown', 0.0)
335           },
336           'storage': {
337               'hydro': None,
338           },
339           'source': 'oc.org.do'
340         }
341         production_mix_by_hour.append(production_mix)
342 
343     return production_mix_by_hour
344 
345 
346 if __name__ == '__main__':
347     """Main method, never used by the Electricity Map backend, but handy for testing."""
348 
349     print('fetch_production() ->')
350     print(fetch_production())
351
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/DO.py b/parsers/DO.py
--- a/parsers/DO.py
+++ b/parsers/DO.py
@@ -64,6 +64,7 @@
                  u'MONTE PLATA SOLAR': 'solar',
                  u'MONTE RIO': 'oil',
                  u'PALAMARA': 'oil',
+                 u'PALENQUE': 'oil',
                  u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',
                  u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',
                  u'PIMENTEL 1': 'oil',

{"golden_diff": "diff --git a/parsers/DO.py b/parsers/DO.py\n--- a/parsers/DO.py\n+++ b/parsers/DO.py\n@@ -64,6 +64,7 @@\n                  u'MONTE PLATA SOLAR': 'solar',\n                  u'MONTE RIO': 'oil',\n                  u'PALAMARA': 'oil',\n+                 u'PALENQUE': 'oil',\n                  u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',\n                  u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',\n                  u'PIMENTEL 1': 'oil',\n", "issue": "WARNING - thermal_production: PALENQUE is missing from the DO plant mapping\n\n", "before_files": [{"content": "#!/usr/bin/env python3\n\nimport arrow\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\nfrom math import isnan\nimport numpy as np\nfrom operator import itemgetter\nimport pandas as pd\nimport requests\n\ntry:\n    unicode         # Python 2\nexcept NameError:\n    unicode = str   # Python 3\n\ntry:\n    xrange          # Python 2\nexcept NameError:\n    xrange = range  # Python 3\n\n\n# This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.\n# The data is in MWh but since it is updated hourly we can view it as MW.\n# Solar generation has no data available currently but multiple projects are planned/under construction.\n\nurl = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'\n\ntotal_mapping = {\n                u'Total T\\xe9rmico': 'Thermal',\n                u'Total E\\xf3lico': 'Wind',\n                u'Total Hidroel\\xe9ctrica': 'Hydro',\n                'Total Generado': 'Generated'\n                }\n\n# Power plant types\n# http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/\n# Reporte_diario_de_generacion_31_enero_2017_merged2.pdf\n\nthermal_plants = {\n                 u'AES ANDRES': 'gas',\n                 u'BARAHONA CARBON': 'coal',\n                 u'BERSAL': 'unknown',\n                 u'CEPP 1': 'oil',\n                 u'CEPP 2': 'oil',\n                 u'CESPM 1': 'oil',\n                 u'CESPM 2': 'oil',\n                 u'CESPM 3': 'oil',\n                 u'ESTRELLA DEL MAR 2 CFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 CGN': 'gas',\n                 u'ESTRELLA DEL MAR 2 SFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 SGN': 'gas',\n                 u'HAINA TG': 'oil',\n                 u'INCA KM22': 'oil',\n                 u'ITABO 1': 'coal',\n                 u'ITABO 2': 'coal',\n                 u'LA VEGA': 'oil',\n                 u'LOS MINA 5': 'gas',\n                 u'LOS MINA 6': 'gas',\n                 u'LOS MINA 7': 'gas',\n                 u'LOS OR\\xcdGENES POWER PLANT FUEL OIL': 'oil',\n                 u'LOS OR\\xcdGENES POWER PLANT GAS NATURAL': 'gas',\n                 u'METALDOM': 'oil',\n                 u'MONTE PLATA SOLAR': 'solar',\n                 u'MONTE RIO': 'oil',\n                 u'PALAMARA': 'oil',\n                 u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',\n                 u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',\n                 u'PIMENTEL 1': 'oil',\n                 u'PIMENTEL 2': 'oil',\n                 u'PIMENTEL 3': 'oil',\n                 u'QUISQUEYA 1': 'gas',\n                 u'QUISQUEYA 2': 'gas',\n                 u'RIO SAN JUAN': 'oil',\n                 u'SAN FELIPE': 'oil',\n                 u'SAN FELIPE CC': 'gas',\n                 u'SAN FELIPE VAP': 'oil',\n                 u'SAN LORENZO 1': 'gas',\n                 u'SAN PEDRO BIO-ENERGY': 'biomass',\n                 u'SAN PEDRO VAPOR': 'oil',\n                 u'SULTANA DEL ESTE': 'oil'\n                 }\n\n\ndef get_data(session=None):\n    \"\"\"\n    Makes a request to source url.\n    Finds main table and creates a list of all table elements in unicode string format.\n    Returns a list.\n    \"\"\"\n\n    data = []\n    s = session or requests.Session()\n    data_req = s.get(url)\n    soup = BeautifulSoup(data_req.content, 'lxml')\n\n    tbs = soup.find(\"table\", id=\"PostdespachoUnidadesTermicasGrid_DXMainTable\")\n    rows = tbs.find_all(\"td\")\n\n    for row in rows:\n        num = row.getText().strip()\n        data.append(unicode(num))\n\n    return data\n\n\ndef floater(item):\n    \"\"\"\n    Attempts to convert any item given to a float.  Returns item if it fails.\n    \"\"\"\n\n    try:\n        return float(item)\n    except ValueError:\n        return item\n\n\ndef chunker(big_lst):\n    \"\"\"\n    Breaks a big list into a list of lists.  Removes any list with no data then turns remaining\n    lists into key: value pairs with first element from the list being the key.\n    Returns a dictionary.\n    \"\"\"\n\n    chunks = [big_lst[x:x + 27] for x in xrange(0, len(big_lst), 27)]\n\n    # Remove the list if it contains no data.\n    for chunk in chunks:\n        if any(chunk):\n            continue\n        else:\n            chunks.remove(chunk)\n\n    chunked_list = {words[0]: words[1:] for words in chunks}\n\n    return chunked_list\n\n\ndef data_formatter(data):\n    \"\"\"\n    Takes data and finds relevant sections.  Formats and breaks data into usable parts.\n    Returns a nested dictionary.\n    \"\"\"\n\n    find_thermal_index = data.index(u'GRUPO: T\\xe9rmica')\n    find_totals_index = data.index(u'Total T\\xe9rmico')\n    find_totals_end = data.index(u'Total Programado')\n\n    ufthermal = data[find_thermal_index + 3:find_totals_index - 59]\n    total_data = data[find_totals_index:find_totals_end]\n\n    # Remove all company names.\n    for val in ufthermal:\n        if ':' in val:\n            i = ufthermal.index(val)\n            del ufthermal[i:i + 3]\n\n    formatted_thermal = chunker([floater(item) for item in ufthermal])\n    mapped_totals = [total_mapping.get(x, x) for x in total_data]\n    formatted_totals = chunker([floater(item) for item in mapped_totals])\n\n    return {'totals': formatted_totals, 'thermal': formatted_thermal}\n\n\ndef data_parser(formatted_data):\n    \"\"\"\n    Converts formatted data into a pandas dataframe.  Removes any empty rows.\n    Returns a DataFrame.\n    \"\"\"\n\n    hours = list(range(1, 24)) + [0] + [25, 26]\n    dft = pd.DataFrame(formatted_data, index=hours)\n\n    dft = dft.drop(dft.index[[-1, -2]])\n    dft = dft.replace(u'', np.nan)\n    dft = dft.dropna(how='all')\n\n    return dft\n\n\ndef thermal_production(df, logger):\n    \"\"\"\n    Takes DataFrame and finds thermal generation for each hour.\n    Removes any non generating plants then maps plants to type.\n    Sums type instances and returns a dictionary.\n    \"\"\"\n\n    therms = []\n    unmapped = set()\n    for hour in df.index.values:\n        dt = hour\n        currentt = df.loc[[hour]]\n\n        # Create current plant output.\n        tp = {}\n        for item in list(df):\n            v = currentt.iloc[0][item]\n            tp[item] = v\n\n        current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}\n\n        for plant in current_plants.keys():\n            if plant not in thermal_plants.keys():\n                unmapped.add(plant)\n\n        mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.items()]\n\n        thermalDict = defaultdict(lambda: 0.0)\n\n        # Sum values for duplicate keys.\n        for key, val in mapped_plants:\n            thermalDict[key] += val\n\n        thermalDict['datetime'] = dt\n        thermalDict = dict(thermalDict)\n        therms.append(thermalDict)\n\n    for plant in unmapped:\n        logger.warning(\n            '{} is missing from the DO plant mapping!'.format(plant),\n            extra={'key': 'DO'})\n\n    return therms\n\n\ndef total_production(df):\n    \"\"\"\n    Takes DataFrame and finds generation totals for each hour.\n    Returns a dictionary.\n    \"\"\"\n\n    vals = []\n    # The Dominican Republic does not observe daylight savings time.\n    for hour in df.index.values:\n        dt = hour\n        current = df.loc[[hour]]\n        hydro = current.iloc[0]['Hydro']\n        wind = current.iloc[0]['Wind']\n        if wind > -10:\n            wind = max(wind, 0)\n\n        # Wind and hydro totals do not always update exactly on the new hour.\n        # In this case we set them to None because they are unknown rather than zero.\n        if isnan(wind):\n            wind = None\n        if isnan(hydro):\n            hydro = None\n\n        prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}\n        vals.append(prod)\n\n    return vals\n\n\ndef merge_production(thermal, total):\n    \"\"\"\n    Takes thermal generation and total generation and merges them using 'datetime' key.\n    Returns a defaultdict.\n    \"\"\"\n\n    d = defaultdict(dict)\n    for each in (thermal, total):\n        for elem in each:\n            d[elem['datetime']].update(elem)\n\n    final = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    def get_datetime(hour):\n        at = arrow.now('America/Dominica').floor('day')\n        dt = (at.shift(hours=int(hour) - 1)).datetime\n        return dt\n\n    for item in final:\n        i = item['datetime']\n        j = get_datetime(i)\n        item['datetime'] = j\n\n    return final\n\n\ndef fetch_production(zone_key='DO', session=None, target_datetime=None, logger=None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n    Arguments:\n    zone_key (optional) -- used in case a parser is able to fetch multiple countries\n    Return:\n    A dictionary in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    dat = data_formatter(get_data(session=None))\n    tot = data_parser(dat['totals'])\n    th = data_parser(dat['thermal'])\n    thermal = thermal_production(th, logger)\n    total = total_production(tot)\n    merge = merge_production(thermal, total)\n\n    production_mix_by_hour = []\n    for hour in merge:\n        production_mix = {\n          'zoneKey': zone_key,\n          'datetime': hour['datetime'],\n          'production': {\n              'biomass': hour.get('biomass', 0.0),\n              'coal': hour.get('coal', 0.0),\n              'gas': hour.get('gas', 0.0),\n              'hydro': hour.get('hydro', 0.0),\n              'nuclear': 0.0,\n              'oil': hour.get('oil', 0.0),\n              'solar': None,\n              'wind': hour.get('wind', 0.0),\n              'geothermal': 0.0,\n              'unknown': hour.get('unknown', 0.0)\n          },\n          'storage': {\n              'hydro': None,\n          },\n          'source': 'oc.org.do'\n        }\n        production_mix_by_hour.append(production_mix)\n\n    return production_mix_by_hour\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n", "path": "parsers/DO.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\nimport arrow\nfrom bs4 import BeautifulSoup\nfrom collections import defaultdict\nfrom math import isnan\nimport numpy as np\nfrom operator import itemgetter\nimport pandas as pd\nimport requests\n\ntry:\n    unicode         # Python 2\nexcept NameError:\n    unicode = str   # Python 3\n\ntry:\n    xrange          # Python 2\nexcept NameError:\n    xrange = range  # Python 3\n\n\n# This parser gets hourly electricity generation data from oc.org.do for the Dominican Republic.\n# The data is in MWh but since it is updated hourly we can view it as MW.\n# Solar generation has no data available currently but multiple projects are planned/under construction.\n\nurl = 'http://184.168.74.190:81/ReportesGraficos/ReportePostdespacho.aspx'\n\ntotal_mapping = {\n                u'Total T\\xe9rmico': 'Thermal',\n                u'Total E\\xf3lico': 'Wind',\n                u'Total Hidroel\\xe9ctrica': 'Hydro',\n                'Total Generado': 'Generated'\n                }\n\n# Power plant types\n# http://www.sie.gob.do/images/Estadisticas/MEM/GeneracionDiariaEnero2017/\n# Reporte_diario_de_generacion_31_enero_2017_merged2.pdf\n\nthermal_plants = {\n                 u'AES ANDRES': 'gas',\n                 u'BARAHONA CARBON': 'coal',\n                 u'BERSAL': 'unknown',\n                 u'CEPP 1': 'oil',\n                 u'CEPP 2': 'oil',\n                 u'CESPM 1': 'oil',\n                 u'CESPM 2': 'oil',\n                 u'CESPM 3': 'oil',\n                 u'ESTRELLA DEL MAR 2 CFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 CGN': 'gas',\n                 u'ESTRELLA DEL MAR 2 SFO': 'oil',\n                 u'ESTRELLA DEL MAR 2 SGN': 'gas',\n                 u'HAINA TG': 'oil',\n                 u'INCA KM22': 'oil',\n                 u'ITABO 1': 'coal',\n                 u'ITABO 2': 'coal',\n                 u'LA VEGA': 'oil',\n                 u'LOS MINA 5': 'gas',\n                 u'LOS MINA 6': 'gas',\n                 u'LOS MINA 7': 'gas',\n                 u'LOS OR\\xcdGENES POWER PLANT FUEL OIL': 'oil',\n                 u'LOS OR\\xcdGENES POWER PLANT GAS NATURAL': 'gas',\n                 u'METALDOM': 'oil',\n                 u'MONTE PLATA SOLAR': 'solar',\n                 u'MONTE RIO': 'oil',\n                 u'PALAMARA': 'oil',\n                 u'PALENQUE': 'oil',\n                 u'PARQUE ENERGETICO LOS MINA CC PARCIAL': 'gas',\n                 u'PARQUE ENERGETICO LOS MINA CC TOTAL': 'gas',\n                 u'PIMENTEL 1': 'oil',\n                 u'PIMENTEL 2': 'oil',\n                 u'PIMENTEL 3': 'oil',\n                 u'QUISQUEYA 1': 'gas',\n                 u'QUISQUEYA 2': 'gas',\n                 u'RIO SAN JUAN': 'oil',\n                 u'SAN FELIPE': 'oil',\n                 u'SAN FELIPE CC': 'gas',\n                 u'SAN FELIPE VAP': 'oil',\n                 u'SAN LORENZO 1': 'gas',\n                 u'SAN PEDRO BIO-ENERGY': 'biomass',\n                 u'SAN PEDRO VAPOR': 'oil',\n                 u'SULTANA DEL ESTE': 'oil'\n                 }\n\n\ndef get_data(session=None):\n    \"\"\"\n    Makes a request to source url.\n    Finds main table and creates a list of all table elements in unicode string format.\n    Returns a list.\n    \"\"\"\n\n    data = []\n    s = session or requests.Session()\n    data_req = s.get(url)\n    soup = BeautifulSoup(data_req.content, 'lxml')\n\n    tbs = soup.find(\"table\", id=\"PostdespachoUnidadesTermicasGrid_DXMainTable\")\n    rows = tbs.find_all(\"td\")\n\n    for row in rows:\n        num = row.getText().strip()\n        data.append(unicode(num))\n\n    return data\n\n\ndef floater(item):\n    \"\"\"\n    Attempts to convert any item given to a float.  Returns item if it fails.\n    \"\"\"\n\n    try:\n        return float(item)\n    except ValueError:\n        return item\n\n\ndef chunker(big_lst):\n    \"\"\"\n    Breaks a big list into a list of lists.  Removes any list with no data then turns remaining\n    lists into key: value pairs with first element from the list being the key.\n    Returns a dictionary.\n    \"\"\"\n\n    chunks = [big_lst[x:x + 27] for x in xrange(0, len(big_lst), 27)]\n\n    # Remove the list if it contains no data.\n    for chunk in chunks:\n        if any(chunk):\n            continue\n        else:\n            chunks.remove(chunk)\n\n    chunked_list = {words[0]: words[1:] for words in chunks}\n\n    return chunked_list\n\n\ndef data_formatter(data):\n    \"\"\"\n    Takes data and finds relevant sections.  Formats and breaks data into usable parts.\n    Returns a nested dictionary.\n    \"\"\"\n\n    find_thermal_index = data.index(u'GRUPO: T\\xe9rmica')\n    find_totals_index = data.index(u'Total T\\xe9rmico')\n    find_totals_end = data.index(u'Total Programado')\n\n    ufthermal = data[find_thermal_index + 3:find_totals_index - 59]\n    total_data = data[find_totals_index:find_totals_end]\n\n    # Remove all company names.\n    for val in ufthermal:\n        if ':' in val:\n            i = ufthermal.index(val)\n            del ufthermal[i:i + 3]\n\n    formatted_thermal = chunker([floater(item) for item in ufthermal])\n    mapped_totals = [total_mapping.get(x, x) for x in total_data]\n    formatted_totals = chunker([floater(item) for item in mapped_totals])\n\n    return {'totals': formatted_totals, 'thermal': formatted_thermal}\n\n\ndef data_parser(formatted_data):\n    \"\"\"\n    Converts formatted data into a pandas dataframe.  Removes any empty rows.\n    Returns a DataFrame.\n    \"\"\"\n\n    hours = list(range(1, 24)) + [0] + [25, 26]\n    dft = pd.DataFrame(formatted_data, index=hours)\n\n    dft = dft.drop(dft.index[[-1, -2]])\n    dft = dft.replace(u'', np.nan)\n    dft = dft.dropna(how='all')\n\n    return dft\n\n\ndef thermal_production(df, logger):\n    \"\"\"\n    Takes DataFrame and finds thermal generation for each hour.\n    Removes any non generating plants then maps plants to type.\n    Sums type instances and returns a dictionary.\n    \"\"\"\n\n    therms = []\n    unmapped = set()\n    for hour in df.index.values:\n        dt = hour\n        currentt = df.loc[[hour]]\n\n        # Create current plant output.\n        tp = {}\n        for item in list(df):\n            v = currentt.iloc[0][item]\n            tp[item] = v\n\n        current_plants = {k: tp[k] for k in tp if not isnan(tp[k])}\n\n        for plant in current_plants.keys():\n            if plant not in thermal_plants.keys():\n                unmapped.add(plant)\n\n        mapped_plants = [(thermal_plants.get(plant, 'unknown'), val) for plant, val in current_plants.items()]\n\n        thermalDict = defaultdict(lambda: 0.0)\n\n        # Sum values for duplicate keys.\n        for key, val in mapped_plants:\n            thermalDict[key] += val\n\n        thermalDict['datetime'] = dt\n        thermalDict = dict(thermalDict)\n        therms.append(thermalDict)\n\n    for plant in unmapped:\n        logger.warning(\n            '{} is missing from the DO plant mapping!'.format(plant),\n            extra={'key': 'DO'})\n\n    return therms\n\n\ndef total_production(df):\n    \"\"\"\n    Takes DataFrame and finds generation totals for each hour.\n    Returns a dictionary.\n    \"\"\"\n\n    vals = []\n    # The Dominican Republic does not observe daylight savings time.\n    for hour in df.index.values:\n        dt = hour\n        current = df.loc[[hour]]\n        hydro = current.iloc[0]['Hydro']\n        wind = current.iloc[0]['Wind']\n        if wind > -10:\n            wind = max(wind, 0)\n\n        # Wind and hydro totals do not always update exactly on the new hour.\n        # In this case we set them to None because they are unknown rather than zero.\n        if isnan(wind):\n            wind = None\n        if isnan(hydro):\n            hydro = None\n\n        prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}\n        vals.append(prod)\n\n    return vals\n\n\ndef merge_production(thermal, total):\n    \"\"\"\n    Takes thermal generation and total generation and merges them using 'datetime' key.\n    Returns a defaultdict.\n    \"\"\"\n\n    d = defaultdict(dict)\n    for each in (thermal, total):\n        for elem in each:\n            d[elem['datetime']].update(elem)\n\n    final = sorted(d.values(), key=itemgetter(\"datetime\"))\n\n    def get_datetime(hour):\n        at = arrow.now('America/Dominica').floor('day')\n        dt = (at.shift(hours=int(hour) - 1)).datetime\n        return dt\n\n    for item in final:\n        i = item['datetime']\n        j = get_datetime(i)\n        item['datetime'] = j\n\n    return final\n\n\ndef fetch_production(zone_key='DO', session=None, target_datetime=None, logger=None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given country\n    Arguments:\n    zone_key (optional) -- used in case a parser is able to fetch multiple countries\n    Return:\n    A dictionary in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        raise NotImplementedError('This parser is not yet able to parse past dates')\n\n    dat = data_formatter(get_data(session=None))\n    tot = data_parser(dat['totals'])\n    th = data_parser(dat['thermal'])\n    thermal = thermal_production(th, logger)\n    total = total_production(tot)\n    merge = merge_production(thermal, total)\n\n    production_mix_by_hour = []\n    for hour in merge:\n        production_mix = {\n          'zoneKey': zone_key,\n          'datetime': hour['datetime'],\n          'production': {\n              'biomass': hour.get('biomass', 0.0),\n              'coal': hour.get('coal', 0.0),\n              'gas': hour.get('gas', 0.0),\n              'hydro': hour.get('hydro', 0.0),\n              'nuclear': 0.0,\n              'oil': hour.get('oil', 0.0),\n              'solar': None,\n              'wind': hour.get('wind', 0.0),\n              'geothermal': 0.0,\n              'unknown': hour.get('unknown', 0.0)\n          },\n          'storage': {\n              'hydro': None,\n          },\n          'source': 'oc.org.do'\n        }\n        production_mix_by_hour.append(production_mix)\n\n    return production_mix_by_hour\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    print('fetch_production() ->')\n    print(fetch_production())\n", "path": "parsers/DO.py"}]}