We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
United States NY ISO calculation on Dual Fuel is probably wrong
Just found your site today, really cool. I'd actually done a tech demo with streaming data from the nyiso as part of an MQTT / Kubernetes talk. Demo site at http://ny-power.org.

The Dual Fuel column is getting mapped to pretty dirty fossil fuel systems, which I think is probably inaccurate. My understanding of the dual fuel plants is they mostly burn Natural Gas, but can burn Oil instead when Natural Gas is constrained and targeted for heating (typically during peak heating demand in the winter). I looked up the total Oil burned for electricity generation in 2016 (the last year that numbers are available), and it was actually really low. So when I was simulating it I gave it a kind of 30% Oil / 70% NG number as an approximation. It's probably as good a guess as one can get. But the worse than coal numbers are I think pretty far off.

Is there a way in the system to provide a custom value, or is it just mapping back to the IPCC averaged numbers? 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsers/US_NY.py`
Content:
```
1 #!/usr/bin/env python3
2 
3 """Real time parser for the state of New York."""
4 from collections import defaultdict
5 from datetime import timedelta
6 from operator import itemgetter
7 from urllib.error import HTTPError
8 
9 import arrow
10 import pandas as pd
11 
12 mapping = {
13     'Dual Fuel': 'unknown',
14     'Natural Gas': 'gas',
15     'Nuclear': 'nuclear',
16     'Other Fossil Fuels': 'unknown',
17     'Other Renewables': 'unknown',
18     'Wind': 'wind',
19     'Hydro': 'hydro'
20 }
21 
22 
23 def read_csv_data(url):
24     """
25     Gets csv data from a url and returns a dataframe.
26     """
27 
28     csv_data = pd.read_csv(url)
29 
30     return csv_data
31 
32 
33 def timestamp_converter(timestamp_string):
34     """
35     Converts timestamps in nyiso data into aware datetime objects.
36     """
37 
38     dt_naive = arrow.get(timestamp_string, 'MM/DD/YYYY HH:mm:ss')
39     dt_aware = dt_naive.replace(tzinfo='America/New_York').datetime
40 
41     return dt_aware
42 
43 
44 def data_parser(df):
45     """
46     Takes dataframe and loops over rows to form dictionaries consisting of datetime and generation type.
47     Merges these dictionaries using datetime key.
48     Maps to type and returns a list of tuples containing datetime string and production.
49     """
50 
51     chunks = []
52     for row in df.itertuples():
53         piece = {}
54         piece['datetime'] = row[1]
55         piece[row[3]] = row[4]
56         chunks.append(piece)
57 
58     # Join dicts on shared 'datetime' keys.
59     combine = defaultdict(dict)
60     for elem in chunks:
61         combine[elem['datetime']].update(elem)
62 
63     ordered = sorted(combine.values(), key=itemgetter("datetime"))
64 
65     mapped_generation = []
66     for item in ordered:
67         mapped_types = [(mapping.get(k, k), v) for k, v in item.items()]
68 
69         # Need to avoid multiple 'unknown' keys overwriting.
70         complete_production = defaultdict(lambda: 0.0)
71         for key, val in mapped_types:
72             try:
73                 complete_production[key] += val
74             except TypeError:
75                 # Datetime is a string at this point!
76                 complete_production[key] = val
77 
78         dt = complete_production.pop('datetime')
79         final = (dt, dict(complete_production))
80         mapped_generation.append(final)
81 
82     return mapped_generation
83 
84 
85 def fetch_production(zone_key='US-NY', session=None, target_datetime=None, logger=None):
86     """
87     Requests the last known production mix (in MW) of a given zone
88 
89     Arguments:
90     zone_key: used in case a parser is able to fetch multiple zones
91     session: requests session passed in order to re-use an existing session,
92       not used here due to difficulty providing it to pandas
93     target_datetime: the datetime for which we want production data. If not provided, we should
94       default it to now. The provided target_datetime is timezone-aware in UTC.
95     logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically
96 
97     Return:
98     A list of dictionaries in the form:
99     {
100       'zoneKey': 'FR',
101       'datetime': '2017-01-01T00:00:00Z',
102       'production': {
103           'biomass': 0.0,
104           'coal': 0.0,
105           'gas': 0.0,
106           'hydro': 0.0,
107           'nuclear': null,
108           'oil': 0.0,
109           'solar': 0.0,
110           'wind': 0.0,
111           'geothermal': 0.0,
112           'unknown': 0.0
113       },
114       'storage': {
115           'hydro': -10.0,
116       },
117       'source': 'mysource.com'
118     }
119     """
120     if target_datetime:
121         # ensure we have an arrow object
122         target_datetime = arrow.get(target_datetime)
123     else:
124         target_datetime = arrow.now('America/New_York')
125 
126     ny_date = target_datetime.format('YYYYMMDD')
127     mix_url = 'http://mis.nyiso.com/public/csv/rtfuelmix/{}rtfuelmix.csv'.format(ny_date)
128     try:
129         raw_data = read_csv_data(mix_url)
130     except HTTPError:
131         # this can happen when target_datetime has no data available
132         return None
133 
134     clean_data = data_parser(raw_data)
135 
136     production_mix = []
137     for datapoint in clean_data:
138         data = {
139             'zoneKey': zone_key,
140             'datetime': timestamp_converter(datapoint[0]),
141             'production': datapoint[1],
142             'storage': {'hydro': None},
143             'source': 'nyiso.com'
144         }
145 
146         production_mix.append(data)
147 
148     return production_mix
149 
150 
151 def fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):
152     """Requests the last known power exchange (in MW) between two zones
153 
154     Arguments:
155     zone_key1, zone_key2: specifies which exchange to get
156     session: requests session passed in order to re-use an existing session,
157       not used here due to difficulty providing it to pandas
158     target_datetime: the datetime for which we want production data. If not provided, we should
159       default it to now. The provided target_datetime is timezone-aware in UTC.
160     logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically
161 
162     Return:
163     A list of dictionaries in the form:
164     {
165       'sortedZoneKeys': 'DK->NO',
166       'datetime': '2017-01-01T00:00:00Z',
167       'netFlow': 0.0,
168       'source': 'mysource.com'
169     }
170     where net flow is from DK into NO
171     """
172     url = 'http://mis.nyiso.com/public/csv/ExternalLimitsFlows/{}ExternalLimitsFlows.csv'
173 
174     sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))
175 
176     # In the source CSV, positive is flow into NY, negative is flow out of NY.
177     # In Electricity Map, A->B means flow to B is positive.
178     if sorted_zone_keys == 'US-NEISO->US-NY':
179         direction = 1
180         relevant_exchanges = ['SCH - NE - NY', 'SCH - NPX_1385', 'SCH - NPX_CSC']
181     elif sorted_zone_keys == 'US-NY->US-PJM':
182         direction = -1
183         relevant_exchanges = ['SCH - PJ - NY', 'SCH - PJM_HTP', 'SCH - PJM_NEPTUNE', 'SCH - PJM_VFT']
184     elif sorted_zone_keys == 'CA-ON->US-NY':
185         direction = 1
186         relevant_exchanges = ['SCH - OH - NY']
187     elif sorted_zone_keys == 'CA-QC->US-NY':
188         direction = 1
189         relevant_exchanges = ['SCH - HQ_CEDARS', 'SCH - HQ - NY']
190     else:
191         raise NotImplementedError('Exchange pair not supported: {}'.format(sorted_zone_keys))
192 
193     if target_datetime:
194         # ensure we have an arrow object
195         target_datetime = arrow.get(target_datetime)
196     else:
197         target_datetime = arrow.now('America/New_York')
198     ny_date = target_datetime.format('YYYYMMDD')
199     exchange_url = url.format(ny_date)
200 
201     try:
202         exchange_data = read_csv_data(exchange_url)
203     except HTTPError:
204         # this can happen when target_datetime has no data available
205         return None
206 
207     new_england_exs = exchange_data.loc[exchange_data['Interface Name'].isin(relevant_exchanges)]
208     consolidated_flows = new_england_exs.reset_index().groupby("Timestamp").sum()
209 
210     now = arrow.utcnow()
211 
212     exchange_5min = []
213     for row in consolidated_flows.itertuples():
214         flow = float(row[3]) * direction
215         # Timestamp for exchange does not include seconds.
216         dt = timestamp_converter(row[0] + ':00')
217 
218         if (dt > now) and ((dt - now) < timedelta(seconds=300)):
219             # NYISO exchanges CSV (and only the exchanges CSV) includes data
220             # up to 5 minutes in the future (but only 5 minutes in the future).
221             # This also happens on their official website.
222             # Electricity Map raises error with data in the future, so skip
223             # that datapoint. If it's more than 5 minutes in the future,
224             # it's weird/unexpected and thus worthy of failure and logging.
225             continue
226 
227         exchange = {
228             'sortedZoneKeys': sorted_zone_keys,
229             'datetime': dt,
230             'netFlow': flow,
231             'source': 'nyiso.com'
232         }
233 
234         exchange_5min.append(exchange)
235 
236     return exchange_5min
237 
238 
239 if __name__ == '__main__':
240     """Main method, never used by the Electricity Map backend, but handy for testing."""
241 
242     from pprint import pprint
243     print('fetch_production() ->')
244     pprint(fetch_production())
245 
246     print('fetch_production(target_datetime=arrow.get("2018-03-13T12:00Z") ->')
247     pprint(fetch_production(target_datetime=arrow.get("2018-03-13T12:00Z")))
248 
249     print('fetch_production(target_datetime=arrow.get("2007-03-13T12:00Z") ->')
250     pprint(fetch_production(target_datetime=arrow.get("2007-03-13T12:00Z")))
251 
252     print('fetch_exchange(US-NY, US-NEISO)')
253     pprint(fetch_exchange('US-NY', 'US-NEISO'))
254 
255     print('fetch_exchange("US-NY", "CA-QC")')
256     pprint(fetch_exchange('US-NY', 'CA-QC'))
257 
258     print('fetch_exchange("US-NY", "CA-QC", target_datetime=arrow.get("2018-03-13T12:00Z"))')
259     pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get("2018-03-13T12:00Z")))
260 
261     print('fetch_exchange("US-NY", "CA-QC", target_datetime=arrow.get("2007-03-13T12:00Z")))')
262     pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get('2007-03-13T12:00Z')))
263
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsers/US_NY.py b/parsers/US_NY.py
--- a/parsers/US_NY.py
+++ b/parsers/US_NY.py
@@ -9,8 +9,15 @@
 import arrow
 import pandas as pd
 
+# Dual Fuel systems can run either Natural Gas or Oil, they represent
+# significantly more capacity in NY State than plants that can only
+# burn Natural Gas. When looking up fuel usage for NY in 2016 in
+# https://www.eia.gov/electricity/data/state/annual_generation_state.xls
+# 100 times more energy came from NG than Oil. That means Oil
+# consumption in the Dual Fuel systems is roughly ~1%, and to a first
+# approximation it's just Natural Gas.
 mapping = {
-    'Dual Fuel': 'unknown',
+    'Dual Fuel': 'gas',
     'Natural Gas': 'gas',
     'Nuclear': 'nuclear',
     'Other Fossil Fuels': 'unknown',

{"golden_diff": "diff --git a/parsers/US_NY.py b/parsers/US_NY.py\n--- a/parsers/US_NY.py\n+++ b/parsers/US_NY.py\n@@ -9,8 +9,15 @@\n import arrow\n import pandas as pd\n \n+# Dual Fuel systems can run either Natural Gas or Oil, they represent\n+# significantly more capacity in NY State than plants that can only\n+# burn Natural Gas. When looking up fuel usage for NY in 2016 in\n+# https://www.eia.gov/electricity/data/state/annual_generation_state.xls\n+# 100 times more energy came from NG than Oil. That means Oil\n+# consumption in the Dual Fuel systems is roughly ~1%, and to a first\n+# approximation it's just Natural Gas.\n mapping = {\n-    'Dual Fuel': 'unknown',\n+    'Dual Fuel': 'gas',\n     'Natural Gas': 'gas',\n     'Nuclear': 'nuclear',\n     'Other Fossil Fuels': 'unknown',\n", "issue": "United States NY ISO calculation on Dual Fuel is probably wrong\nJust found your site today, really cool. I'd actually done a tech demo with streaming data from the nyiso as part of an MQTT / Kubernetes talk. Demo site at http://ny-power.org.\r\n\r\nThe Dual Fuel column is getting mapped to pretty dirty fossil fuel systems, which I think is probably inaccurate. My understanding of the dual fuel plants is they mostly burn Natural Gas, but can burn Oil instead when Natural Gas is constrained and targeted for heating (typically during peak heating demand in the winter). I looked up the total Oil burned for electricity generation in 2016 (the last year that numbers are available), and it was actually really low. So when I was simulating it I gave it a kind of 30% Oil / 70% NG number as an approximation. It's probably as good a guess as one can get. But the worse than coal numbers are I think pretty far off.\r\n\r\nIs there a way in the system to provide a custom value, or is it just mapping back to the IPCC averaged numbers? \n", "before_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"Real time parser for the state of New York.\"\"\"\nfrom collections import defaultdict\nfrom datetime import timedelta\nfrom operator import itemgetter\nfrom urllib.error import HTTPError\n\nimport arrow\nimport pandas as pd\n\nmapping = {\n    'Dual Fuel': 'unknown',\n    'Natural Gas': 'gas',\n    'Nuclear': 'nuclear',\n    'Other Fossil Fuels': 'unknown',\n    'Other Renewables': 'unknown',\n    'Wind': 'wind',\n    'Hydro': 'hydro'\n}\n\n\ndef read_csv_data(url):\n    \"\"\"\n    Gets csv data from a url and returns a dataframe.\n    \"\"\"\n\n    csv_data = pd.read_csv(url)\n\n    return csv_data\n\n\ndef timestamp_converter(timestamp_string):\n    \"\"\"\n    Converts timestamps in nyiso data into aware datetime objects.\n    \"\"\"\n\n    dt_naive = arrow.get(timestamp_string, 'MM/DD/YYYY HH:mm:ss')\n    dt_aware = dt_naive.replace(tzinfo='America/New_York').datetime\n\n    return dt_aware\n\n\ndef data_parser(df):\n    \"\"\"\n    Takes dataframe and loops over rows to form dictionaries consisting of datetime and generation type.\n    Merges these dictionaries using datetime key.\n    Maps to type and returns a list of tuples containing datetime string and production.\n    \"\"\"\n\n    chunks = []\n    for row in df.itertuples():\n        piece = {}\n        piece['datetime'] = row[1]\n        piece[row[3]] = row[4]\n        chunks.append(piece)\n\n    # Join dicts on shared 'datetime' keys.\n    combine = defaultdict(dict)\n    for elem in chunks:\n        combine[elem['datetime']].update(elem)\n\n    ordered = sorted(combine.values(), key=itemgetter(\"datetime\"))\n\n    mapped_generation = []\n    for item in ordered:\n        mapped_types = [(mapping.get(k, k), v) for k, v in item.items()]\n\n        # Need to avoid multiple 'unknown' keys overwriting.\n        complete_production = defaultdict(lambda: 0.0)\n        for key, val in mapped_types:\n            try:\n                complete_production[key] += val\n            except TypeError:\n                # Datetime is a string at this point!\n                complete_production[key] = val\n\n        dt = complete_production.pop('datetime')\n        final = (dt, dict(complete_production))\n        mapped_generation.append(final)\n\n    return mapped_generation\n\n\ndef fetch_production(zone_key='US-NY', session=None, target_datetime=None, logger=None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given zone\n\n    Arguments:\n    zone_key: used in case a parser is able to fetch multiple zones\n    session: requests session passed in order to re-use an existing session,\n      not used here due to difficulty providing it to pandas\n    target_datetime: the datetime for which we want production data. If not provided, we should\n      default it to now. The provided target_datetime is timezone-aware in UTC.\n    logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically\n\n    Return:\n    A list of dictionaries in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        # ensure we have an arrow object\n        target_datetime = arrow.get(target_datetime)\n    else:\n        target_datetime = arrow.now('America/New_York')\n\n    ny_date = target_datetime.format('YYYYMMDD')\n    mix_url = 'http://mis.nyiso.com/public/csv/rtfuelmix/{}rtfuelmix.csv'.format(ny_date)\n    try:\n        raw_data = read_csv_data(mix_url)\n    except HTTPError:\n        # this can happen when target_datetime has no data available\n        return None\n\n    clean_data = data_parser(raw_data)\n\n    production_mix = []\n    for datapoint in clean_data:\n        data = {\n            'zoneKey': zone_key,\n            'datetime': timestamp_converter(datapoint[0]),\n            'production': datapoint[1],\n            'storage': {'hydro': None},\n            'source': 'nyiso.com'\n        }\n\n        production_mix.append(data)\n\n    return production_mix\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two zones\n\n    Arguments:\n    zone_key1, zone_key2: specifies which exchange to get\n    session: requests session passed in order to re-use an existing session,\n      not used here due to difficulty providing it to pandas\n    target_datetime: the datetime for which we want production data. If not provided, we should\n      default it to now. The provided target_datetime is timezone-aware in UTC.\n    logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically\n\n    Return:\n    A list of dictionaries in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    where net flow is from DK into NO\n    \"\"\"\n    url = 'http://mis.nyiso.com/public/csv/ExternalLimitsFlows/{}ExternalLimitsFlows.csv'\n\n    sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))\n\n    # In the source CSV, positive is flow into NY, negative is flow out of NY.\n    # In Electricity Map, A->B means flow to B is positive.\n    if sorted_zone_keys == 'US-NEISO->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - NE - NY', 'SCH - NPX_1385', 'SCH - NPX_CSC']\n    elif sorted_zone_keys == 'US-NY->US-PJM':\n        direction = -1\n        relevant_exchanges = ['SCH - PJ - NY', 'SCH - PJM_HTP', 'SCH - PJM_NEPTUNE', 'SCH - PJM_VFT']\n    elif sorted_zone_keys == 'CA-ON->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - OH - NY']\n    elif sorted_zone_keys == 'CA-QC->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - HQ_CEDARS', 'SCH - HQ - NY']\n    else:\n        raise NotImplementedError('Exchange pair not supported: {}'.format(sorted_zone_keys))\n\n    if target_datetime:\n        # ensure we have an arrow object\n        target_datetime = arrow.get(target_datetime)\n    else:\n        target_datetime = arrow.now('America/New_York')\n    ny_date = target_datetime.format('YYYYMMDD')\n    exchange_url = url.format(ny_date)\n\n    try:\n        exchange_data = read_csv_data(exchange_url)\n    except HTTPError:\n        # this can happen when target_datetime has no data available\n        return None\n\n    new_england_exs = exchange_data.loc[exchange_data['Interface Name'].isin(relevant_exchanges)]\n    consolidated_flows = new_england_exs.reset_index().groupby(\"Timestamp\").sum()\n\n    now = arrow.utcnow()\n\n    exchange_5min = []\n    for row in consolidated_flows.itertuples():\n        flow = float(row[3]) * direction\n        # Timestamp for exchange does not include seconds.\n        dt = timestamp_converter(row[0] + ':00')\n\n        if (dt > now) and ((dt - now) < timedelta(seconds=300)):\n            # NYISO exchanges CSV (and only the exchanges CSV) includes data\n            # up to 5 minutes in the future (but only 5 minutes in the future).\n            # This also happens on their official website.\n            # Electricity Map raises error with data in the future, so skip\n            # that datapoint. If it's more than 5 minutes in the future,\n            # it's weird/unexpected and thus worthy of failure and logging.\n            continue\n\n        exchange = {\n            'sortedZoneKeys': sorted_zone_keys,\n            'datetime': dt,\n            'netFlow': flow,\n            'source': 'nyiso.com'\n        }\n\n        exchange_5min.append(exchange)\n\n    return exchange_5min\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    from pprint import pprint\n    print('fetch_production() ->')\n    pprint(fetch_production())\n\n    print('fetch_production(target_datetime=arrow.get(\"2018-03-13T12:00Z\") ->')\n    pprint(fetch_production(target_datetime=arrow.get(\"2018-03-13T12:00Z\")))\n\n    print('fetch_production(target_datetime=arrow.get(\"2007-03-13T12:00Z\") ->')\n    pprint(fetch_production(target_datetime=arrow.get(\"2007-03-13T12:00Z\")))\n\n    print('fetch_exchange(US-NY, US-NEISO)')\n    pprint(fetch_exchange('US-NY', 'US-NEISO'))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\")')\n    pprint(fetch_exchange('US-NY', 'CA-QC'))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\", target_datetime=arrow.get(\"2018-03-13T12:00Z\"))')\n    pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get(\"2018-03-13T12:00Z\")))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\", target_datetime=arrow.get(\"2007-03-13T12:00Z\")))')\n    pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get('2007-03-13T12:00Z')))\n", "path": "parsers/US_NY.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\n\"\"\"Real time parser for the state of New York.\"\"\"\nfrom collections import defaultdict\nfrom datetime import timedelta\nfrom operator import itemgetter\nfrom urllib.error import HTTPError\n\nimport arrow\nimport pandas as pd\n\n# Dual Fuel systems can run either Natural Gas or Oil, they represent\n# significantly more capacity in NY State than plants that can only\n# burn Natural Gas. When looking up fuel usage for NY in 2016 in\n# https://www.eia.gov/electricity/data/state/annual_generation_state.xls\n# 100 times more energy came from NG than Oil. That means Oil\n# consumption in the Dual Fuel systems is roughly ~1%, and to a first\n# approximation it's just Natural Gas.\nmapping = {\n    'Dual Fuel': 'gas',\n    'Natural Gas': 'gas',\n    'Nuclear': 'nuclear',\n    'Other Fossil Fuels': 'unknown',\n    'Other Renewables': 'unknown',\n    'Wind': 'wind',\n    'Hydro': 'hydro'\n}\n\n\ndef read_csv_data(url):\n    \"\"\"\n    Gets csv data from a url and returns a dataframe.\n    \"\"\"\n\n    csv_data = pd.read_csv(url)\n\n    return csv_data\n\n\ndef timestamp_converter(timestamp_string):\n    \"\"\"\n    Converts timestamps in nyiso data into aware datetime objects.\n    \"\"\"\n\n    dt_naive = arrow.get(timestamp_string, 'MM/DD/YYYY HH:mm:ss')\n    dt_aware = dt_naive.replace(tzinfo='America/New_York').datetime\n\n    return dt_aware\n\n\ndef data_parser(df):\n    \"\"\"\n    Takes dataframe and loops over rows to form dictionaries consisting of datetime and generation type.\n    Merges these dictionaries using datetime key.\n    Maps to type and returns a list of tuples containing datetime string and production.\n    \"\"\"\n\n    chunks = []\n    for row in df.itertuples():\n        piece = {}\n        piece['datetime'] = row[1]\n        piece[row[3]] = row[4]\n        chunks.append(piece)\n\n    # Join dicts on shared 'datetime' keys.\n    combine = defaultdict(dict)\n    for elem in chunks:\n        combine[elem['datetime']].update(elem)\n\n    ordered = sorted(combine.values(), key=itemgetter(\"datetime\"))\n\n    mapped_generation = []\n    for item in ordered:\n        mapped_types = [(mapping.get(k, k), v) for k, v in item.items()]\n\n        # Need to avoid multiple 'unknown' keys overwriting.\n        complete_production = defaultdict(lambda: 0.0)\n        for key, val in mapped_types:\n            try:\n                complete_production[key] += val\n            except TypeError:\n                # Datetime is a string at this point!\n                complete_production[key] = val\n\n        dt = complete_production.pop('datetime')\n        final = (dt, dict(complete_production))\n        mapped_generation.append(final)\n\n    return mapped_generation\n\n\ndef fetch_production(zone_key='US-NY', session=None, target_datetime=None, logger=None):\n    \"\"\"\n    Requests the last known production mix (in MW) of a given zone\n\n    Arguments:\n    zone_key: used in case a parser is able to fetch multiple zones\n    session: requests session passed in order to re-use an existing session,\n      not used here due to difficulty providing it to pandas\n    target_datetime: the datetime for which we want production data. If not provided, we should\n      default it to now. The provided target_datetime is timezone-aware in UTC.\n    logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically\n\n    Return:\n    A list of dictionaries in the form:\n    {\n      'zoneKey': 'FR',\n      'datetime': '2017-01-01T00:00:00Z',\n      'production': {\n          'biomass': 0.0,\n          'coal': 0.0,\n          'gas': 0.0,\n          'hydro': 0.0,\n          'nuclear': null,\n          'oil': 0.0,\n          'solar': 0.0,\n          'wind': 0.0,\n          'geothermal': 0.0,\n          'unknown': 0.0\n      },\n      'storage': {\n          'hydro': -10.0,\n      },\n      'source': 'mysource.com'\n    }\n    \"\"\"\n    if target_datetime:\n        # ensure we have an arrow object\n        target_datetime = arrow.get(target_datetime)\n    else:\n        target_datetime = arrow.now('America/New_York')\n\n    ny_date = target_datetime.format('YYYYMMDD')\n    mix_url = 'http://mis.nyiso.com/public/csv/rtfuelmix/{}rtfuelmix.csv'.format(ny_date)\n    try:\n        raw_data = read_csv_data(mix_url)\n    except HTTPError:\n        # this can happen when target_datetime has no data available\n        return None\n\n    clean_data = data_parser(raw_data)\n\n    production_mix = []\n    for datapoint in clean_data:\n        data = {\n            'zoneKey': zone_key,\n            'datetime': timestamp_converter(datapoint[0]),\n            'production': datapoint[1],\n            'storage': {'hydro': None},\n            'source': 'nyiso.com'\n        }\n\n        production_mix.append(data)\n\n    return production_mix\n\n\ndef fetch_exchange(zone_key1, zone_key2, session=None, target_datetime=None, logger=None):\n    \"\"\"Requests the last known power exchange (in MW) between two zones\n\n    Arguments:\n    zone_key1, zone_key2: specifies which exchange to get\n    session: requests session passed in order to re-use an existing session,\n      not used here due to difficulty providing it to pandas\n    target_datetime: the datetime for which we want production data. If not provided, we should\n      default it to now. The provided target_datetime is timezone-aware in UTC.\n    logger: an instance of a `logging.Logger`; all raised exceptions are also logged automatically\n\n    Return:\n    A list of dictionaries in the form:\n    {\n      'sortedZoneKeys': 'DK->NO',\n      'datetime': '2017-01-01T00:00:00Z',\n      'netFlow': 0.0,\n      'source': 'mysource.com'\n    }\n    where net flow is from DK into NO\n    \"\"\"\n    url = 'http://mis.nyiso.com/public/csv/ExternalLimitsFlows/{}ExternalLimitsFlows.csv'\n\n    sorted_zone_keys = '->'.join(sorted([zone_key1, zone_key2]))\n\n    # In the source CSV, positive is flow into NY, negative is flow out of NY.\n    # In Electricity Map, A->B means flow to B is positive.\n    if sorted_zone_keys == 'US-NEISO->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - NE - NY', 'SCH - NPX_1385', 'SCH - NPX_CSC']\n    elif sorted_zone_keys == 'US-NY->US-PJM':\n        direction = -1\n        relevant_exchanges = ['SCH - PJ - NY', 'SCH - PJM_HTP', 'SCH - PJM_NEPTUNE', 'SCH - PJM_VFT']\n    elif sorted_zone_keys == 'CA-ON->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - OH - NY']\n    elif sorted_zone_keys == 'CA-QC->US-NY':\n        direction = 1\n        relevant_exchanges = ['SCH - HQ_CEDARS', 'SCH - HQ - NY']\n    else:\n        raise NotImplementedError('Exchange pair not supported: {}'.format(sorted_zone_keys))\n\n    if target_datetime:\n        # ensure we have an arrow object\n        target_datetime = arrow.get(target_datetime)\n    else:\n        target_datetime = arrow.now('America/New_York')\n    ny_date = target_datetime.format('YYYYMMDD')\n    exchange_url = url.format(ny_date)\n\n    try:\n        exchange_data = read_csv_data(exchange_url)\n    except HTTPError:\n        # this can happen when target_datetime has no data available\n        return None\n\n    new_england_exs = exchange_data.loc[exchange_data['Interface Name'].isin(relevant_exchanges)]\n    consolidated_flows = new_england_exs.reset_index().groupby(\"Timestamp\").sum()\n\n    now = arrow.utcnow()\n\n    exchange_5min = []\n    for row in consolidated_flows.itertuples():\n        flow = float(row[3]) * direction\n        # Timestamp for exchange does not include seconds.\n        dt = timestamp_converter(row[0] + ':00')\n\n        if (dt > now) and ((dt - now) < timedelta(seconds=300)):\n            # NYISO exchanges CSV (and only the exchanges CSV) includes data\n            # up to 5 minutes in the future (but only 5 minutes in the future).\n            # This also happens on their official website.\n            # Electricity Map raises error with data in the future, so skip\n            # that datapoint. If it's more than 5 minutes in the future,\n            # it's weird/unexpected and thus worthy of failure and logging.\n            continue\n\n        exchange = {\n            'sortedZoneKeys': sorted_zone_keys,\n            'datetime': dt,\n            'netFlow': flow,\n            'source': 'nyiso.com'\n        }\n\n        exchange_5min.append(exchange)\n\n    return exchange_5min\n\n\nif __name__ == '__main__':\n    \"\"\"Main method, never used by the Electricity Map backend, but handy for testing.\"\"\"\n\n    from pprint import pprint\n    print('fetch_production() ->')\n    pprint(fetch_production())\n\n    print('fetch_production(target_datetime=arrow.get(\"2018-03-13T12:00Z\") ->')\n    pprint(fetch_production(target_datetime=arrow.get(\"2018-03-13T12:00Z\")))\n\n    print('fetch_production(target_datetime=arrow.get(\"2007-03-13T12:00Z\") ->')\n    pprint(fetch_production(target_datetime=arrow.get(\"2007-03-13T12:00Z\")))\n\n    print('fetch_exchange(US-NY, US-NEISO)')\n    pprint(fetch_exchange('US-NY', 'US-NEISO'))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\")')\n    pprint(fetch_exchange('US-NY', 'CA-QC'))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\", target_datetime=arrow.get(\"2018-03-13T12:00Z\"))')\n    pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get(\"2018-03-13T12:00Z\")))\n\n    print('fetch_exchange(\"US-NY\", \"CA-QC\", target_datetime=arrow.get(\"2007-03-13T12:00Z\")))')\n    pprint(fetch_exchange('US-NY', 'CA-QC', target_datetime=arrow.get('2007-03-13T12:00Z')))\n", "path": "parsers/US_NY.py"}]}