We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
RAMSES key error
### Bug report

**Bug summary**

When creating a derived field for a RAMSES dataset that depends on hydro and rt (any combination as long as the fields aren't stored in the same file), a `KeyError` is raised. This is due to the fact that the code is trying to read all the fields from the same file.

**Code for reproduction**

This requires [ramses_rt_00088](http://yt-project.org/data/ramses_rt_00088.tar.gz). See https://gist.github.com/cphyc/dc445db37e09bf2a89adf50409866f26.

**Version Information**
<!--Please specify your platform and versions of the relevant libraries you are
using:-->
  * Operating System: Arch linux
  * Python Version: 3.6.5
  * yt version: master
  * Other Libraries (if applicable): 


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `yt/frontends/ramses/io.py`
Content:
```
1 """
2 RAMSES-specific IO
3 
4 
5 
6 """
7 
8 #-----------------------------------------------------------------------------
9 # Copyright (c) 2013, yt Development Team.
10 #
11 # Distributed under the terms of the Modified BSD License.
12 #
13 # The full license is in the file COPYING.txt, distributed with this software.
14 #-----------------------------------------------------------------------------
15 
16 from collections import defaultdict
17 import numpy as np
18 
19 from yt.utilities.io_handler import \
20     BaseIOHandler
21 from yt.utilities.logger import ytLogger as mylog
22 from yt.utilities.physical_ratios import cm_per_km, cm_per_mpc
23 from yt.utilities.cython_fortran_utils import FortranFile
24 from yt.utilities.exceptions import YTFieldTypeNotFound, YTParticleOutputFormatNotImplemented, \
25     YTFileNotParseable
26 import re
27 
28 def convert_ramses_ages(ds, conformal_ages):
29     tf = ds.t_frw
30     dtau = ds.dtau
31     tauf = ds.tau_frw
32     tsim = ds.time_simu
33     h100 = ds.hubble_constant
34     nOver2 = ds.n_frw/2
35     unit_t = ds.parameters['unit_t']
36     t_scale = 1./(h100 * 100 * cm_per_km / cm_per_mpc) / unit_t
37 
38     # calculate index into lookup table (n_frw elements in
39     # lookup table)
40     dage =  1 + (10*conformal_ages/dtau)
41     dage = np.minimum(dage, nOver2 + (dage - nOver2)/10.)
42     iage = np.array(dage, dtype=np.int32)
43 
44     # linearly interpolate physical times from tf and tauf lookup
45     # tables.
46     t = ((tf[iage]*(conformal_ages - tauf[iage - 1]) /
47           (tauf[iage] - tauf[iage - 1])))
48     t = t + ((tf[iage-1]*(conformal_ages - tauf[iage]) /
49               (tauf[iage-1]-tauf[iage])))
50     return (tsim - t)*t_scale
51 
52 
53 def _ramses_particle_file_handler(fname, foffsets, data_types,
54                                   subset, fields, count):
55     '''General file handler, called by _read_particle_subset
56 
57     Parameters
58     ----------
59     fname : string
60         filename to read from
61     foffsets: dict
62         Offsets in file of the fields
63     data_types: dict
64         Data type of the fields
65     subset: ``RAMSESDomainSubset``
66         A RAMSES domain subset object
67     fields: list of tuple
68         The fields to read
69     count: integer
70         The number of elements to count
71     '''
72     tr = {}
73     ds = subset.domain.ds
74     with FortranFile(fname) as fd:
75         # We do *all* conversion into boxlen here.
76         # This means that no other conversions need to be applied to convert
77         # positions into the same domain as the octs themselves.
78         for field in sorted(fields, key=lambda a: foffsets[a]):
79             if count == 0:
80                 tr[field] = np.empty(0, dtype=data_types[field])
81                 continue
82             fd.seek(foffsets[field])
83             dt = data_types[field]
84             tr[field] = fd.read_vector(dt)
85             if field[1].startswith("particle_position"):
86                 np.divide(tr[field], ds["boxlen"], tr[field])
87             if ds.cosmological_simulation and field[1] == "particle_birth_time":
88                 conformal_age = tr[field]
89                 tr[field] = convert_ramses_ages(ds, conformal_age)
90                 # arbitrarily set particles with zero conformal_age to zero
91                 # particle_age. This corresponds to DM particles.
92                 tr[field][conformal_age == 0] = 0
93     return tr
94 
95 
96 class IOHandlerRAMSES(BaseIOHandler):
97     _dataset_type = "ramses"
98 
99     def _generic_fluid_handler(self, chunks, selector, fields, size, ftype):
100         tr = defaultdict(list)
101 
102         for chunk in chunks:
103             for subset in chunk.objs:
104                 fname = None
105                 for fh in subset.domain.field_handlers:
106                     if fh.ftype == ftype:
107                         file_handler = fh
108                         fname = fh.fname
109                         break
110 
111                 if fname is None:
112                     raise YTFieldTypeNotFound(ftype)
113 
114                 # Now we read the entire thing
115                 with FortranFile(fname) as fd:
116                     # This contains the boundary information, so we skim through
117                     # and pick off the right vectors
118                     rv = subset.fill(fd, fields, selector, file_handler)
119                 for ft, f in fields:
120                     d = rv.pop(f)
121                     mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
122                         f, d.size, d.min(), d.max(), d.size)
123                     tr[(ft, f)].append(d)
124         d = {}
125         for field in fields:
126             d[field] = np.concatenate(tr.pop(field))
127 
128         return d
129 
130     def _read_fluid_selection(self, chunks, selector, fields, size):
131         d = {}
132 
133         # Group fields by field type
134         for ft in set(f[0] for f in fields):
135             # Select the fields for the current reader
136             fields_subs = list(
137                 filter(lambda f: f[0]==ft,
138                        fields))
139 
140             newd = self._generic_fluid_handler(chunks, selector, fields_subs, size,
141                                                ft)
142             d.update(newd)
143 
144         return d
145 
146     def _read_particle_coords(self, chunks, ptf):
147         pn = "particle_position_%s"
148         fields = [(ptype, "particle_position_%s" % ax)
149                   for ptype, field_list in ptf.items()
150                   for ax in 'xyz']
151         for chunk in chunks:
152             for subset in chunk.objs:
153                 rv = self._read_particle_subset(subset, fields)
154                 for ptype in sorted(ptf):
155                     yield ptype, (rv[ptype, pn % 'x'],
156                                   rv[ptype, pn % 'y'],
157                                   rv[ptype, pn % 'z'])
158 
159     def _read_particle_fields(self, chunks, ptf, selector):
160         pn = "particle_position_%s"
161         chunks = list(chunks)
162         fields = [(ptype, fname) for ptype, field_list in ptf.items()
163                                  for fname in field_list]
164         for ptype, field_list in sorted(ptf.items()):
165             for ax in 'xyz':
166                 if pn % ax not in field_list:
167                     fields.append((ptype, pn % ax))
168         for chunk in chunks:
169             for subset in chunk.objs:
170                 rv = self._read_particle_subset(subset, fields)
171                 for ptype, field_list in sorted(ptf.items()):
172                     x, y, z = (np.asarray(rv[ptype, pn % ax], "=f8")
173                                for ax in 'xyz')
174                     mask = selector.select_points(x, y, z, 0.0)
175                     if mask is None:
176                        mask = []
177                     for field in field_list:
178                         data = np.asarray(rv.pop((ptype, field))[mask], "=f8")
179                         yield (ptype, field), data
180 
181 
182     def _read_particle_subset(self, subset, fields):
183         '''Read the particle files.'''
184         tr = {}
185 
186         # Sequential read depending on particle type
187         for ptype in set(f[0] for f in fields):
188 
189             # Select relevant fiels
190             subs_fields = filter(lambda f: f[0] == ptype, fields)
191 
192             ok = False
193             for ph in subset.domain.particle_handlers:
194                 if ph.ptype == ptype:
195                     fname = ph.fname
196                     foffsets = ph.field_offsets
197                     data_types = ph.field_types
198                     ok = True
199                     count = ph.local_particle_count
200                     break
201             if not ok:
202                 raise YTFieldTypeNotFound(ptype)
203 
204             cosmo = self.ds.cosmological_simulation
205             if (ptype, 'particle_birth_time') in foffsets and cosmo:
206                 foffsets[ptype, 'conformal_birth_time'] = \
207                     foffsets[ptype, 'particle_birth_time']
208                 data_types[ptype, 'conformal_birth_time'] = \
209                     data_types[ptype, 'particle_birth_time']
210 
211             tr.update(_ramses_particle_file_handler(
212                 fname, foffsets, data_types, subset, subs_fields,
213                 count=count
214             ))
215 
216         return tr
217 
218 
219 def _read_part_file_descriptor(fname):
220     """
221     Read a file descriptor and returns the array of the fields found.
222     """
223     VERSION_RE = re.compile(r'# version: *(\d+)')
224     VAR_DESC_RE = re.compile(r'\s*(\d+),\s*(\w+),\s*(\w+)')
225 
226     # Mapping
227     mapping = [
228         ('position_x', 'particle_position_x'),
229         ('position_y', 'particle_position_y'),
230         ('position_z', 'particle_position_z'),
231         ('velocity_x', 'particle_velocity_x'),
232         ('velocity_y', 'particle_velocity_y'),
233         ('velocity_z', 'particle_velocity_z'),
234         ('mass', 'particle_mass'),
235         ('identity', 'particle_identity'),
236         ('levelp', 'particle_level'),
237         ('family', 'particle_family'),
238         ('tag', 'particle_tag')
239     ]
240     # Convert in dictionary
241     mapping = {k: v for k, v in mapping}
242 
243     with open(fname, 'r') as f:
244         line = f.readline()
245         tmp = VERSION_RE.match(line)
246         mylog.debug('Reading part file descriptor %s.' % fname)
247         if not tmp:
248             raise YTParticleOutputFormatNotImplemented()
249 
250         version = int(tmp.group(1))
251 
252         if version == 1:
253             # Skip one line (containing the headers)
254             line = f.readline()
255             fields = []
256             for i, line in enumerate(f.readlines()):
257                 tmp = VAR_DESC_RE.match(line)
258                 if not tmp:
259                     raise YTFileNotParseable(fname, i+1)
260 
261                 # ivar = tmp.group(1)
262                 varname = tmp.group(2)
263                 dtype = tmp.group(3)
264 
265                 if varname in mapping:
266                     varname = mapping[varname]
267                 else:
268                     varname = 'particle_%s' % varname
269 
270                 fields.append((varname, dtype))
271         else:
272             raise YTParticleOutputFormatNotImplemented()
273 
274     return fields
275 
276 def _read_fluid_file_descriptor(fname):
277     """
278     Read a file descriptor and returns the array of the fields found.
279     """
280     VERSION_RE = re.compile(r'# version: *(\d+)')
281     VAR_DESC_RE = re.compile(r'\s*(\d+),\s*(\w+),\s*(\w+)')
282 
283     # Mapping
284     mapping = [
285         ('density', 'Density'),
286         ('velocity_x', 'x-velocity'),
287         ('velocity_y', 'y-velocity'),
288         ('velocity_z', 'z-velocity'),
289         ('pressure', 'Pressure'),
290         ('metallicity', 'Metallicity'),
291     ]
292     # Convert in dictionary
293     mapping = {k: v for k, v in mapping}
294 
295     with open(fname, 'r') as f:
296         line = f.readline()
297         tmp = VERSION_RE.match(line)
298         mylog.debug('Reading fluid file descriptor %s.' % fname)
299         if not tmp:
300             return []
301 
302         version = int(tmp.group(1))
303 
304         if version == 1:
305             # Skip one line (containing the headers)
306             line = f.readline()
307             fields = []
308             for i, line in enumerate(f.readlines()):
309                 tmp = VAR_DESC_RE.match(line)
310                 if not tmp:
311                     raise YTFileNotParseable(fname, i+1)
312 
313                 # ivar = tmp.group(1)
314                 varname = tmp.group(2)
315                 dtype = tmp.group(3)
316 
317                 if varname in mapping:
318                     varname = mapping[varname]
319                 else:
320                     varname = 'hydro_%s' % varname
321 
322                 fields.append((varname, dtype))
323         else:
324             mylog.error('Version %s', version)
325             raise YTParticleOutputFormatNotImplemented()
326 
327     return fields
328
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/yt/frontends/ramses/io.py b/yt/frontends/ramses/io.py
--- a/yt/frontends/ramses/io.py
+++ b/yt/frontends/ramses/io.py
@@ -96,53 +96,46 @@
 class IOHandlerRAMSES(BaseIOHandler):
     _dataset_type = "ramses"
 
-    def _generic_fluid_handler(self, chunks, selector, fields, size, ftype):
+    def _read_fluid_selection(self, chunks, selector, fields, size):
         tr = defaultdict(list)
 
+        # Set of field types
+        ftypes = set(f[0] for f in fields)
         for chunk in chunks:
-            for subset in chunk.objs:
-                fname = None
-                for fh in subset.domain.field_handlers:
-                    if fh.ftype == ftype:
-                        file_handler = fh
-                        fname = fh.fname
-                        break
-
-                if fname is None:
-                    raise YTFieldTypeNotFound(ftype)
-
-                # Now we read the entire thing
-                with FortranFile(fname) as fd:
-                    # This contains the boundary information, so we skim through
-                    # and pick off the right vectors
-                    rv = subset.fill(fd, fields, selector, file_handler)
-                for ft, f in fields:
-                    d = rv.pop(f)
-                    mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
-                        f, d.size, d.min(), d.max(), d.size)
-                    tr[(ft, f)].append(d)
+            # Gather fields by type to minimize i/o operations
+            for ft in ftypes:
+                # Get all the fields of the same type
+                field_subs = list(
+                    filter(lambda f: f[0]==ft, fields))
+
+                # Loop over subsets
+                for subset in chunk.objs:
+                    fname = None
+                    for fh in subset.domain.field_handlers:
+                        if fh.ftype == ft:
+                            file_handler = fh
+                            fname = fh.fname
+                            break
+
+                    if fname is None:
+                        raise YTFieldTypeNotFound(ft)
+
+                    # Now we read the entire thing
+                    with FortranFile(fname) as fd:
+                        # This contains the boundary information, so we skim through
+                        # and pick off the right vectors
+                        rv = subset.fill(fd, field_subs, selector, file_handler)
+                    for ft, f in field_subs:
+                        d = rv.pop(f)
+                        mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
+                            f, d.size, d.min(), d.max(), d.size)
+                        tr[(ft, f)].append(d)
         d = {}
         for field in fields:
             d[field] = np.concatenate(tr.pop(field))
 
         return d
 
-    def _read_fluid_selection(self, chunks, selector, fields, size):
-        d = {}
-
-        # Group fields by field type
-        for ft in set(f[0] for f in fields):
-            # Select the fields for the current reader
-            fields_subs = list(
-                filter(lambda f: f[0]==ft,
-                       fields))
-
-            newd = self._generic_fluid_handler(chunks, selector, fields_subs, size,
-                                               ft)
-            d.update(newd)
-
-        return d
-
     def _read_particle_coords(self, chunks, ptf):
         pn = "particle_position_%s"
         fields = [(ptype, "particle_position_%s" % ax)

{"golden_diff": "diff --git a/yt/frontends/ramses/io.py b/yt/frontends/ramses/io.py\n--- a/yt/frontends/ramses/io.py\n+++ b/yt/frontends/ramses/io.py\n@@ -96,53 +96,46 @@\n class IOHandlerRAMSES(BaseIOHandler):\n     _dataset_type = \"ramses\"\n \n-    def _generic_fluid_handler(self, chunks, selector, fields, size, ftype):\n+    def _read_fluid_selection(self, chunks, selector, fields, size):\n         tr = defaultdict(list)\n \n+        # Set of field types\n+        ftypes = set(f[0] for f in fields)\n         for chunk in chunks:\n-            for subset in chunk.objs:\n-                fname = None\n-                for fh in subset.domain.field_handlers:\n-                    if fh.ftype == ftype:\n-                        file_handler = fh\n-                        fname = fh.fname\n-                        break\n-\n-                if fname is None:\n-                    raise YTFieldTypeNotFound(ftype)\n-\n-                # Now we read the entire thing\n-                with FortranFile(fname) as fd:\n-                    # This contains the boundary information, so we skim through\n-                    # and pick off the right vectors\n-                    rv = subset.fill(fd, fields, selector, file_handler)\n-                for ft, f in fields:\n-                    d = rv.pop(f)\n-                    mylog.debug(\"Filling %s with %s (%0.3e %0.3e) (%s zones)\",\n-                        f, d.size, d.min(), d.max(), d.size)\n-                    tr[(ft, f)].append(d)\n+            # Gather fields by type to minimize i/o operations\n+            for ft in ftypes:\n+                # Get all the fields of the same type\n+                field_subs = list(\n+                    filter(lambda f: f[0]==ft, fields))\n+\n+                # Loop over subsets\n+                for subset in chunk.objs:\n+                    fname = None\n+                    for fh in subset.domain.field_handlers:\n+                        if fh.ftype == ft:\n+                            file_handler = fh\n+                            fname = fh.fname\n+                            break\n+\n+                    if fname is None:\n+                        raise YTFieldTypeNotFound(ft)\n+\n+                    # Now we read the entire thing\n+                    with FortranFile(fname) as fd:\n+                        # This contains the boundary information, so we skim through\n+                        # and pick off the right vectors\n+                        rv = subset.fill(fd, field_subs, selector, file_handler)\n+                    for ft, f in field_subs:\n+                        d = rv.pop(f)\n+                        mylog.debug(\"Filling %s with %s (%0.3e %0.3e) (%s zones)\",\n+                            f, d.size, d.min(), d.max(), d.size)\n+                        tr[(ft, f)].append(d)\n         d = {}\n         for field in fields:\n             d[field] = np.concatenate(tr.pop(field))\n \n         return d\n \n-    def _read_fluid_selection(self, chunks, selector, fields, size):\n-        d = {}\n-\n-        # Group fields by field type\n-        for ft in set(f[0] for f in fields):\n-            # Select the fields for the current reader\n-            fields_subs = list(\n-                filter(lambda f: f[0]==ft,\n-                       fields))\n-\n-            newd = self._generic_fluid_handler(chunks, selector, fields_subs, size,\n-                                               ft)\n-            d.update(newd)\n-\n-        return d\n-\n     def _read_particle_coords(self, chunks, ptf):\n         pn = \"particle_position_%s\"\n         fields = [(ptype, \"particle_position_%s\" % ax)\n", "issue": "RAMSES key error\n### Bug report\r\n\r\n**Bug summary**\r\n\r\nWhen creating a derived field for a RAMSES dataset that depends on hydro and rt (any combination as long as the fields aren't stored in the same file), a `KeyError` is raised. This is due to the fact that the code is trying to read all the fields from the same file.\r\n\r\n**Code for reproduction**\r\n\r\nThis requires [ramses_rt_00088](http://yt-project.org/data/ramses_rt_00088.tar.gz). See https://gist.github.com/cphyc/dc445db37e09bf2a89adf50409866f26.\r\n\r\n**Version Information**\r\n<!--Please specify your platform and versions of the relevant libraries you are\r\nusing:-->\r\n  * Operating System: Arch linux\r\n  * Python Version: 3.6.5\r\n  * yt version: master\r\n  * Other Libraries (if applicable): \r\n\n", "before_files": [{"content": "\"\"\"\nRAMSES-specific IO\n\n\n\n\"\"\"\n\n#-----------------------------------------------------------------------------\n# Copyright (c) 2013, yt Development Team.\n#\n# Distributed under the terms of the Modified BSD License.\n#\n# The full license is in the file COPYING.txt, distributed with this software.\n#-----------------------------------------------------------------------------\n\nfrom collections import defaultdict\nimport numpy as np\n\nfrom yt.utilities.io_handler import \\\n    BaseIOHandler\nfrom yt.utilities.logger import ytLogger as mylog\nfrom yt.utilities.physical_ratios import cm_per_km, cm_per_mpc\nfrom yt.utilities.cython_fortran_utils import FortranFile\nfrom yt.utilities.exceptions import YTFieldTypeNotFound, YTParticleOutputFormatNotImplemented, \\\n    YTFileNotParseable\nimport re\n\ndef convert_ramses_ages(ds, conformal_ages):\n    tf = ds.t_frw\n    dtau = ds.dtau\n    tauf = ds.tau_frw\n    tsim = ds.time_simu\n    h100 = ds.hubble_constant\n    nOver2 = ds.n_frw/2\n    unit_t = ds.parameters['unit_t']\n    t_scale = 1./(h100 * 100 * cm_per_km / cm_per_mpc) / unit_t\n\n    # calculate index into lookup table (n_frw elements in\n    # lookup table)\n    dage =  1 + (10*conformal_ages/dtau)\n    dage = np.minimum(dage, nOver2 + (dage - nOver2)/10.)\n    iage = np.array(dage, dtype=np.int32)\n\n    # linearly interpolate physical times from tf and tauf lookup\n    # tables.\n    t = ((tf[iage]*(conformal_ages - tauf[iage - 1]) /\n          (tauf[iage] - tauf[iage - 1])))\n    t = t + ((tf[iage-1]*(conformal_ages - tauf[iage]) /\n              (tauf[iage-1]-tauf[iage])))\n    return (tsim - t)*t_scale\n\n\ndef _ramses_particle_file_handler(fname, foffsets, data_types,\n                                  subset, fields, count):\n    '''General file handler, called by _read_particle_subset\n\n    Parameters\n    ----------\n    fname : string\n        filename to read from\n    foffsets: dict\n        Offsets in file of the fields\n    data_types: dict\n        Data type of the fields\n    subset: ``RAMSESDomainSubset``\n        A RAMSES domain subset object\n    fields: list of tuple\n        The fields to read\n    count: integer\n        The number of elements to count\n    '''\n    tr = {}\n    ds = subset.domain.ds\n    with FortranFile(fname) as fd:\n        # We do *all* conversion into boxlen here.\n        # This means that no other conversions need to be applied to convert\n        # positions into the same domain as the octs themselves.\n        for field in sorted(fields, key=lambda a: foffsets[a]):\n            if count == 0:\n                tr[field] = np.empty(0, dtype=data_types[field])\n                continue\n            fd.seek(foffsets[field])\n            dt = data_types[field]\n            tr[field] = fd.read_vector(dt)\n            if field[1].startswith(\"particle_position\"):\n                np.divide(tr[field], ds[\"boxlen\"], tr[field])\n            if ds.cosmological_simulation and field[1] == \"particle_birth_time\":\n                conformal_age = tr[field]\n                tr[field] = convert_ramses_ages(ds, conformal_age)\n                # arbitrarily set particles with zero conformal_age to zero\n                # particle_age. This corresponds to DM particles.\n                tr[field][conformal_age == 0] = 0\n    return tr\n\n\nclass IOHandlerRAMSES(BaseIOHandler):\n    _dataset_type = \"ramses\"\n\n    def _generic_fluid_handler(self, chunks, selector, fields, size, ftype):\n        tr = defaultdict(list)\n\n        for chunk in chunks:\n            for subset in chunk.objs:\n                fname = None\n                for fh in subset.domain.field_handlers:\n                    if fh.ftype == ftype:\n                        file_handler = fh\n                        fname = fh.fname\n                        break\n\n                if fname is None:\n                    raise YTFieldTypeNotFound(ftype)\n\n                # Now we read the entire thing\n                with FortranFile(fname) as fd:\n                    # This contains the boundary information, so we skim through\n                    # and pick off the right vectors\n                    rv = subset.fill(fd, fields, selector, file_handler)\n                for ft, f in fields:\n                    d = rv.pop(f)\n                    mylog.debug(\"Filling %s with %s (%0.3e %0.3e) (%s zones)\",\n                        f, d.size, d.min(), d.max(), d.size)\n                    tr[(ft, f)].append(d)\n        d = {}\n        for field in fields:\n            d[field] = np.concatenate(tr.pop(field))\n\n        return d\n\n    def _read_fluid_selection(self, chunks, selector, fields, size):\n        d = {}\n\n        # Group fields by field type\n        for ft in set(f[0] for f in fields):\n            # Select the fields for the current reader\n            fields_subs = list(\n                filter(lambda f: f[0]==ft,\n                       fields))\n\n            newd = self._generic_fluid_handler(chunks, selector, fields_subs, size,\n                                               ft)\n            d.update(newd)\n\n        return d\n\n    def _read_particle_coords(self, chunks, ptf):\n        pn = \"particle_position_%s\"\n        fields = [(ptype, \"particle_position_%s\" % ax)\n                  for ptype, field_list in ptf.items()\n                  for ax in 'xyz']\n        for chunk in chunks:\n            for subset in chunk.objs:\n                rv = self._read_particle_subset(subset, fields)\n                for ptype in sorted(ptf):\n                    yield ptype, (rv[ptype, pn % 'x'],\n                                  rv[ptype, pn % 'y'],\n                                  rv[ptype, pn % 'z'])\n\n    def _read_particle_fields(self, chunks, ptf, selector):\n        pn = \"particle_position_%s\"\n        chunks = list(chunks)\n        fields = [(ptype, fname) for ptype, field_list in ptf.items()\n                                 for fname in field_list]\n        for ptype, field_list in sorted(ptf.items()):\n            for ax in 'xyz':\n                if pn % ax not in field_list:\n                    fields.append((ptype, pn % ax))\n        for chunk in chunks:\n            for subset in chunk.objs:\n                rv = self._read_particle_subset(subset, fields)\n                for ptype, field_list in sorted(ptf.items()):\n                    x, y, z = (np.asarray(rv[ptype, pn % ax], \"=f8\")\n                               for ax in 'xyz')\n                    mask = selector.select_points(x, y, z, 0.0)\n                    if mask is None:\n                       mask = []\n                    for field in field_list:\n                        data = np.asarray(rv.pop((ptype, field))[mask], \"=f8\")\n                        yield (ptype, field), data\n\n\n    def _read_particle_subset(self, subset, fields):\n        '''Read the particle files.'''\n        tr = {}\n\n        # Sequential read depending on particle type\n        for ptype in set(f[0] for f in fields):\n\n            # Select relevant fiels\n            subs_fields = filter(lambda f: f[0] == ptype, fields)\n\n            ok = False\n            for ph in subset.domain.particle_handlers:\n                if ph.ptype == ptype:\n                    fname = ph.fname\n                    foffsets = ph.field_offsets\n                    data_types = ph.field_types\n                    ok = True\n                    count = ph.local_particle_count\n                    break\n            if not ok:\n                raise YTFieldTypeNotFound(ptype)\n\n            cosmo = self.ds.cosmological_simulation\n            if (ptype, 'particle_birth_time') in foffsets and cosmo:\n                foffsets[ptype, 'conformal_birth_time'] = \\\n                    foffsets[ptype, 'particle_birth_time']\n                data_types[ptype, 'conformal_birth_time'] = \\\n                    data_types[ptype, 'particle_birth_time']\n\n            tr.update(_ramses_particle_file_handler(\n                fname, foffsets, data_types, subset, subs_fields,\n                count=count\n            ))\n\n        return tr\n\n\ndef _read_part_file_descriptor(fname):\n    \"\"\"\n    Read a file descriptor and returns the array of the fields found.\n    \"\"\"\n    VERSION_RE = re.compile(r'# version: *(\\d+)')\n    VAR_DESC_RE = re.compile(r'\\s*(\\d+),\\s*(\\w+),\\s*(\\w+)')\n\n    # Mapping\n    mapping = [\n        ('position_x', 'particle_position_x'),\n        ('position_y', 'particle_position_y'),\n        ('position_z', 'particle_position_z'),\n        ('velocity_x', 'particle_velocity_x'),\n        ('velocity_y', 'particle_velocity_y'),\n        ('velocity_z', 'particle_velocity_z'),\n        ('mass', 'particle_mass'),\n        ('identity', 'particle_identity'),\n        ('levelp', 'particle_level'),\n        ('family', 'particle_family'),\n        ('tag', 'particle_tag')\n    ]\n    # Convert in dictionary\n    mapping = {k: v for k, v in mapping}\n\n    with open(fname, 'r') as f:\n        line = f.readline()\n        tmp = VERSION_RE.match(line)\n        mylog.debug('Reading part file descriptor %s.' % fname)\n        if not tmp:\n            raise YTParticleOutputFormatNotImplemented()\n\n        version = int(tmp.group(1))\n\n        if version == 1:\n            # Skip one line (containing the headers)\n            line = f.readline()\n            fields = []\n            for i, line in enumerate(f.readlines()):\n                tmp = VAR_DESC_RE.match(line)\n                if not tmp:\n                    raise YTFileNotParseable(fname, i+1)\n\n                # ivar = tmp.group(1)\n                varname = tmp.group(2)\n                dtype = tmp.group(3)\n\n                if varname in mapping:\n                    varname = mapping[varname]\n                else:\n                    varname = 'particle_%s' % varname\n\n                fields.append((varname, dtype))\n        else:\n            raise YTParticleOutputFormatNotImplemented()\n\n    return fields\n\ndef _read_fluid_file_descriptor(fname):\n    \"\"\"\n    Read a file descriptor and returns the array of the fields found.\n    \"\"\"\n    VERSION_RE = re.compile(r'# version: *(\\d+)')\n    VAR_DESC_RE = re.compile(r'\\s*(\\d+),\\s*(\\w+),\\s*(\\w+)')\n\n    # Mapping\n    mapping = [\n        ('density', 'Density'),\n        ('velocity_x', 'x-velocity'),\n        ('velocity_y', 'y-velocity'),\n        ('velocity_z', 'z-velocity'),\n        ('pressure', 'Pressure'),\n        ('metallicity', 'Metallicity'),\n    ]\n    # Convert in dictionary\n    mapping = {k: v for k, v in mapping}\n\n    with open(fname, 'r') as f:\n        line = f.readline()\n        tmp = VERSION_RE.match(line)\n        mylog.debug('Reading fluid file descriptor %s.' % fname)\n        if not tmp:\n            return []\n\n        version = int(tmp.group(1))\n\n        if version == 1:\n            # Skip one line (containing the headers)\n            line = f.readline()\n            fields = []\n            for i, line in enumerate(f.readlines()):\n                tmp = VAR_DESC_RE.match(line)\n                if not tmp:\n                    raise YTFileNotParseable(fname, i+1)\n\n                # ivar = tmp.group(1)\n                varname = tmp.group(2)\n                dtype = tmp.group(3)\n\n                if varname in mapping:\n                    varname = mapping[varname]\n                else:\n                    varname = 'hydro_%s' % varname\n\n                fields.append((varname, dtype))\n        else:\n            mylog.error('Version %s', version)\n            raise YTParticleOutputFormatNotImplemented()\n\n    return fields\n", "path": "yt/frontends/ramses/io.py"}], "after_files": [{"content": "\"\"\"\nRAMSES-specific IO\n\n\n\n\"\"\"\n\n#-----------------------------------------------------------------------------\n# Copyright (c) 2013, yt Development Team.\n#\n# Distributed under the terms of the Modified BSD License.\n#\n# The full license is in the file COPYING.txt, distributed with this software.\n#-----------------------------------------------------------------------------\n\nfrom collections import defaultdict\nimport numpy as np\n\nfrom yt.utilities.io_handler import \\\n    BaseIOHandler\nfrom yt.utilities.logger import ytLogger as mylog\nfrom yt.utilities.physical_ratios import cm_per_km, cm_per_mpc\nfrom yt.utilities.cython_fortran_utils import FortranFile\nfrom yt.utilities.exceptions import YTFieldTypeNotFound, YTParticleOutputFormatNotImplemented, \\\n    YTFileNotParseable\nimport re\n\ndef convert_ramses_ages(ds, conformal_ages):\n    tf = ds.t_frw\n    dtau = ds.dtau\n    tauf = ds.tau_frw\n    tsim = ds.time_simu\n    h100 = ds.hubble_constant\n    nOver2 = ds.n_frw/2\n    unit_t = ds.parameters['unit_t']\n    t_scale = 1./(h100 * 100 * cm_per_km / cm_per_mpc) / unit_t\n\n    # calculate index into lookup table (n_frw elements in\n    # lookup table)\n    dage =  1 + (10*conformal_ages/dtau)\n    dage = np.minimum(dage, nOver2 + (dage - nOver2)/10.)\n    iage = np.array(dage, dtype=np.int32)\n\n    # linearly interpolate physical times from tf and tauf lookup\n    # tables.\n    t = ((tf[iage]*(conformal_ages - tauf[iage - 1]) /\n          (tauf[iage] - tauf[iage - 1])))\n    t = t + ((tf[iage-1]*(conformal_ages - tauf[iage]) /\n              (tauf[iage-1]-tauf[iage])))\n    return (tsim - t)*t_scale\n\n\ndef _ramses_particle_file_handler(fname, foffsets, data_types,\n                                  subset, fields, count):\n    '''General file handler, called by _read_particle_subset\n\n    Parameters\n    ----------\n    fname : string\n        filename to read from\n    foffsets: dict\n        Offsets in file of the fields\n    data_types: dict\n        Data type of the fields\n    subset: ``RAMSESDomainSubset``\n        A RAMSES domain subset object\n    fields: list of tuple\n        The fields to read\n    count: integer\n        The number of elements to count\n    '''\n    tr = {}\n    ds = subset.domain.ds\n    with FortranFile(fname) as fd:\n        # We do *all* conversion into boxlen here.\n        # This means that no other conversions need to be applied to convert\n        # positions into the same domain as the octs themselves.\n        for field in sorted(fields, key=lambda a: foffsets[a]):\n            if count == 0:\n                tr[field] = np.empty(0, dtype=data_types[field])\n                continue\n            fd.seek(foffsets[field])\n            dt = data_types[field]\n            tr[field] = fd.read_vector(dt)\n            if field[1].startswith(\"particle_position\"):\n                np.divide(tr[field], ds[\"boxlen\"], tr[field])\n            if ds.cosmological_simulation and field[1] == \"particle_birth_time\":\n                conformal_age = tr[field]\n                tr[field] = convert_ramses_ages(ds, conformal_age)\n                # arbitrarily set particles with zero conformal_age to zero\n                # particle_age. This corresponds to DM particles.\n                tr[field][conformal_age == 0] = 0\n    return tr\n\n\nclass IOHandlerRAMSES(BaseIOHandler):\n    _dataset_type = \"ramses\"\n\n    def _read_fluid_selection(self, chunks, selector, fields, size):\n        tr = defaultdict(list)\n\n        # Set of field types\n        ftypes = set(f[0] for f in fields)\n        for chunk in chunks:\n            # Gather fields by type to minimize i/o operations\n            for ft in ftypes:\n                # Get all the fields of the same type\n                field_subs = list(\n                    filter(lambda f: f[0]==ft, fields))\n\n                # Loop over subsets\n                for subset in chunk.objs:\n                    fname = None\n                    for fh in subset.domain.field_handlers:\n                        if fh.ftype == ft:\n                            file_handler = fh\n                            fname = fh.fname\n                            break\n\n                    if fname is None:\n                        raise YTFieldTypeNotFound(ft)\n\n                    # Now we read the entire thing\n                    with FortranFile(fname) as fd:\n                        # This contains the boundary information, so we skim through\n                        # and pick off the right vectors\n                        rv = subset.fill(fd, field_subs, selector, file_handler)\n                    for ft, f in field_subs:\n                        d = rv.pop(f)\n                        mylog.debug(\"Filling %s with %s (%0.3e %0.3e) (%s zones)\",\n                            f, d.size, d.min(), d.max(), d.size)\n                        tr[(ft, f)].append(d)\n        d = {}\n        for field in fields:\n            d[field] = np.concatenate(tr.pop(field))\n\n        return d\n\n    def _read_particle_coords(self, chunks, ptf):\n        pn = \"particle_position_%s\"\n        fields = [(ptype, \"particle_position_%s\" % ax)\n                  for ptype, field_list in ptf.items()\n                  for ax in 'xyz']\n        for chunk in chunks:\n            for subset in chunk.objs:\n                rv = self._read_particle_subset(subset, fields)\n                for ptype in sorted(ptf):\n                    yield ptype, (rv[ptype, pn % 'x'],\n                                  rv[ptype, pn % 'y'],\n                                  rv[ptype, pn % 'z'])\n\n    def _read_particle_fields(self, chunks, ptf, selector):\n        pn = \"particle_position_%s\"\n        chunks = list(chunks)\n        fields = [(ptype, fname) for ptype, field_list in ptf.items()\n                                 for fname in field_list]\n        for ptype, field_list in sorted(ptf.items()):\n            for ax in 'xyz':\n                if pn % ax not in field_list:\n                    fields.append((ptype, pn % ax))\n        for chunk in chunks:\n            for subset in chunk.objs:\n                rv = self._read_particle_subset(subset, fields)\n                for ptype, field_list in sorted(ptf.items()):\n                    x, y, z = (np.asarray(rv[ptype, pn % ax], \"=f8\")\n                               for ax in 'xyz')\n                    mask = selector.select_points(x, y, z, 0.0)\n                    if mask is None:\n                       mask = []\n                    for field in field_list:\n                        data = np.asarray(rv.pop((ptype, field))[mask], \"=f8\")\n                        yield (ptype, field), data\n\n\n    def _read_particle_subset(self, subset, fields):\n        '''Read the particle files.'''\n        tr = {}\n\n        # Sequential read depending on particle type\n        for ptype in set(f[0] for f in fields):\n\n            # Select relevant fiels\n            subs_fields = filter(lambda f: f[0] == ptype, fields)\n\n            ok = False\n            for ph in subset.domain.particle_handlers:\n                if ph.ptype == ptype:\n                    fname = ph.fname\n                    foffsets = ph.field_offsets\n                    data_types = ph.field_types\n                    ok = True\n                    count = ph.local_particle_count\n                    break\n            if not ok:\n                raise YTFieldTypeNotFound(ptype)\n\n            cosmo = self.ds.cosmological_simulation\n            if (ptype, 'particle_birth_time') in foffsets and cosmo:\n                foffsets[ptype, 'conformal_birth_time'] = \\\n                    foffsets[ptype, 'particle_birth_time']\n                data_types[ptype, 'conformal_birth_time'] = \\\n                    data_types[ptype, 'particle_birth_time']\n\n            tr.update(_ramses_particle_file_handler(\n                fname, foffsets, data_types, subset, subs_fields,\n                count=count\n            ))\n\n        return tr\n\n\ndef _read_part_file_descriptor(fname):\n    \"\"\"\n    Read a file descriptor and returns the array of the fields found.\n    \"\"\"\n    VERSION_RE = re.compile(r'# version: *(\\d+)')\n    VAR_DESC_RE = re.compile(r'\\s*(\\d+),\\s*(\\w+),\\s*(\\w+)')\n\n    # Mapping\n    mapping = [\n        ('position_x', 'particle_position_x'),\n        ('position_y', 'particle_position_y'),\n        ('position_z', 'particle_position_z'),\n        ('velocity_x', 'particle_velocity_x'),\n        ('velocity_y', 'particle_velocity_y'),\n        ('velocity_z', 'particle_velocity_z'),\n        ('mass', 'particle_mass'),\n        ('identity', 'particle_identity'),\n        ('levelp', 'particle_level'),\n        ('family', 'particle_family'),\n        ('tag', 'particle_tag')\n    ]\n    # Convert in dictionary\n    mapping = {k: v for k, v in mapping}\n\n    with open(fname, 'r') as f:\n        line = f.readline()\n        tmp = VERSION_RE.match(line)\n        mylog.debug('Reading part file descriptor %s.' % fname)\n        if not tmp:\n            raise YTParticleOutputFormatNotImplemented()\n\n        version = int(tmp.group(1))\n\n        if version == 1:\n            # Skip one line (containing the headers)\n            line = f.readline()\n            fields = []\n            for i, line in enumerate(f.readlines()):\n                tmp = VAR_DESC_RE.match(line)\n                if not tmp:\n                    raise YTFileNotParseable(fname, i+1)\n\n                # ivar = tmp.group(1)\n                varname = tmp.group(2)\n                dtype = tmp.group(3)\n\n                if varname in mapping:\n                    varname = mapping[varname]\n                else:\n                    varname = 'particle_%s' % varname\n\n                fields.append((varname, dtype))\n        else:\n            raise YTParticleOutputFormatNotImplemented()\n\n    return fields\n\ndef _read_fluid_file_descriptor(fname):\n    \"\"\"\n    Read a file descriptor and returns the array of the fields found.\n    \"\"\"\n    VERSION_RE = re.compile(r'# version: *(\\d+)')\n    VAR_DESC_RE = re.compile(r'\\s*(\\d+),\\s*(\\w+),\\s*(\\w+)')\n\n    # Mapping\n    mapping = [\n        ('density', 'Density'),\n        ('velocity_x', 'x-velocity'),\n        ('velocity_y', 'y-velocity'),\n        ('velocity_z', 'z-velocity'),\n        ('pressure', 'Pressure'),\n        ('metallicity', 'Metallicity'),\n    ]\n    # Convert in dictionary\n    mapping = {k: v for k, v in mapping}\n\n    with open(fname, 'r') as f:\n        line = f.readline()\n        tmp = VERSION_RE.match(line)\n        mylog.debug('Reading fluid file descriptor %s.' % fname)\n        if not tmp:\n            return []\n\n        version = int(tmp.group(1))\n\n        if version == 1:\n            # Skip one line (containing the headers)\n            line = f.readline()\n            fields = []\n            for i, line in enumerate(f.readlines()):\n                tmp = VAR_DESC_RE.match(line)\n                if not tmp:\n                    raise YTFileNotParseable(fname, i+1)\n\n                # ivar = tmp.group(1)\n                varname = tmp.group(2)\n                dtype = tmp.group(3)\n\n                if varname in mapping:\n                    varname = mapping[varname]\n                else:\n                    varname = 'hydro_%s' % varname\n\n                fields.append((varname, dtype))\n        else:\n            mylog.error('Version %s', version)\n            raise YTParticleOutputFormatNotImplemented()\n\n    return fields\n", "path": "yt/frontends/ramses/io.py"}]}