Read hdf5 file from Cell Ranger v3 or later versions.
511246
def _read_v3_10x_h5(filename):
    
    with tables.open_file(str(filename), 'r') as f:
        try:
            dsets = {}
            for node in f.walk_nodes('/matrix', 'Array'):
                dsets[node.name] = node.read()
            from scipy.sparse import csr_matrix
            M, N = dsets['shape']
            data = dsets['data']
            if dsets['data'].dtype == np.dtype('int32'):
                data = dsets['data'].view('float32')
                data[:] = dsets['data']
            matrix = csr_matrix((data, dsets['indices'], dsets['indptr']),
                                shape=(N, M))
            adata = AnnData(matrix,
                            {'obs_names': dsets['barcodes'].astype(str)},
                            {'var_names': dsets['name'].astype(str),
                             'gene_ids': dsets['id'].astype(str),
                             'feature_types': dsets['feature_type'].astype(str),
                             'genome': dsets['genome'].astype(str)})
            logg.info(t=True)
            return adata
        except KeyError:
            raise Exception('File is missing one or more required datasets.')
104560
def dict_from_hdf5(dict_like, h5group):
    
    # Read attributes
    for name, value in h5group.attrs.items():
        dict_like[name] = value
800685
def hdf5(self):
        """"""
        if self._rundir['hdf5'] is UNDETERMINED:
            h5_folder = self.path / self.par['ioin']['hdf5_output_folder']
            if (h5_folder / 'Data.xmf').is_file():
                self._rundir['hdf5'] = h5_folder
            else:
                self._rundir['hdf5'] = None
        return self._rundir['hdf5']
359558
def _read_header(filename):
    
    header = _read_header_text(filename)
    first_row = header[0]
    EXTRA_ROWS = 3  # drop DefaultValue1 LowRange1 HighRange1

    hdr = {}
    for group in finditer('(\w*)= ([\w.]*)', first_row):
        hdr[group.group(1)] = group.group(2)

    if first_row.startswith('BCI2000V'):
        VERSION = hdr['BCI2000V']

    else:
        VERSION = '1'
        hdr['DataFormat'] = 'int16'

    for row in header[1:]:
        if row.startswith('['):  # remove '[ ... Definition ]'
            section = row[2:-14].replace(' ', '')

            if section == 'StateVector':
                hdr[section] = []
            else:
                hdr[section] = {} # defaultdict(dict)
            continue

        if row == '':
            continue

        elif section == 'StateVector':
            statevector = {key: value for key, value in list(zip(STATEVECTOR, row.split(' ')))}
            hdr[section].append(statevector)

        else:
            group = match('(?P<subsection>[\w:%]*) (?P<format>\w*) (?P<key>\w*)= (?P<value>.*) // ', row)
            onerow = group.groupdict()

            values = onerow['value'].split(' ')
            if len(values) > EXTRA_ROWS:
                value = ' '.join(onerow['value'].split(' ')[:-EXTRA_ROWS])
            else:
                value = ' '.join(values)

            hdr[section][onerow['key']] = value  # similar to matlab's output

    return hdr
207657
def from_hdf5(cls, f):
        
        if isinstance(f, str):
            import h5py
            f = h5py.File(f)

        pos = quantity_from_hdf5(f['pos'])
        vel = quantity_from_hdf5(f['vel'])

        frame = None
        if 'frame' in f:
            g = f['frame']

            frame_mod = g.attrs['module']
            frame_cls = g.attrs['class']
            frame_units = [u.Unit(x.decode('utf-8')) for x in g['units']]

            if u.dimensionless_unscaled in frame_units:
                units = DimensionlessUnitSystem()
            else:
                units = UnitSystem(*frame_units)

            pars = dict()
            for k in g['parameters']:
                pars[k] = quantity_from_hdf5(g['parameters/'+k])

            exec("from {0} import {1}".format(frame_mod, frame_cls))
            frame_cls = eval(frame_cls)

            frame = frame_cls(units=units, **pars)

        return cls(pos=pos, vel=vel, frame=frame)
404178
def loadhdf(filename, groupname="data", to_dataframe=False):
    
    data = {}
    with _h5py.File(filename, "r") as f:
        for key, value in f[groupname].items():
            data[key] = np.array(value)
    if to_dataframe:
        return _pd.DataFrame(data)
    else:
        return data
566021
def read_geom_h5(xdmf_file, snapshot):
    
    header = {}
    xdmf_root = xmlET.parse(str(xdmf_file)).getroot()
    if snapshot is None:
        return None, xdmf_root

    # Domain, Temporal Collection, Snapshot
    # should check that this is indeed the required snapshot
    elt_snap = xdmf_root[0][0][snapshot]
    header['ti_ad'] = float(elt_snap.find('Time').get('Value'))
    header['mo_lambda'] = _maybe_get(elt_snap, 'mo_lambda', 'Value', float)
    header['mo_thick_sol'] = _maybe_get(elt_snap, 'mo_thick_sol', 'Value',
                                        float)
    header['ntb'] = 1
    coord_h5 = []  # all the coordinate files
    coord_shape = []  # shape of meshes
    twod = None
    for elt_subdomain in elt_snap.findall('Grid'):
        if elt_subdomain.get('Name').startswith('meshYang'):
            header['ntb'] = 2
            break  # iterate only through meshYin
        elt_geom = elt_subdomain.find('Geometry')
        if elt_geom.get('Type') == 'X_Y' and twod is None:
            twod = ''
            for data_item in elt_geom.findall('DataItem'):
                coord = data_item.text.strip()[-1]
                if coord in 'XYZ':
                    twod += coord
        data_item = elt_geom.find('DataItem')
        coord_shape.append(_get_dim(data_item))
        coord_h5.append(
            xdmf_file.parent / data_item.text.strip().split(':/', 1)[0])
    _read_coord_h5(coord_h5, coord_shape, header, twod)
    return header, xdmf_root
359067
def load(self, cellpy_file, parent_level="CellpyData"):
        

        try:
            self.logger.debug("loading cellpy-file (hdf5):")
            self.logger.debug(cellpy_file)
            new_datasets = self._load_hdf5(cellpy_file, parent_level)
            self.logger.debug("cellpy-file loaded")
        except AttributeError:
            new_datasets = []
            self.logger.warning("This cellpy-file version is not supported by"
                                "current reader (try to update cellpy).")

        if new_datasets:
            for dataset in new_datasets:
                self.datasets.append(dataset)
        else:
            # raise LoadError
            self.logger.warning("Could not load")
            self.logger.warning(str(cellpy_file))

        self.number_of_datasets = len(self.datasets)
        self.status_datasets = self._validate_datasets()
        self._invent_a_name(cellpy_file)
        return self
731428
def read_data( self, step, filename = None ):
        """"""
        filename = get_default( filename, self.filename )

        out = {}

        fd = open( self.filename, 'r' )
        while 1:
            line = skip_read_line(fd, no_eof=True).split()
            if line[0] == 'POINT_DATA':
                break

        n_nod = int(line[1])

        while 1:
            line = skip_read_line(fd)
            if not line:
                break

            line = line.split()

            if line[0] == 'SCALARS':
                name, dtype, nc = line[1:]
                assert_(int(nc) == 1)
                fd.readline() # skip lookup table line
                
                data = nm.zeros((n_nod,), dtype=nm.float64)
                ii = 0
                while ii < n_nod:
                    data[ii] = float(fd.readline())
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = data,
                                    dofs = None )

            elif line[0] == 'VECTORS':
                name, dtype = line[1:]
                data = []
                ii = 0
                while ii < n_nod:
                    data.append([float(val) for val in fd.readline().split()])
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = nm.array(data, dtype=nm.float64),
                                    dofs = None )

            elif line[0] == 'CELL_DATA':
                break

            line = fd.readline()

        fd.close()

        return out
612026
def write_hdf5_cells(E, filename):
    ''' '''

    E = E.tocsr()

    hf = h5py.File(filename, 'w')
    counts_group = hf.create_group('counts')
    gix_group = hf.create_group('gene_ix')

    hf.attrs['ncells'] = E.shape[0]
    hf.attrs['ngenes'] = E.shape[1]

    for iC in range(E.shape[0]):
        counts = E[iC,:].A.squeeze()
        gene_ix = np.nonzero(counts)[0]
        counts = counts[gene_ix]
        counts_group.create_dataset(str(iC), data = counts)
        gix_group.create_dataset(str(iC), data = gene_ix)

    hf.close()
1256964
def to_hdf5(self, f):
        

        f = super(Orbit, self).to_hdf5(f)

        if self.potential is not None:
            import yaml
            from ..potential.potential.io import to_dict
            f['potential'] = yaml.dump(to_dict(self.potential)).encode('utf-8')

        if self.t:
            quantity_to_hdf5(f, 'time', self.t)

        return f
1368251
def setupimgh5(f: Union[Path, h5py.File],
               Nframetotal: int, Nrow: int, Ncol: int, dtype=np.uint16,
               writemode='r+', key='/rawimg', cmdlog: str=None):
    
    if isinstance(f, (str, Path)):  # assume new HDF5 file wanted
        f = Path(f).expanduser()
        if f.is_dir():
            raise IOError(
                'must provide specific HDF5 filename, not just a directory')

        print('creating', f)
        with h5py.File(f, writemode) as F:
            setupimgh5(F, Nframetotal, Nrow, Ncol, dtype, writemode, key)

    elif isinstance(f, h5py.File):
        h = f.create_dataset(key,
                             shape=(Nframetotal, Nrow, Ncol),
                             dtype=dtype,
                             chunks=(1, Nrow, Ncol),  # each image is a chunk
                             compression='gzip',
                             # no difference in filesize from 1 to 5, except much faster to use lower numbers!
                             compression_opts=1,
                             shuffle=True,
                             fletcher32=True,
                             track_times=True)
        h.attrs["CLASS"] = np.string_("IMAGE")
        h.attrs["IMAGE_VERSION"] = np.string_("1.2")
        h.attrs["IMAGE_SUBCLASS"] = np.string_("IMAGE_GRAYSCALE")
        h.attrs["DISPLAY_ORIGIN"] = np.string_("LL")
        h.attrs['IMAGE_WHITE_IS_ZERO'] = np.uint8(0)

        if cmdlog and isinstance(cmdlog, str):
            f['/cmdlog'] = cmdlog
    else:
        raise TypeError(
            f'{type(f)} is not correct, must be filename or h5py.File HDF5 file handle')
1338147
def read_array(fd, n_row, n_col, dtype):
    
    if n_col is None:
        idx = fd.tell()
        row = fd.readline().split()
        fd.seek(idx)
        n_col = len(row)

    count = n_row * n_col
    val = nm.fromfile(fd, sep=' ', count=count)

    if val.shape[0] < count:
        raise ValueError('(%d, %d) array reading failed!' % (n_row, n_col))

    val = nm.asarray(val, dtype=dtype)
    val.shape = (n_row, n_col)

    return val
1108459
def readBerSANS(filename):
    """"""
    hed = {'Comment': ''}
    translate = {'Lambda': 'Wavelength',
                 'Title': 'Owner',
                 'SampleName': 'Title',
                 'BeamcenterX': 'BeamPosY',
                 'BeamcenterY': 'BeamPosX',
                 'Time': 'MeasTime',
                 'TotalTime': 'MeasTime',
                 'Moni1': 'Monitor',
                 'Moni2': 'Monitor',
                 'Moni': 'Monitor',
                 'Transmission': 'Transm',
                 }
    with open(filename, 'rt') as f:
        comment_next = False
        for l in f:
            l = l.strip()
            if comment_next:
                hed['Comment'] = hed['Comment'] + '\n' + l
                comment_next = False
            elif l.startswith('%Counts'):
                break
            elif l.startswith('%Comment'):
                comment_next = True
            elif l.startswith('%'):
                continue
            elif l.split('=', 1)[0] in translate:
                hed[translate[l.split('=', 1)[0]]] = misc.parse_number(
                    l.split('=', 1)[1])
            else:
                try:
                    hed[l.split('=', 1)[0]] = misc.parse_number(
                        l.split('=', 1)[1])
                except IndexError:
                    print(l.split('=', 1))
    if 'FileName' in hed:
        m = re.match('D(\d+)\.(\d+)', hed['FileName'])
        if m is not None:
            hed['FSN'] = int(m.groups()[0])
            hed['suffix'] = int(m.groups()[1])
    if 'FileDate' in hed:
        hed['Date'] = dateutil.parser.parse(hed['FileDate'])
    if 'FileTime' in hed:
        hed['Date'] = datetime.datetime.combine(
            hed['Date'].date(), dateutil.parser.parse(hed['FileTime']).time())
    hed['__Origin__'] = 'BerSANS'
    if 'SD' in hed:
        hed['Dist'] = hed['SD'] * 1000
    if hed['Comment'].startswith('\n'):
        hed['Comment'] = hed['Comment'][1:]
    hed['__particle__'] = 'neutron'
    hed['Wavelength'] *= 10  # convert from nanometres to Angstroems
    return hed
1123350
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, maskMap):
        
        # Assign file extension attribute to file object
        self.fileExtension = extension

        if isinstance(maskMap, RasterMapFile) and maskMap.fileExtension == 'msk':
            # Vars from mask map
            columns = maskMap.columns
            rows = maskMap.rows
            upperLeftX = maskMap.west
            upperLeftY = maskMap.north

            # Derive the cell size (GSSHA cells are square, so it is the same in both directions)
            cellSizeX = int(abs(maskMap.west - maskMap.east) / columns)
            cellSizeY = -1 * cellSizeX

            # Dictionary of keywords/cards and parse function names
            KEYWORDS = {'DATASET': wdc.datasetHeaderChunk,
                        'TS': wdc.datasetScalarTimeStepChunk}

            # Open file and read plain text into text field
            with open(path, 'r') as f:
                chunks = pt.chunk(KEYWORDS, f)

            # Parse header chunk first
            header = wdc.datasetHeaderChunk('DATASET', chunks['DATASET'][0])

            # Parse each time step chunk and aggregate
            timeStepRasters = []

            for chunk in chunks['TS']:
                timeStepRasters.append(wdc.datasetScalarTimeStepChunk(chunk, columns, header['numberCells']))

            # Set WMS dataset file properties
            self.name = header['name']
            self.numberCells = header['numberCells']
            self.numberData = header['numberData']
            self.objectID = header['objectID']

            if header['type'] == 'BEGSCL':
                self.objectType = header['objectType']
                self.type = self.SCALAR_TYPE

            elif header['type'] == 'BEGVEC':
                self.vectorType = header['objectType']
                self.type = self.VECTOR_TYPE

            # Create WMS raster dataset files for each raster
            for timeStep, timeStepRaster in enumerate(timeStepRasters):
                # Create new WMS raster dataset file object
                wmsRasterDatasetFile = WMSDatasetRaster()

                # Set the wms dataset for this WMS raster dataset file
                wmsRasterDatasetFile.wmsDataset = self

                # Set the time step and timestamp and other properties
                wmsRasterDatasetFile.iStatus = timeStepRaster['iStatus']
                wmsRasterDatasetFile.timestamp = timeStepRaster['timestamp']
                wmsRasterDatasetFile.timeStep = timeStep + 1

                # If spatial is enabled create PostGIS rasters
                if spatial:
                    # Process the values/cell array
                    wmsRasterDatasetFile.raster = RasterLoader.makeSingleBandWKBRaster(session,
                                                                                       columns, rows,
                                                                                       upperLeftX, upperLeftY,
                                                                                       cellSizeX, cellSizeY,
                                                                                       0, 0,
                                                                                       spatialReferenceID,
                                                                                       timeStepRaster['cellArray'])

                # Otherwise, set the raster text properties
                else:
                    wmsRasterDatasetFile.rasterText = timeStepRaster['rasterText']

            # Add current file object to the session
            session.add(self)

        else:
            log.warning("Could not read {0}. Mask Map must be supplied "
                     "to read WMS Datasets.".format(filename))
176180
def from_hdf5(cls, f):
        
        # TODO: this is duplicated code from PhaseSpacePosition
        if isinstance(f, str):
            import h5py
            f = h5py.File(f)

        pos = quantity_from_hdf5(f['pos'])
        vel = quantity_from_hdf5(f['vel'])

        time = None
        if 'time' in f:
            time = quantity_from_hdf5(f['time'])

        frame = None
        if 'frame' in f:
            g = f['frame']

            frame_mod = g.attrs['module']
            frame_cls = g.attrs['class']
            frame_units = [u.Unit(x.decode('utf-8')) for x in g['units']]

            if u.dimensionless_unscaled in frame_units:
                units = DimensionlessUnitSystem()
            else:
                units = UnitSystem(*frame_units)

            pars = dict()
            for k in g['parameters']:
                pars[k] = quantity_from_hdf5(g['parameters/'+k])

            exec("from {0} import {1}".format(frame_mod, frame_cls))
            frame_cls = eval(frame_cls)

            frame = frame_cls(units=units, **pars)

        potential = None
        if 'potential' in f:
            import yaml
            from ..potential.potential.io import from_dict
            _dict = yaml.load(f['potential'][()].decode('utf-8'))
            potential = from_dict(_dict)

        return cls(pos=pos, vel=vel, t=time,
                   frame=frame, potential=potential)
975505
def read_hdf5_dict(h5f, names=None, path=None, on_missing='error', **kwargs):
    
    if path:
        h5f = h5f[path]

    # allow alternative keyword argument name (FIXME)
    if names is None:
        names = kwargs.pop('flags', None)

    # try and get list of names automatically
    if names is None:
        try:
            names = find_flag_groups(h5f, strict=True)
        except KeyError:
            names = None
        if not names:
            raise ValueError("Failed to automatically parse available flag "
                             "names from HDF5, please give a list of names "
                             "to read via the ``names=`` keyword")

    # read data
    out = DataQualityDict()
    for name in names:
        try:
            out[name] = read_hdf5_flag(h5f, name, **kwargs)
        except KeyError as exc:
            if on_missing == 'ignore':
                pass
            elif on_missing == 'warn':
                warnings.warn(str(exc))
            else:
                raise ValueError('no H5Group found for flag '
                                 '{0!r}'.format(name))

    return out