Open a Biocam hdf5 file, read and return the recording info, pick te correct method to access raw data, and return this to the caller.
507537
def openBiocamFile(filename, verbose=False):
    """"""
    rf = h5py.File(filename, 'r')
    # Read recording variables
    recVars = rf.require_group('3BRecInfo/3BRecVars/')
    # bitDepth = recVars['BitDepth'].value[0]
    # maxV = recVars['MaxVolt'].value[0]
    # minV = recVars['MinVolt'].value[0]
    nFrames = recVars['NRecFrames'][0]
    samplingRate = recVars['SamplingRate'][0]
    signalInv = recVars['SignalInversion'][0]
    # Read chip variables
    chipVars = rf.require_group('3BRecInfo/3BMeaChip/')
    nCols = chipVars['NCols'][0]
    # Get the actual number of channels used in the recording
    file_format = rf['3BData'].attrs.get('Version')
    if file_format == 100:
        nRecCh = len(rf['3BData/Raw'][0])
    elif file_format == 101:
        nRecCh = int(1. * rf['3BData/Raw'].shape[0] / nFrames)
    else:
        raise Exception('Unknown data file format.')

    if verbose:
        print('# 3Brain data format:', file_format, 'signal inversion', signalInv)
        print('#       signal range: ', recVars['MinVolt'][0], '- ', recVars['MaxVolt'][0])
        print('# channels: ', nRecCh)
        print('# frames: ', nFrames)
        print('# sampling rate: ', samplingRate)
    # get channel locations
    r = rf['3BRecInfo/3BMeaStreams/Raw/Chs'][()]['Row']
    c = rf['3BRecInfo/3BMeaStreams/Raw/Chs'][()]['Col']
    rawIndices = np.vstack((r, c)).T
    # assign channel numbers
    chIndices = np.array([(x - 1) + (y - 1) * nCols for (y, x) in rawIndices])
    # determine correct function to read data
    if verbose:
        print("# Signal inversion looks like " + str(signalInv) + ", guessing correct method for data access.")
        print("# If your results look wrong, signal polarity is may be wrong.")
    if file_format == 100:
        if signalInv == -1:
            read_function = readHDF5t_100
        else:
            read_function = readHDF5t_100_i
    else:
        if signalInv == -1:
            read_function = readHDF5t_101_i
        else:
            read_function = readHDF5t_101
    return (rf, nFrames, samplingRate, nRecCh, chIndices, file_format, signalInv, rawIndices, read_function)
206482
def _get_raw_movielens_data():
    

    path = _get_movielens_path()

    if not os.path.isfile(path):
        _download_movielens(path)

    with zipfile.ZipFile(path) as datafile:
        return (datafile.read('ml-100k/ua.base').decode().split('\n'),
                datafile.read('ml-100k/ua.test').decode().split('\n'))
1074455
def to_hdf5(self, filepath: str):
        
        try:
            self.start("Saving data to Hdf5...")
            dd.io.save(filepath, self.df)
            self.end("Finished saving Hdf5 data")
        except Exception as e:
            self.err(e, "Can not convert data to Hdf5")
103658
def vid2h5(data: np.ndarray, ut1, rawind, ticks, outfn: Path, P: dict, i: int=0,
           Nfile: int=1, det=None, tstart=None, cmdlog: str=None):
    assert outfn, 'must provide a filename to write'

    outfn = Path(outfn).expanduser()
    assert not outfn.is_dir(
    ), 'must provide FILEname, you provided directory {outfn}'
    outfn.parent.mkdir(exist_ok=True)

    txtupd = f'convert file {i} / {Nfile}'
    if 'spoolfn' in P:
        txtupd += f' {P["spoolfn"].name}'
    txtupd += f' => {outfn}'
    
    print(txtupd + '\r', end="")
    # %% saving
    if outfn.suffix == '.h5':
        """
        Reference: https://www.hdfgroup.org/HDF5/doc/ADGuide/ImageSpec.html
        Thanks to Eric Piel of Delmic for pointing out this spec
        * the HDF5 attributess set are necessary to put HDFView into image mode and enables
        other conforming readers to easily play images stacks as video.
        * the string_() calls are necessary to make fixed length strings per HDF5 spec
        """
        # NOTE write mode r+ to not overwrite incremental images
        if outfn.is_file():
            writemode = 'r+'  # append
        else:
            writemode = 'w'

        if data is not None:

            N = data.shape[0] * Nfile
            ind = slice(i * data.shape[0], (i + 1) * data.shape[0])
        else:  # FIXME haven't reevaluated for update only case
            for q in (ut1, rawind, ticks):
                if q is not None:
                    N = len(q)
                    break
            ind = slice(None)

        with h5py.File(outfn, writemode) as f:
            if data is not None:
                if 'rawimg' not in f:  # first run
                    setupimgh5(f, N, data.shape[1], data.shape[2])

                f['/rawimg'][ind, ...] = data

            if ut1 is not None:
                print(f'writing from {datetime.utcfromtimestamp(ut1[0])}'
                      f'to {datetime.utcfromtimestamp(ut1[-1])}')
                if 'ut1_unix' not in f:
                    fut1 = f.create_dataset(
                        '/ut1_unix', shape=(N,), dtype=float, fletcher32=True)
                    fut1.attrs['units'] = 'seconds since Unix epoch Jan 1 1970 midnight'

                f['/ut1_unix'][ind] = ut1

            if tstart is not None and 'tstart' not in f:
                f['/tstart'] = tstart

            if rawind is not None:
                if 'rawind' not in f:
                    fri = f.create_dataset(
                        '/rawind', shape=(N,), dtype=np.int64, fletcher32=True)
                    fri.attrs['units'] = 'one-based index since camera program started this session'

                f['/rawind'][ind] = rawind

            if ticks is not None:
                if 'ticks' not in f:
                    ftk = f.create_dataset(
                        '/ticks', shape=(N,), dtype=np.uint64, fletcher32=True)  # Uint64
                    ftk.attrs['units'] = 'FPGA tick counter for each image frame'

                f['/ticks'][ind] = ticks

            if 'spoolfn' in P:
                # http://docs.h5py.org/en/latest/strings.html
                if 'spoolfn' not in f:
                    fsp = f.create_dataset(
                        '/spoolfn', shape=(N,), dtype=h5py.special_dtype(vlen=bytes))
                    fsp.attrs['description'] = 'input filename data was extracted from'

                f['/spoolfn'][ind] = P['spoolfn'].name

            if det is not None:
                if 'detect' not in f:
                    fdt = f.create_dataset('/detect', shape=(N,), dtype=int)
                    fdt.attrs['description'] = '# of auroral detections this frame'

                f['/detect'][i] = det[i]

            if 'params' not in f:
                cparam = np.array((
                    P['kineticsec'],
                    P['rotccw'],
                    P['transpose'],
                    P['flipud'],
                    P['fliplr'],
                    1),
                    dtype=[('kineticsec', 'f8'),
                           ('rotccw', 'i1'),
                           ('transpose', 'i1'),
                           ('flipud', 'i1'),
                           ('fliplr', 'i1'),
                           ('questionable_ut1', 'i1')]
                )

                # cannot use fletcher32 here, typeerror
                f.create_dataset('/params', data=cparam)

            if 'sensorloc' not in f and 'sensorloc' in P:
                loc = P['sensorloc']
                lparam = np.array((loc[0], loc[1], loc[2]),
                                  dtype=[('lat', 'f8'), ('lon', 'f8'), ('alt_m', 'f8')])

                # cannot use fletcher32 here, typeerror
                Ld = f.create_dataset('/sensorloc', data=lparam)
                Ld.attrs['units'] = 'WGS-84 lat (deg),lon (deg), altitude (meters)'

            if 'cmdlog' not in f:
                if isinstance(cmdlog, (tuple, list)):
                    cmdlog = ' '.join(cmdlog)
                # cannot use fletcher32 here, typeerror
                f['/cmdlog'] = str(cmdlog)

            if 'header' not in f and 'header' in P:
                f['/header'] = str(P['header'])

            if 'hdf5version' not in f:
                f['/hdf5version'] = h5py.version.hdf5_version_tuple

    elif outfn.suffix == '.fits':
        from astropy.io import fits
        # NOTE the with... syntax does NOT yet work with astropy.io.fits
        hdu = fits.PrimaryHDU(data)
        hdu.writeto(outfn, clobber=False, checksum=True)
        # no close
        """
        Note: the orientation of this FITS in NASA FV program and the preview
        image shown in Python should/must have the same orientation and pixel indexing')
        """

    elif outfn.suffix == '.mat':
        from scipy.io import savemat
        # matlab is fortran order
        matdata = {'imgdata': data.transpose(1, 2, 0)}
        savemat(outfn, matdata, oned_as='column')
    else:
        raise ValueError(f'what kind of file is {outfn}')
373587
def _open_file(filename):
        
        if filename is None:
            raise DataSourceError("Trace filename is not defined")

        try:
            trace_file = open(filename, "r")
        except IOError as e:
            raise DataSourceError("Unable to open trace file %s" % filename, e)
        else:
            LOG.debug("Opened trace file %s", filename)
            return trace_file
330319
def to_hdf5(input):
        
    with performance.Monitor('to_hdf5') as mon:
        for input_file in input:
            if input_file.endswith('.npz'):
                output = convert_npz_hdf5(input_file, input_file[:-3] + 'hdf5')
            elif input_file.endswith('.xml'):  # for source model files
                output = convert_xml_hdf5(input_file, input_file[:-3] + 'hdf5')
            else:
                continue
            print('Generated %s' % output)
    print(mon)
461736
def open(self):
        

        if self.handle is None:
            self.handle = fits.open(self.fname, mode='readonly')

        if self.extn:
            if len(self.extn) == 1:
                hdu = self.handle[self.extn[0]]
            else:
                hdu = self.handle[self.extn[0],self.extn[1]]
        else:
            hdu = self.handle[0]
        if isinstance(hdu,fits.hdu.compressed.CompImageHDU):
            self.compress = True
        return hdu
656801
def open(self):
        """"""
        if self._fh:
            return  # file is open

        if isinstance(self._file, pathlib.Path):
            self._file = str(self._file)
        if isinstance(self._file, basestring):
            # file name
            self._file = os.path.realpath(self._file)
            self._dir, self._name = os.path.split(self._file)
            self._fh = open(self._file, self._mode)
            self._close = True
            if self._offset is None:
                self._offset = 0
        elif isinstance(self._file, FileHandle):
            # FileHandle
            self._fh = self._file._fh
            if self._offset is None:
                self._offset = 0
            self._offset += self._file._offset
            self._close = False
            if not self._name:
                if self._offset:
                    name, ext = os.path.splitext(self._file._name)
                    self._name = '%s@%i%s' % (name, self._offset, ext)
                else:
                    self._name = self._file._name
            if self._mode and self._mode != self._file._mode:
                raise ValueError('FileHandle has wrong mode')
            self._mode = self._file._mode
            self._dir = self._file._dir
        elif hasattr(self._file, 'seek'):
            # binary stream: open file, BytesIO
            try:
                self._file.tell()
            except Exception:
                raise ValueError('binary stream is not seekable')
            self._fh = self._file
            if self._offset is None:
                self._offset = self._file.tell()
            self._close = False
            if not self._name:
                try:
                    self._dir, self._name = os.path.split(self._fh.name)
                except AttributeError:
                    self._name = 'Unnamed binary stream'
            try:
                self._mode = self._fh.mode
            except AttributeError:
                pass
        else:
            raise ValueError('The first parameter must be a file name, '
                             'seekable binary stream, or FileHandle')

        if self._offset:
            self._fh.seek(self._offset)

        if self._size is None:
            pos = self._fh.tell()
            self._fh.seek(self._offset, 2)
            self._size = self._fh.tell()
            self._fh.seek(pos)

        try:
            self._fh.fileno()
            self.is_file = True
        except Exception:
            self.is_file = False
834379
def read_hdf(filename: PathLike, key: str) -> AnnData:
    
    with h5py.File(filename, 'r') as f:
        # the following is necessary in Python 3, because only
        # a view and not a list is returned
        keys = [k for k in f.keys()]
        if key == '':
            raise ValueError((
                'The file {} stores the following sheets:\n{}\n'
                'Call read/read_hdf5 with one of them.'
            ).format(filename, keys))
        # read array
        X = f[key][()]
        # try to find row and column names
        rows_cols = [{}, {}]
        for iname, name in enumerate(['row_names', 'col_names']):
            if name in keys:
                rows_cols[iname][name] = f[name][()]
    adata = AnnData(X, rows_cols[0], rows_cols[1], dtype=X.dtype.name)
    return adata
156689
def _get_raw_data(self, name):
        """"""
        # try legacy first, then hdf5
        filestem = ''
        for filestem, list_fvar in self._files.items():
            if name in list_fvar:
                break
        fieldfile = self.step.sdat.filename(filestem, self.step.isnap,
                                            force_legacy=True)
        if not fieldfile.is_file():
            fieldfile = self.step.sdat.filename(filestem, self.step.isnap)
        parsed_data = None
        if fieldfile.is_file():
            parsed_data = stagyyparsers.fields(fieldfile)
        elif self.step.sdat.hdf5 and self._filesh5:
            for filestem, list_fvar in self._filesh5.items():
                if name in list_fvar:
                    break
            parsed_data = stagyyparsers.read_field_h5(
                self.step.sdat.hdf5 / 'Data.xmf', filestem, self.step.isnap)
        return list_fvar, parsed_data
979644
def _loadData(self, data):
        
        self._data = data
        self.allowCameraUpload = cast(bool, data.attrib.get('allowCameraUpload'))
        self.allowChannelAccess = cast(bool, data.attrib.get('allowChannelAccess'))
        self.allowMediaDeletion = cast(bool, data.attrib.get('allowMediaDeletion'))
        self.allowSharing = cast(bool, data.attrib.get('allowSharing'))
        self.allowSync = cast(bool, data.attrib.get('allowSync'))
        self.backgroundProcessing = cast(bool, data.attrib.get('backgroundProcessing'))
        self.certificate = cast(bool, data.attrib.get('certificate'))
        self.companionProxy = cast(bool, data.attrib.get('companionProxy'))
        self.diagnostics = utils.toList(data.attrib.get('diagnostics'))
        self.eventStream = cast(bool, data.attrib.get('eventStream'))
        self.friendlyName = data.attrib.get('friendlyName')
        self.hubSearch = cast(bool, data.attrib.get('hubSearch'))
        self.machineIdentifier = data.attrib.get('machineIdentifier')
        self.multiuser = cast(bool, data.attrib.get('multiuser'))
        self.myPlex = cast(bool, data.attrib.get('myPlex'))
        self.myPlexMappingState = data.attrib.get('myPlexMappingState')
        self.myPlexSigninState = data.attrib.get('myPlexSigninState')
        self.myPlexSubscription = cast(bool, data.attrib.get('myPlexSubscription'))
        self.myPlexUsername = data.attrib.get('myPlexUsername')
        self.ownerFeatures = utils.toList(data.attrib.get('ownerFeatures'))
        self.photoAutoTag = cast(bool, data.attrib.get('photoAutoTag'))
        self.platform = data.attrib.get('platform')
        self.platformVersion = data.attrib.get('platformVersion')
        self.pluginHost = cast(bool, data.attrib.get('pluginHost'))
        self.readOnlyLibraries = cast(int, data.attrib.get('readOnlyLibraries'))
        self.requestParametersInCookie = cast(bool, data.attrib.get('requestParametersInCookie'))
        self.streamingBrainVersion = data.attrib.get('streamingBrainVersion')
        self.sync = cast(bool, data.attrib.get('sync'))
        self.transcoderActiveVideoSessions = int(data.attrib.get('transcoderActiveVideoSessions', 0))
        self.transcoderAudio = cast(bool, data.attrib.get('transcoderAudio'))
        self.transcoderLyrics = cast(bool, data.attrib.get('transcoderLyrics'))
        self.transcoderPhoto = cast(bool, data.attrib.get('transcoderPhoto'))
        self.transcoderSubtitles = cast(bool, data.attrib.get('transcoderSubtitles'))
        self.transcoderVideo = cast(bool, data.attrib.get('transcoderVideo'))
        self.transcoderVideoBitrates = utils.toList(data.attrib.get('transcoderVideoBitrates'))
        self.transcoderVideoQualities = utils.toList(data.attrib.get('transcoderVideoQualities'))
        self.transcoderVideoResolutions = utils.toList(data.attrib.get('transcoderVideoResolutions'))
        self.updatedAt = utils.toDatetime(data.attrib.get('updatedAt'))
        self.updater = cast(bool, data.attrib.get('updater'))
        self.version = data.attrib.get('version')
        self.voiceSearch = cast(bool, data.attrib.get('voiceSearch'))
495830
def with_read_hdf5(func):
    
    @wraps(func)
    def decorated_func(fobj, *args, **kwargs):
        # pylint: disable=missing-docstring
        if not isinstance(fobj, h5py.HLObject):
            if isinstance(fobj, FILE_LIKE):
                fobj = fobj.name
            with h5py.File(fobj, 'r') as h5f:
                return func(h5f, *args, **kwargs)
        return func(fobj, *args, **kwargs)

    return decorated_func
916646
def read_file(self):
        
        with open(self.filename, mode='r+', encoding='utf8') as text_file:
            self.raw_file = text_file.read()  # pylint: disable= attribute-defined-outside-init
        self.file_lines = [x.rstrip() for x in self.raw_file.splitlines()]
342851
def _load_h5(filename):
    ''''''
    from neurom.io import hdf5
    return hdf5.read(filename,
                     remove_duplicates=False,
                     data_wrapper=DataWrapper)
917737
def get_attrs(self, key):
        
        try:
            dset = h5py.File.__getitem__(self.hdf5, key)
        except KeyError:
            if self.parent != ():
                dset = h5py.File.__getitem__(self.parent.hdf5, key)
            else:
                raise
        return dict(dset.attrs)
841950
def decode(self,
             dataset_split=None,
             decode_from_file=False,
             checkpoint_path=None):
    """"""
    if decode_from_file:
      decoding.decode_from_file(self._estimator,
                                self._decode_hparams.decode_from_file,
                                self._hparams,
                                self._decode_hparams,
                                self._decode_hparams.decode_to_file)
    else:
      decoding.decode_from_dataset(
          self._estimator,
          self._hparams.problem.name,
          self._hparams,
          self._decode_hparams,
          dataset_split=dataset_split,
          checkpoint_path=checkpoint_path)
881835
def __ReadFromFile(self, datapath):
        
        def _create_meta(_datapath):
            """Just simply returns some dict. This functions exists in order to keep DRY"""
            meta = {
                'series_number': 0,
                'datadir': _datapath
            }
            return meta

        path, ext = os.path.splitext(datapath)
        ext = ext[1:]
        if ext in ('pklz', 'pkl'):
            logger.debug('pklz format detected')
            from . import misc
            data = misc.obj_from_file(datapath, filetype='pkl')
            data3d = data.pop('data3d')
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(data)

        elif ext in ['hdf5']:
            from . import hdf5_io
            datap = hdf5_io.load_dict_from_hdf5(datapath)
            # datap = self.read_hdf5(datapath)
            data3d = datap.pop('data3d')

            # back compatibility
            if 'metadata' in datap.keys():
                datap = datap['metadata']
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(datap)

        elif ext in ['idx']:
            from . import idxformat
            idxreader = idxformat.IDXReader()
            data3d, metadata = idxreader.read(datapath)
        elif ext in ['dcm', 'DCM', 'dicom']:
            data3d, metadata = self._read_with_sitk(datapath)
            metadata = self._fix_sitk_bug(datapath, metadata)
        elif ext in ["bz2"]:
            new_datapath = tgz.untar(datapath)
            data3d, metadata = self.__ReadFromDirectory(new_datapath)
        else:
            logger.debug('file format "' + str(ext) + '"')
            # reading raw file
            data3d, metadata = self._read_with_sitk(datapath)
        return data3d, metadata