Reads variable star data in `GCVS format`_.

    :param filename: path to GCVS data file (usually ``iii.dat``)

    .. _`GCVS format`: http://www.sai.msu.su/gcvs/gcvs/iii/html/
16464
def read_gcvs(filename):
    
    with open(filename, 'r') as fp:
        parser = GcvsParser(fp)
        for star in parser:
            yield star
253910
def load_hdf(cls, filename, path=''): #perhaps this doesn't need to be written?
        

        new = StarPopulation.load_hdf(filename, path=path)

        #setup lazy loading of starmodel if present
        try:
            with pd.HDFStore(filename) as store:
                if '{}/starmodel'.format(path) in store:
                    new._starmodel = None
                    new._starmodel_file = filename
                    new._starmodel_path = '{}/starmodel'.format(path)
        except:
            pass

        try:
            new._make_kde()
        except NoTrapfitError:
            logging.warning('Trapezoid fit not done.')
        return new
841518
def load_data(self):
        
        file_objects = self.file_objects
        var = self.variable
        valid_date = self.valid_dates
        data = self.data
        unknown_names = self.unknown_names
        unknown_units = self.unknown_units
        member = self.member
        lat = self.lat
        lon = self.lon
      
        if self.sector_ind_path:
            inds_file = pd.read_csv(self.sector_ind_path+'sector_data_indices.csv') 
            inds = inds_file.loc[:,'indices']  
        out_x = self.mapping_data["x"]
        
        if not file_objects:
            print()
            print("No {0} model runs on {1}".format(member,self.run_date))
            print()
            units = None
            return self.data, units

    
        for f, file in enumerate(file_objects):
            grib = pygrib.open(file)
            if type(var) is int:
                data_values = grib[var].values
                #lat, lon = grib[var].latlons()
                #proj = Proj(grib[var].projparams)
                if grib[var].units == 'unknown':
                    Id = grib[var].parameterNumber
                    units = self.unknown_units[Id] 
                else:
                    units = grib[var].units
            elif type(var) is str:
                if '_' in var:
                    variable = var.split('_')[0]
                    level = int(var.split('_')[1])
                    if variable in unknown_names.values():
                        Id, units = self.format_grib_name(variable)
                        data_values = grib.select(parameterNumber=Id, level=level)[0].values
                        #lat, lon =  grib.select(parameterNumber=Id, level=level)[0].latlons()
                        #proj = Proj(grib.select(parameterNumber=Id, level=level)[0].projparams)

                    else:
                        data_values = grib.select(name=variable, level=level)[0].values
                        units = grib.select(name=variable, level=level)[0].units
                        #lat, lon  = grib.select(name=variable, level=level)[0].latlons()
                        #proj = Proj(grib.select(name=variable, level=level)[0].projparams)
                else:
                    if var in unknown_names.values():
                        Id, units = self.format_grib_name(var)
                        data_values = grib.select(parameterNumber=Id)[0].values
                        #lat, lon = grib.select(parameterNumber=Id)[0].latlons() 
                        #proj = Proj(grib.select(parameterNumber=Id)[0].projparams)

                    elif len(grib.select(name=var)) > 1:
                        raise NameError("Multiple '{0}' records found. Rename with level:'{0}_level'".format(var))

                    else:
                        data_values = grib.select(name=var)[0].values
                        units = grib.select(name=var)[0].units
                        #lat, lon = grib.select(name=var)[0].latlons()
                        #proj = Proj(grib.select(name=var)[0].projparams)

            if data is None:
                data = np.empty((len(valid_date), out_x.shape[0], out_x.shape[1]), dtype=float)
                if self.sector_ind_path:
                    data[f] = data_values[:].flatten()[inds].reshape(out_x.shape)
                else:
                    data[f]=data_values[:]
            else:
                if self.sector_ind_path:
                    data[f] = data_values[:].flatten()[inds].reshape(out_x.shape)
                else:
                    data[f]=data_values[:]
        
        return data, units
588179
def read_csv_file(self, file_name):
        
        result = []
        with open(os.path.join(self.__path(), os.path.basename(file_name)),
                  'rt') as csvfile:
            headers_reader = csv.reader(csvfile, delimiter=',', quotechar='|')
            for type_row in headers_reader:
                for t in type_row:
                    result.append(t)
        return result
66122
def load_data(data_file, key_name = None, local_file = False, format = 'json'):
    
    if local_file:
        if data_file.startswith('/'):
            src_file = data_file
        else:
            src_dir = os.getcwd()
            src_file = os.path.join(src_dir, data_file)
    else:
        src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
        if not os.path.isdir(src_dir):
            src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../data')
        src_file = os.path.join(src_dir, data_file)
    with open(src_file) as f:
        if format == 'json':
            data = json.load(f)
        elif format == 'yaml':
            data = yaml.load(f)
        elif format not in ['json', 'yaml'] and not key_name:
            data = f.read()
        else:
            printError('Error, argument \'key_name\' may not be used with data in %s format.' % format)
            return None
    if key_name:
        data = data[key_name]
    return data
788124
def load_data_file(self, fname, filemode='a'):
        
        self.close_data()
        self.datafile = open_acqdata(fname, filemode=filemode)

        self.explorer.set(datafile=self.datafile)
        self.protocoler.set(datafile=self.datafile)
        self.charter.set(datafile=self.datafile)
        self.bs_calibrator.set(datafile=self.datafile)
        self.tone_calibrator.set(datafile=self.datafile)
        self.set_calibration(None)

        self.current_cellid = dict(self.datafile.get_info('')).get('total cells', 0)
1269561
def read_file(filename):
    
    sanitized_filename = sanitize(filename)

    f = open(sanitized_filename)

    specs = None

    try:
        specs = _parse_tsplib(f)
    except ParseException:
        raise
    finally: # 'finally' is executed even when we re-raise exceptions
        f.close()

    if specs['TYPE'] != 'CVRP':
        raise Exception('Not a CVRP TSPLIB problem. Found: {}'.format(specs['TYPE']))
    
    #additional params for graph/network (temporary)
    specs['VOLTAGE'] = 20000
    specs['CABLETYPE'] = 1

    #return (Graph(specs), specs)
    return Graph(specs)
861696
def load_csv(filename, **kw):
    '''
    
    '''
    import pandas
    if any(filename.endswith(s) for s in ('.gz', '.bz2', '.lzma')):
        with gzip.open(filename, 'rt') as fl: data = pandas.read_csv(fl, **kw)
    else:
        with open(filename, 'rt') as fl: data = pandas.read_csv(fl, **kw)
    return data
39790
def read_csv(filename, keys=None, convert_types=False, **kwargs):
    '''
    

    '''
    df = pd.read_csv(filename)
    if convert_types:
        df = convert_types_slow(df)
    if keys:
        df = df[df['key'].isin(keys)]
    df = process_one(df)
    return df
1015734
def read_sla(self,filename,params=None,force=False,timerange=None,datatype=None,**kwargs):
        
        
        
        from time import time
        from datetime import timedelta
#        a = time()
        
        self.message(2,'Reading AVISO DT data ({0})'.format(datatype))
        
                #Open file
        self._filename = filename
        try:
            self._ncfile = ncfile(self._filename, "r")
        except Exception,e:
            self.warning(1, repr(e))
            return {}
        
        #Get delimiter
        if os.path.basename(filename).count('.') > os.path.basename(filename).count('_'): delim='.'
        else : delim = '_'
        
        #Gat sat name
        splitted=os.path.basename(filename).split(delim)
        if len(splitted) > 3 :
            if (datatype == 'DT') | (datatype == 'NRT') : sat_name = splitted[2] if splitted[0] == 'nrt' else splitted[3]
            elif datatype == 'PISTACH' : sat_name = 'J2'
            else : sat_name = 'J2'
        else :
            sat_name="N/A"
        
     
        #Get list of recorded parameters:
        par_list=[i.encode() for i in self._ncfile.variables.keys()]
        for i in ['BeginDates','Longitudes','Latitudes'] : par_list.pop(par_list.index(i))
        nparam=len(par_list)
        
        self.message(2,'Recorded parameters : '+str(nparam)+' -> '+str(par_list))
     
        lon = self.load_ncVar('Longitudes',**kwargs)
        lon['data'] = recale(lon['data'], degrees=True, zero_2pi=True) #shift longitudes
        lat = self.load_ncVar('Latitudes',**kwargs)
        
#        lon = self.load_ncVar('Longitudes',Longitudes=(self.limit[1],self.limit[3]),**kwargs)
#        lon['data'] = recale(lon['data'], degrees=True, zero_2pi=True) #shift longitudes
#        lat = self.load_ncVar('Latitudes',Latitudes=(self.limit[0],self.limit[1]),**kwargs)

        
        #Extract within limits
        ind, flag = in_limits(lon['data'],lat['data'],limit=self.limit)
        dim_lon = lon['_dimensions']
        lat = lat['data'].compress(flag)
        lon = lon['data'].compress(flag)
#         dist=cumulative_distance(lat, lon)
        
        sz=np.shape(lon)
        ndims=np.size(sz)
        
        #Get dates
        stDate = self.load_ncVar('BeginDates',**kwargs)['data']
        dumVar = self.load_ncVar('Cycles',**kwargs)
        nbCyc = dumVar['data']
        Ncycs = dumVar['_dimensions']['Cycles']
        Ntra = dumVar['_dimensions']['Tracks']
        nbTra = self.load_ncVar('Tracks',**kwargs)['data']
        
#        if np.size(stDate) == 1 : stDate.
        
        DeltaT = self._ncfile.variables['DeltaT'][:]  / 86400. #* self._ncfile.variables['DeltaT'].scale_factor
        npts = self.load_ncVar('NbPoints',**kwargs)['data']
        dumind=np.cumsum(npts)
        
        #Loop 1
#        date=np.ma.array([],mask=[])
#        cycles=np.ma.array([],mask=[])
#        tracks=np.ma.array([],mask=[])
#        for i in xrange(Ncycs) :
#            np.ma.concatenate((nbTra,nbTra))
#        
#        for i,nc in enumerate(nbCyc.data.flatten()):
#            N=npts[i]
#            curInd=np.array(list(set(xrange(dumind[i]-N,dumind[i]) if N > 0 else []).intersection(ind)))
#            ncur=len(curInd)
#            date=np.ma.concatenate((date,(curInd - dumind[0])*DeltaT+stDate.flatten()[i]))
#            cycles=np.ma.concatenate((cycles,np.ma.array((nbCyc.data.flatten()[i],)*ncur)))#,mask=np.repeat(nbCyc.mask[i][j],ncur))))
#            tracks=np.ma.concatenate((tracks,np.ma.array((nbTra.data.flatten()[i],)*ncur)))
        
        #Loop 2
        
        date = ()
        cycles = ()
        tracks = ()
        
#        rowind = (0,)*Ntra
        
#        nind=0
#        for i in xrange(Ncycs): nind+=(npts*(~nbCyc.mask.T[i])).sum()
        
        indcopy=ind.copy()
#        npts_copy=npts.copy()
        npts[npts.mask]=0
        dumind[dumind.mask]=0
        
        nbTra_copy=nbTra.copy()
        
        toto=npts.copy()
        
        concat_npts = not( nbCyc.shape[-1] > 1)
        
        #loop over cycles
        for i in np.arange(1,Ncycs,1.0,dtype=int) :
            nbTra=np.ma.concatenate((nbTra,nbTra_copy))
            if concat_npts : npts=np.ma.concatenate((npts,tuple((~nbCyc.T[i].mask)*1*npts)))
#            rowind+=(i,)*Ntra      
        
        if concat_npts: npts=npts.reshape(nbCyc.shape[::-1]).T
        else : npts=nbCyc
        
        nbTra=nbTra.reshape(nbCyc.shape[::-1]).T
#        rowind=np.reshape(rowind,nbCyc.shape[::-1]).T
        
#        npts.mask=nbCyc.mask
        nbTra.mask=nbCyc.mask
        
        npts=npts.flatten()
        nbTra=nbTra.flatten()
#        rowind=rowind.flatten()
        
        nbCyc_flatten=nbCyc.flatten()
        nbTra_flatten=nbTra.flatten()
        stDate_flatten=stDate.flatten()
        
#        nind=0
        
        outInd=[]
        
        for i,nc in enumerate(nbCyc.data.flatten()):
            N=npts[i]
            Nprev=npts[i-Ncycs] if i >= (Ncycs) and np.remainder(float(i),Ncycs) == 0 else 0
            indcopy-=Nprev #if rowind[i] == 0 else 0
            curInd=tuple(sorted(set(xrange(N) if N > 0 else []).intersection(indcopy)))
            ncur=len(curInd)
#            nind+=ncur
            outInd+=map(operator.sub, curInd,(( (curInd[0] if len(curInd) > 0 else 0) - (outInd[-1] +1 if len(outInd) > 0 else 0) - len(ind)*(np.remainder(float(i),Ncycs)),)*ncur))
            curInd=tuple(map(operator.mul, curInd, (DeltaT,)*ncur))     
            date+=tuple(map(operator.add, curInd, (stDate_flatten[i],)*ncur))
            cycles+=(nbCyc_flatten[i],)*ncur
            tracks+=(nbTra_flatten[i],)*ncur
        
        date=np.ma.masked_array(date,mask=False)
        cycles=np.ma.masked_array(cycles,mask=False)
        tracks=np.ma.masked_array(tracks,mask=False)
                
        #Loop 3
#        date=np.ma.array([],mask=[])
#        cycles=np.ma.array([],mask=[])
#        tracks=np.ma.array([],mask=[])
#        for j in xrange(Ncycs) :
#            for i,N in enumerate(npts.data) :
##                curFg=(ind >= dumind[i]-N) & (ind <= dumind[i])
#                curInd=np.array(list(set(xrange(dumind[i]-N,dumind[i]) if N > 0 else []).intersection(ind)))
#                ncur=len(curInd)
#                date=np.ma.concatenate((date,(curInd - dumind[0])*DeltaT+stDate[i][j]))
#                cycles=np.ma.concatenate((cycles,np.ma.array((nbCyc.data[i][j],)*ncur)))#,mask=np.repeat(nbCyc.mask[i][j],ncur))))
#                tracks=np.ma.concatenate((tracks,np.ma.array((nbTra.data[i],)*ncur)))#,mask=np.repeat(nbCyc.mask[i][j],ncur))))
        
        outInd=np.array(outInd,dtype=int)
        
        #Check output index
#         print outInd.shape[0],npts.cumsum().max()
        
        nt=len(date)
        date.mask=(False,)*nt
        cycles.mask=date.mask
        tracks.mask=date.mask
#        date=date.reshape((Ncycs,)+(npts.sum(),)).T
#        mask=date.mask
#        date=date.compressed()
#        cycles=cycles.reshape((Ncycs,)+(npts.sum(),)).T.compressed()
#        tracks=tracks.reshape((Ncycs,)+(npts.sum(),)).T.compressed()
        
        
#        lon=np.repeat(lon,Ncycs)
#        lat=np.repeat(lat,Ncycs)
#        mask=~lon.mask
        
        dimStr = dim_lon
        dimStr.pop('Data')
        nrec=len(date)
        dimStr.update({'time':nrec})
        
        for i in ['DeltaT','NbPoints','Cycles','Tracks','DataIndexes'] : par_list.pop(par_list.index(i))
        
        outStr={'_dimensions':dimStr,
                'lon':lon,
                'lat':lat,
                'date':date,
                'cycle':cycles,
                'track':tracks}
        
        
        
        for param in par_list :
            a = time()
            dumVar = self.load_ncVar(param,Data=ind,**kwargs) #Load variables            
            runtime = time() - a
#            print 'runtime:', timedelta(seconds=runtime)
            
            dimStr=dumVar['_dimensions']
            dimStr.pop('Cycles')
            dimStr.pop('Data')
            dimStr['time']=nrec
            dimStr['_ndims']=len(dimStr.keys())-1
            
            #update dimensions
            curDim = [str(dimname) for dimname in dimStr.keys()[1:]] #[str(dimname) for dimname in self._ncfile.variables['LONGITUDE'].dimensions]
            curDimval = [dimStr[dim] for dim in curDim] #[len(self._ncfile.dimensions[dimname]) for dimname in curDim]
            flag = [(np.array(dimname) == outStr['_dimensions'].keys()).sum() == 0 for dimname in curDim] #find dimensions to update
            dimUpdate = np.array(curDim).compress(flag)
            for enum in enumerate(dimUpdate) : 
                self.message(3, 'Appending dimensions {0}:{1} to dataStructure'.format(enum[1],np.array(curDimval).compress(flag)[enum[0]]))
                outStr['_dimensions'].update({enum[1]:np.array(curDimval).compress(flag)[enum[0]]}) #Append new dimension
                outStr['_dimensions']['_ndims']+=1 #update dimension counts
            
#            dumStr = {param.lower() : dumVar['data']}
            dumStr = {param.lower() : dumVar['data'].flatten()[outInd]}
#            cmd = 'dumStr = {\''+param.lower()+'\':dumVar[\'data\']}'
#            self.message(4, 'exec : '+cmd)
#            exec(cmd)
            outStr.update(dumStr)
        
        id=np.repeat(sat_name,outStr['_dimensions']['time'])
        
        
        outStr.update({'id':id})
        self._ncfile.close()
        
        #Checkit [len(outStr[k]) for k in outStr.keys()]
        return outStr
427226
def read(fname):
    '''
    
    '''
    file_path = os.path.join(SETUP_DIRNAME, fname)
    with codecs.open(file_path, encoding='utf-8') as rfh:
        return rfh.read()
571758
def read(self, filename, rowprefix=None, colprefix=None, delim=":"):
        """"""

        self.matrix = scipy.io.mmread(filename)
        with open(filename + ".rownames") as in_handle:
            self.rownames = [x.strip() for x in in_handle]
            if rowprefix:
                self.rownames = [rowprefix + delim + x for x in self.rownames]
        with open(filename + ".colnames") as in_handle:
            self.colnames = [x.strip() for x in in_handle]
            if colprefix:
                self.colnames = [colprefix + delim + x for x in self.colnames]
1296771
def read(filename):
    
    # The format is specified at
    # <http://www.intes.de>.

    cells = {}
    meshio_to_permas_type = {
        "vertex": (1, "PLOT1"),
        "line": (2, "PLOTL2"),
        "triangle": (3, "TRIA3"),
        "quad": (4, "QUAD4"),
        "tetra": (4, "TET4"),
        "hexahedron": (8, "HEXE8"),
        "wedge": (6, "PENTA6"),
        "pyramid": (5, "PYRA5"),
    }

    opener = gzip.open if filename.endswith(".gz") else open

    with opener(filename, "rb") as f:
        while True:
            line = f.readline().decode("utf-8")
            if not line or re.search("\\$END STRUCTURE", line):
                break
            for meshio_type, permas_ele in meshio_to_permas_type.items():
                num_nodes = permas_ele[0]
                permas_type = permas_ele[1]

                if re.search("\\$ELEMENT TYPE = {}".format(permas_type), line):
                    while True:
                        line = f.readline().decode("utf-8")
                        if not line or line.startswith("!"):
                            break
                        data = numpy.array(line.split(), dtype=int)
                        if meshio_type in cells:
                            cells[meshio_type].append(data[-num_nodes:])
                        else:
                            cells[meshio_type] = [data[-num_nodes:]]

            if re.search("\\$COOR", line):
                points = []
                while True:
                    line = f.readline().decode("utf-8")
                    if not line or line.startswith("!"):
                        break
                    for r in numpy.array(line.split(), dtype=float)[1:]:
                        points.append(r)

    points = numpy.array(points)
    points = numpy.reshape(points, newshape=(len(points) // 3, 3))
    for key in cells:
        # Subtract one to account for the fact that python indices
        # are 0-based.
        cells[key] = numpy.array(cells[key], dtype=int) - 1

    return Mesh(points, cells)
1273291
def _readFile(self, sldir, fileName, sep):
        '''
        

        '''

        if sldir.endswith(os.sep):
            fileName = str(sldir)+str(fileName)
        else:
            fileName = str(sldir)+os.sep+str(fileName)


        fileLines=[] #list of lines in the file
        header=[]    #list of Header lines
        dataCols=[]  #Dictionary of data column names
        data=[]      #List of Data lists
        cols=[]      #List of column names

        f=open(fileName,'r')
        fileLines=f.readlines()
        i=0
        if self.datatype != 'trajectory':

            while i<len(fileLines):
                if fileLines[i].startswith(self.header_char):
                    tmp=fileLines[i].lstrip(self.header_char)
                    header.append(tmp.strip())
                else:
                    break
                i+=1

            cols=fileLines[i].split(sep)

            tmp=[]
            tmp1=[]
            for j in range(len(cols)):
                tmp1=cols[j].strip()
                if tmp1 !='':
                    tmp.append(tmp1)
            cols=tmp
            i+=1
        else:
            header={}
            while fileLines[i].startswith('#') or '=' in fileLines[i]:
                if fileLines[i].startswith('#') and cols==[]:
                    cols=fileLines[i].strip('#')
                    cols=cols.strip()
                    cols=cols.split()
                elif fileLines[i].startswith('#'):
                    tmp1=fileLines[i].strip('#')
                    tmp1=tmp1.strip()
                    self.headerLines.append(tmp1)
                elif not fileLines[i].startswith('#'):
                    tmp=fileLines[i].split('=')
                    tmp[0]=tmp[0].strip()
                    tmp[1]=tmp[1].strip()
                    if header=={}:
                        header={str(tmp[0]):str(tmp[1])}
                    else:
                        header[str(tmp[0])]=str(tmp[1])
                i+=1
        while i<len(fileLines):
            if fileLines[i].startswith('#'):
                i=i+1
            else:
                tmp=fileLines[i].split()
                for j in range(len(tmp)):
                    tmp[j]=tmp[j].strip()
                data.append(tmp)
                i+=1
        tmp=[]
        tmp1=[]
        for j in range(len(data)):
            for k in range(len(data[j])):
                tmp1=data[j][k].strip()
                if tmp1 !='':
                    tmp.append(tmp1)
            data[j]=tmp
            tmp=[]
        tmp=[]

        for j in range(len(cols)):
            for k in range(len(data)):
                try:
                    a=float(data[k][j])
                    tmp.append(a)
                except ValueError:
                    tmp.append(data[k][j])
                #else:
                 #   tmp.append(float(data[k][j])) # previously tmp.append(float(data[k][j]))
            tmp=array(tmp)

            if j == 0:
                dataCols={cols[j]:tmp}
            else:
                dataCols[cols[j]]=tmp
            tmp=[]

        return header,dataCols
905890
def readFromFile(self, filename):
        '''
        
        '''
        s = dict(np.load(filename))
        try:
            self.coeffs = s['coeffs'][()]
        except KeyError:
            #LEGENCY - remove
            self.coeffs = s
        try:
            self.opts = s['opts'][()]
        except KeyError:
            pass
        return self.coeffs
953377
def _read(self, filename):
        """"""
        data = {}
        parsers = [
            FirstDataParser(), CoordinateParser(), EnergyGradParser(),
            SkipApproxHessian(), HessianParser(), MassParser(),
        ]
        with open(filename) as f:
            while True:
                line = f.readline()
                if line == "":
                    break
                # at each line, a parsers checks if it has to process a piece of
                # file. If that happens, the parser gets control over the file
                # and reads as many lines as it needs to collect data for some
                # attributes.
                for parser in parsers:
                    if parser.test(line, data):
                        parser.read(line, f, data)
                        break
        self.__dict__.update(data)
1219693
def get_multisample_vcf(fnames, name, caller, data):
    
    unique_fnames = []
    for f in fnames:
        if f not in unique_fnames:
            unique_fnames.append(f)
    out_dir = utils.safe_makedir(os.path.join(data["dirs"]["work"], "gemini"))
    if len(unique_fnames) > 1:
        gemini_vcf = os.path.join(out_dir, "%s-%s.vcf.gz" % (name, caller))
        vrn_file_batch = None
        for variant in data.get("variants", []):
            if variant["variantcaller"] == caller and variant.get("vrn_file_batch"):
                vrn_file_batch = variant["vrn_file_batch"]
        if vrn_file_batch:
            utils.symlink_plus(vrn_file_batch, gemini_vcf)
            return gemini_vcf
        else:
            return vcfutils.merge_variant_files(unique_fnames, gemini_vcf, dd.get_ref_file(data),
                                                data["config"])
    else:
        gemini_vcf = os.path.join(out_dir, "%s-%s%s" % (name, caller, utils.splitext_plus(unique_fnames[0])[1]))
        utils.symlink_plus(unique_fnames[0], gemini_vcf)
        return gemini_vcf