Import a .mat file with single potentials (a b m) into a pandas
    DataFrame

    Also export some variables of the MD struct into a separate structure
10514
def _read_mat_mnu0(filename):
    
    print('read_mag_single_file: {0}'.format(filename))

    mat = sio.loadmat(filename, squeeze_me=True)
    # check the version
    version = mat['MP']['Version'].item()
    if version != 'FZJ-EZ-2017':
        raise Exception(
            'This data format is not supported (expected: FZJ-EZ-2017)' +
            ' got: {}'.format(version)
        )

    df_emd = _extract_emd(mat, filename=filename)
    df_md = _extract_md(mat)

    return df_emd, df_md
89289
def load_data_to_net(net, inst_net):
  ''' '''
  net.dat['nodes'] = inst_net['nodes']
  net.dat['mat'] = inst_net['mat']
  data_formats.mat_to_numpy_arr(net)
1130575
def get_structure_from_dict(d):
    

    species = []
    frac_coords = []
    masses = []
    if 'points' in d:
        for p in d['points']:
            species.append(p['symbol'])
            frac_coords.append(p['coordinates'])
            masses.append(p['mass'])
    elif 'atoms' in d:
        for p in d['atoms']:
            species.append(p['symbol'])
            frac_coords.append(p['position'])
            masses.append(p['mass'])
    else:
        raise ValueError('The dict does not contain structural information')

    return Structure(d['lattice'], species, frac_coords,
                     site_properties={"phonopy_masses": masses})
1266881
def from_df(cls, df:DataFrame, path:PathOrStr, cols:IntsOrStrs=0, folder:PathOrStr=None, suffix:str='', **kwargs)->'ItemList':
        ""
        suffix = suffix or ''
        res = super().from_df(df, path=path, cols=cols, **kwargs)
        pref = f'{res.path}{os.path.sep}'
        if folder is not None: pref += f'{folder}{os.path.sep}'
        res.items = np.char.add(np.char.add(pref, res.items.astype(str)), suffix)
        return res
651975
def convert_coco_stuff_mat(data_dir, out_dir):
    """"""
    sets = ['train', 'val']
    categories = []
    json_name = 'coco_stuff_%s.json'
    ann_dict = {}
    for data_set in sets:
        file_list = os.path.join(data_dir, '%s.txt')
        images = []
        with open(file_list % data_set) as f:
            for img_id, img_name in enumerate(f):
                img_name = img_name.replace('coco', 'COCO').strip('\n')
                image = {}
                mat_file = os.path.join(
                    data_dir, 'annotations/%s.mat' % img_name)
                data = h5py.File(mat_file, 'r')
                labelMap = data.get('S')
                if len(categories) == 0:
                    labelNames = data.get('names')
                    for idx, n in enumerate(labelNames):
                        categories.append(
                            {"id": idx, "name": ''.join(chr(i) for i in data[
                                n[0]])})
                    ann_dict['categories'] = categories
                scipy.misc.imsave(
                    os.path.join(data_dir, img_name + '.png'), labelMap)
                image['width'] = labelMap.shape[0]
                image['height'] = labelMap.shape[1]
                image['file_name'] = img_name
                image['seg_file_name'] = img_name
                image['id'] = img_id
                images.append(image)
        ann_dict['images'] = images
        print("Num images: %s" % len(images))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))
218028
def as_struct_array(*columns):
    
    columns = [np.asarray(c) for c in columns]
    rows = len(columns[0])

    names = ['f'+str(i) for i in range(len(columns))]
    dtype = [(names[i], c.dtype, c.shape[1:]) for i, c in enumerate(columns)]
    data = np.empty(rows, dtype)
    for i, c in enumerate(columns):
        data[names[i]] = c
    return data
172678
def pack(cls, data):
        ''''''
        return struct.pack('>ll', len(data) + 4, cls.FRAME_TYPE) + data
705231
def arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
    
    # figure out the index, if necessary
    if index is None:
        index = extract_index(arrays)
    else:
        index = ensure_index(index)

    # don't force copy because getting jammed in an ndarray anyway
    arrays = _homogenize(arrays, index, dtype)

    # from BlockManager perspective
    axes = [ensure_index(columns), index]

    return create_block_manager_from_arrays(arrays, arr_names, axes)
717785
def from_dict(cls, d):
        
        cohp_dict = {}
        efermi = d["efermi"]
        energies = d["energies"]
        structure = Structure.from_dict(d["structure"])
        if "bonds" in d.keys():
            bonds = {bond: {"length": d["bonds"][bond]["length"],
                            "sites": tuple(PeriodicSite.from_dict(site)
                                           for site in d["bonds"][bond]["sites"])}
                     for bond in d["bonds"]}
        else:
            bonds = None
        for label in d["COHP"]:
            cohp = {Spin(int(spin)): np.array(d["COHP"][label][spin])
                    for spin in d["COHP"][label]}
            try:
                icohp = {Spin(int(spin)): np.array(d["ICOHP"][label][spin])
                         for spin in d["ICOHP"][label]}
            except KeyError:
                icohp = None
            if label == "average":
                avg_cohp = Cohp(efermi, energies, cohp, icohp=icohp)
            else:
                cohp_dict[label] = Cohp(efermi, energies, cohp, icohp=icohp)

        if "orb_res_cohp" in d.keys():
            orb_cohp = {}
            for label in d["orb_res_cohp"]:
                orb_cohp[label] = {}
                for orb in d["orb_res_cohp"][label]:
                    cohp = {Spin(int(s)): np.array(
                        d["orb_res_cohp"][label][orb]["COHP"][s],
                        dtype=float)
                        for s in d["orb_res_cohp"][label][orb]["COHP"]}
                    try:
                        icohp = {Spin(int(s)): np.array(
                            d["orb_res_cohp"][label][orb]["ICOHP"][s],
                            dtype=float)
                            for s in d["orb_res_cohp"][label][orb]["ICOHP"]}
                    except KeyError:
                        icohp = None
                    orbitals = [tuple((int(o[0]), Orbital[o[1]])) for o in
                                d["orb_res_cohp"][label][orb]["orbitals"]]
                    orb_cohp[label][orb] = {"COHP": cohp, "ICOHP": icohp,
                                            "orbitals": orbitals}
                # If no total COHPs are present, calculate the total
                # COHPs from the single-orbital populations. Total COHPs
                # may not be present when the cohpgenerator keyword is used
                # in LOBSTER versions 2.2.0 and earlier.
                if label not in d["COHP"] or d["COHP"][label] is None:
                    cohp = {Spin.up: np.sum(np.array(
                        [orb_cohp[label][orb]["COHP"][Spin.up]
                         for orb in orb_cohp[label]]), axis=0)}
                    try:
                        cohp[Spin.down] = np.sum(np.array(
                            [orb_cohp[label][orb]["COHP"][Spin.down]
                             for orb in orb_cohp[label]]), axis=0)
                    except KeyError:
                        pass

                orb_res_icohp = None in [orb_cohp[label][orb]["ICOHP"]
                                         for orb in orb_cohp[label]]
                if (label not in d["ICOHP"] or
                    d["ICOHP"][label] is None) and orb_res_icohp:
                    icohp = {Spin.up: np.sum(np.array(
                        [orb_cohp[label][orb]["ICOHP"][Spin.up]
                         for orb in orb_cohp[label]]), axis=0)}
                    try:
                        icohp[Spin.down] = np.sum(np.array(
                            [orb_cohp[label][orb]["ICOHP"][Spin.down]
                             for orb in orb_cohp[label]]), axis=0)
                    except KeyError:
                        pass
        else:
            orb_cohp = None

        if "average" not in d["COHP"].keys():
            # calculate average
            cohp = np.array([np.array(c)
                             for c in d["COHP"].values()]).mean(axis=0)
            try:
                icohp = np.array([np.array(c)
                                  for c in d["ICOHP"].values()]).mean(axis=0)
            except KeyError:
                icohp = None
            avg_cohp = Cohp(efermi, energies, cohp, icohp=icohp)

        return CompleteCohp(structure, avg_cohp, cohp_dict, bonds=bonds,
                            are_coops=d["are_coops"], orb_res_cohp=orb_cohp)
482126
def objify(self, doc, columns=None):
        
        cols = columns or doc[METADATA][COLUMNS]
        data = {}

        for col in cols:
            # if there is missing data in a chunk, we can default to NaN
            # and pandas will autofill the missing values to the correct length
            if col not in doc[METADATA][LENGTHS]:
                d = [np.nan]
            else:
                d = decompress(doc[DATA][doc[METADATA][LENGTHS][col][0]: doc[METADATA][LENGTHS][col][1] + 1])
                # d is ready-only but that's not an issue since DataFrame will copy the data anyway.
                d = np.frombuffer(d, doc[METADATA][DTYPE][col])

                if MASK in doc[METADATA] and col in doc[METADATA][MASK]:
                    mask_data = decompress(doc[METADATA][MASK][col])
                    mask = np.frombuffer(mask_data, 'bool')
                    d = ma.masked_array(d, mask)
            data[col] = d

        # Copy into
        return pd.DataFrame(data, columns=cols, copy=True)[cols]
793840
def load_data(filename):
    
    data = pandas.read_csv(filename, header=None, delimiter='\t', skiprows=9)
    return data.as_matrix()
713873
def multi_index_df_to_component_dfs(multi_index_df, rid="rid", cid="cid"):
    

    # Id level of the multiindex will become the index
    rids = list(multi_index_df.index.get_level_values(rid))
    cids = list(multi_index_df.columns.get_level_values(cid))

    # It's possible that the index and/or columns of multi_index_df are not
    # actually multi-index; need to check for this and there are more than one level in index(python3)
    if isinstance(multi_index_df.index, pd.MultiIndex):

        # check if there are more than one levels in index (python3)
        if len(multi_index_df.index.names) > 1:

            # If so, drop rid because it won't go into the body of the metadata
            mi_df_index = multi_index_df.index.droplevel(rid)

            # Names of the multiindex levels become the headers
            rhds = list(mi_df_index.names)

            # Assemble metadata values
            row_metadata = np.array([mi_df_index.get_level_values(level).values for level in list(rhds)]).T

        # if there is one level in index (python3), then rhds and row metadata should be empty
        else:
            rhds = []
            row_metadata = []

    # If the index is not multi-index, then rhds and row metadata should be empty
    else:
        rhds = []
        row_metadata = []

    # Check if columns of multi_index_df are in fact multi-index
    if isinstance(multi_index_df.columns, pd.MultiIndex):

        # Check if there are more than one levels in columns(python3)
        if len(multi_index_df.columns.names) > 1:

            # If so, drop cid because it won't go into the body of the metadata
            mi_df_columns = multi_index_df.columns.droplevel(cid)

            # Names of the multiindex levels become the headers
            chds = list(mi_df_columns.names)

            # Assemble metadata values
            col_metadata = np.array([mi_df_columns.get_level_values(level).values for level in list(chds)]).T

        # If there is one level in columns (python3), then rhds and row metadata should be empty
        else:
            chds = []
            col_metadata = []
    # If the columns are not multi-index, then rhds and row metadata should be empty
    else:
        chds = []
        col_metadata = []

    # Create component dfs
    row_metadata_df = pd.DataFrame.from_records(row_metadata, index=pd.Index(rids, name="rid"), columns=pd.Index(rhds, name="rhd"))
    col_metadata_df = pd.DataFrame.from_records(col_metadata, index=pd.Index(cids, name="cid"), columns=pd.Index(chds, name="chd"))
    data_df = pd.DataFrame(multi_index_df.values, index=pd.Index(rids, name="rid"), columns=pd.Index(cids, name="cid"))

    return data_df, row_metadata_df, col_metadata_df
1135256
def _init_from_npy2d(self, mat, missing):
        
        if len(mat.shape) != 2:
            raise ValueError('Input numpy.ndarray must be 2 dimensional')
        data = np.array(mat.reshape(mat.size), dtype=np.float32)
        self.handle = ctypes.c_void_p()
        _check_call(_LIB.XGDMatrixCreateFromMat(data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
                                                mat.shape[0], mat.shape[1],
                                                ctypes.c_float(missing),
                                                ctypes.byref(self.handle)))
994719
def _construct_from_dataframe(self):
        
        direct_edges = {}
        feature_side_grouping = self.feature_pathway_df.groupby(
            ["feature", "side"])
        for (feature, _), group in feature_side_grouping:
            co_occurring_pathways = group["pathway"].tolist()
            pairings = list(itertools.combinations(co_occurring_pathways, 2))
            if not pairings:
                continue
            self.features.add(feature)
            for pathway0, pathway1 in pairings:
                vertex0_id = self.add_pathway(pathway0)
                vertex1_id = self.add_pathway(pathway1)
                new_edge = self.edge_tuple(vertex0_id, vertex1_id)
                if new_edge not in direct_edges:
                    direct_edges[new_edge] = []
                direct_edges[new_edge].append(feature)
        self._augment_network(direct_edges)
652344
def as_df():
    """"""
    records = []
    for sym, ele in vars(_this).items():
        if sym not in ["Element", "Isotope"] and not sym.startswith("_"):
            for k, v in vars(ele).items():
                if k.startswith("_") and k[1].isdigit():
                    records.append({kk: vv for kk, vv in vars(v).items() if not kk.startswith("_")})
    return _DF.from_records(records)
840889
def __init_from_np2d(self, mat, params_str, ref_dataset):
        """"""
        if len(mat.shape) != 2:
            raise ValueError('Input numpy.ndarray must be 2 dimensional')

        self.handle = ctypes.c_void_p()
        if mat.dtype == np.float32 or mat.dtype == np.float64:
            data = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
        else:
            # change non-float data to float data, need to copy
            data = np.array(mat.reshape(mat.size), dtype=np.float32)

        ptr_data, type_ptr_data, _ = c_float_array(data)
        _safe_call(_LIB.LGBM_DatasetCreateFromMat(
            ptr_data,
            ctypes.c_int(type_ptr_data),
            ctypes.c_int(mat.shape[0]),
            ctypes.c_int(mat.shape[1]),
            ctypes.c_int(C_API_IS_ROW_MAJOR),
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
        return self
773460
def FixmatFactory(fixmatfile, categories = None, var_name = 'fixmat', field_name='x'):
    
    try:
        data = loadmat(fixmatfile, struct_as_record = False)
        keys = list(data.keys())
        data = data[var_name][0][0]
    except KeyError:
        raise RuntimeError('%s is not a field of the matlab structure. Possible'+
                'Keys are %s'%str(keys))
    
    num_fix = data.__getattribute__(field_name).size

    # Get a list with fieldnames and a list with parameters
    fields = {}
    parameters = {}
    for field in data._fieldnames:
        if data.__getattribute__(field).size == num_fix:
            fields[field] = data.__getattribute__(field)
        else:            
            parameters[field] = data.__getattribute__(field)[0].tolist()
            if len(parameters[field]) == 1:
                parameters[field] = parameters[field][0]
    
    # Generate FixMat
    fixmat = FixMat(categories = categories)
    fixmat._fields = list(fields.keys())
    for (field, value) in list(fields.items()):
        fixmat.__dict__[field] = value.reshape(-1,) 

    fixmat._parameters = parameters
    fixmat._subjects = None
    for (field, value) in list(parameters.items()):
        fixmat.__dict__[field] = value
    fixmat._num_fix = num_fix
    return fixmat