Flatten compound columns (DataFrames within columns) into top level columns
6745
def normalize(df):
    """"""
    # in ipython notebook, explore and describe the DataFrame columns
    print('Of the {} columns, {} are actually DataFrames'.format(len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))
    # remove dataframes with only 2 columns and one is the _str of the other:
    for col in df.columns:
        if isinstance(df[col], pd.DataFrame):
            print('Column {} is a {}-wide DataFrame'.format(col, len(df[col].columns)))
            if df[col].columns[1] == df[col].columns[0] + '_str':
                print('Column {} looks easy because it has sub-columns {}'.format(col, df[col].columns))
                df[col] = df[col][df[col].columns[1]]
            else:
                try:
                    assert(float(df[col].iloc[:, 0].max()) == float(df[col].iloc[:, 1].max()))
                    df[col] = df[col].fillna(-1, inplace=False)
                    series = pd.Series([int(Decimal(x)) for x in df[col].iloc[:, 1].values]).astype('int64').copy()
                    del df[col]
                    df[col] = series
                    print('Finished converting column {} to type {}({})'.format(col, type(df[col]), df[col].dtype))
                except:
                    print_exc()

    print('Of the {} columns, {} are still DataFrames after trying to convert both columns to long integers'.format(
        len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))
    return df
117095
def column_stack_2d(data):
    """"""
    return list(list(itt.chain.from_iterable(_)) for _ in zip(*data))
79970
def _data_from_df(df):
        ''' 

        '''
        _df = df.copy()

        # Flatten columns
        if isinstance(df.columns, pd.MultiIndex):
            try:
                _df.columns = ['_'.join(col) for col in _df.columns.values]
            except TypeError:
                raise TypeError('Could not flatten MultiIndex columns. '
                                'use string column names or flatten manually')
        # Transform columns CategoricalIndex in list
        if isinstance(df.columns, pd.CategoricalIndex):
            _df.columns = df.columns.tolist()
        # Flatten index
        index_name = ColumnDataSource._df_index_name(df)
        if index_name == 'index':
            _df.index = pd.Index(_df.index.values)
        else:
            _df.index = pd.Index(_df.index.values, name=index_name)
        _df.reset_index(inplace=True)

        tmp_data = {c: v.values for c, v in _df.iteritems()}

        new_data = {}
        for k, v in tmp_data.items():
            new_data[k] = v

        return new_data
1141718
def sorted_feed_cols(df):
    
    cols = df.columns
    ind = [int(c.split("feed")[1]) for c in cols]
    cols = zip(ind,cols)
    cols.sort()
    cols = [c[1] for c in cols]
    return cols
1098602
def transform_dataframe(self, dataframe):
        
        coord_fields = self.get_coord_fields()
        header_fields = self.get_header_fields()

        # Remove any pairs that don't have data for both x & y
        for i in range(len(coord_fields)):
            dataframe = dataframe.unstack()
        dataframe = dataframe.dropna(axis=1, how='all')
        dataframe = dataframe.dropna(axis=0, how='any')

        # Unstack series header
        for i in range(len(header_fields)):
            dataframe = dataframe.unstack()

        # Compute new column headers
        columns = []
        for i in range(len(header_fields) + 1):
            columns.append([])

        for col in dataframe.columns:
            value_name = col[0]
            coord_names = list(col[1:len(coord_fields) + 1])
            header_names = list(col[len(coord_fields) + 1:])
            coord_name = ''
            for name in coord_names:
                if name != self.index_none_value:
                    coord_name += name + '-'
            coord_name += value_name
            columns[0].append(coord_name)
            for i, header_name in enumerate(header_names):
                columns[1 + i].append(header_name)

        dataframe.columns = columns
        dataframe.columns.names = [''] + header_fields

        return dataframe
459371
def merge_indexes(
        indexes,  # type: Dict[Any, Union[Any, List[Any]]]
        variables,  # type: Dict[Any, Variable]
        coord_names,  # type: Set
        append=False,  # type: bool
):
    # type: (...) -> Tuple[OrderedDict[Any, Variable], Set]
    
    vars_to_replace = {}  # Dict[Any, Variable]
    vars_to_remove = []  # type: list

    for dim, var_names in indexes.items():
        if isinstance(var_names, str):
            var_names = [var_names]

        names, codes, levels = [], [], []  # type: (list, list, list)
        current_index_variable = variables.get(dim)

        for n in var_names:
            var = variables[n]
            if (current_index_variable is not None and
                    var.dims != current_index_variable.dims):
                raise ValueError(
                    "dimension mismatch between %r %s and %r %s"
                    % (dim, current_index_variable.dims, n, var.dims))

        if current_index_variable is not None and append:
            current_index = current_index_variable.to_index()
            if isinstance(current_index, pd.MultiIndex):
                try:
                    current_codes = current_index.codes
                except AttributeError:
                    # fpr pandas<0.24
                    current_codes = current_index.labels
                names.extend(current_index.names)
                codes.extend(current_codes)
                levels.extend(current_index.levels)
            else:
                names.append('%s_level_0' % dim)
                cat = pd.Categorical(current_index.values, ordered=True)
                codes.append(cat.codes)
                levels.append(cat.categories)

        if not len(names) and len(var_names) == 1:
            idx = pd.Index(variables[var_names[0]].values)

        else:
            for n in var_names:
                names.append(n)
                var = variables[n]
                cat = pd.Categorical(var.values, ordered=True)
                codes.append(cat.codes)
                levels.append(cat.categories)

            idx = pd.MultiIndex(levels, codes, names=names)

        vars_to_replace[dim] = IndexVariable(dim, idx)
        vars_to_remove.extend(var_names)

    new_variables = OrderedDict([(k, v) for k, v in variables.items()
                                 if k not in vars_to_remove])
    new_variables.update(vars_to_replace)
    new_coord_names = coord_names | set(vars_to_replace)
    new_coord_names -= set(vars_to_remove)

    return new_variables, new_coord_names
1062377
def flatten(in_list):
    """"""
    out_list = []
    for val in in_list:
        if isinstance(val, list):
            out_list.extend(val)
        else:
            out_list.append(val)

    return out_list
1203617
def merge(cls, tables, fillna=False):
        """"""
        cols = set(itertools.chain(*[table.dtype.descr for table in tables]))

        tables_to_merge = []
        for table in tables:
            missing_cols = cols - set(table.dtype.descr)

            if missing_cols:
                if fillna:
                    n = len(table)
                    n_cols = len(missing_cols)
                    col_names = []
                    for col_name, col_dtype in missing_cols:
                        if 'f' not in col_dtype:
                            raise ValueError(
                                "Cannot create NaNs for non-float"
                                " type column '{}'".format(col_name)
                            )
                        col_names.append(col_name)

                    table = table.append_columns(
                        col_names, np.full((n_cols, n), np.nan)
                    )
                else:
                    raise ValueError(
                        "Table columns do not match. Use fill_na=True"
                        " if you want to append missing values with NaNs"
                    )
            tables_to_merge.append(table)

        first_table = tables_to_merge[0]

        merged_table = sum(tables_to_merge[1:], first_table)

        merged_table.h5loc = first_table.h5loc
        merged_table.h5singleton = first_table.h5singleton
        merged_table.split_h5 = first_table.split_h5
        merged_table.name = first_table.name

        return merged_table
1107124
def rows_to_columns(matrix):
#hammer
    """"""
    num_rows = len(matrix)
    num_cols = len(matrix[0])

    data = []
    for i in range(0, num_cols):
        data.append([matrix[j][i] for j in range(0, num_rows)])

    return data
574793
def flatten(nested_list: list) -> list:
    """"""
    return list(sorted(filter(lambda y: y is not None,
                              list(map(lambda x: (nested_list.extend(x)  # noqa: T484
                                                  if isinstance(x, list) else x),
                                       nested_list)))))
735094
def pad_chunk_columns(chunk):
    """"""
    columns = set()
    for record in chunk:
        columns.update(record.keys())
    for record in chunk:
        for column in columns:
            record.setdefault(column, None)
    return chunk
413208
def combineColumns(cols):
    ''
    return Column("+".join(c.name for c in cols),
                  getter=lambda col,row,cols=cols,ch=' ': ch.join(c.getDisplayValue(row) for c in cols))
1328938
def tsplit(df, shape):
    """"""
    if isinstance(df, (pd.DataFrame, pd.Series)):
        return df.iloc[0:shape], df.iloc[shape:]
    else:
        return df[0:shape], df[shape:]
395673
def unpack_column_flat(self, value, components=None, offset=False, computed_type=None):
        
        if components:
            if isinstance(components, str):
                components = [components]
        else:
            components = self._dict.keys()

            # TODO: add this

        # we need to split the flat array by the lengths of each mesh
        N_lower = 0
        N_upper = 0
        offsetN = 0.0
        value_dict = {}
        for comp in components:
            if isinstance(self[comp], Meshes):
                # then we need to recursively extract to the underlying meshes
                # pass
                meshes = self[comp]._dict
            else:
                meshes = {comp: self[comp]}

            for c, mesh in meshes.items():
                if computed_type=='vertices' or (computed_type is None and mesh._compute_at_vertices):
                    N = mesh.Nvertices
                else:
                    N = mesh.Ntriangles
                N_upper += N
                value_dict[c] = value[N_lower:N_upper] - offsetN
                if offset:
                    offsetN += N
                N_lower += N

        return value_dict
646004
def nth(self, n, *args, **kwargs):
        """"""
        # TODO: Stop collecting the entire frame for each key.
        self._prep_pandas_groupby()
        myargs = self._myargs
        mykwargs = self._mykwargs
        nthRDD = self._regroup_mergedRDD().mapValues(
            lambda r: r.nth(
                n, *args, **kwargs)).values()
        return DataFrame.fromDataFrameRDD(nthRDD, self.sql_ctx)
1353248
def clean_dataframe(df):
    """"""
    df = df.fillna(method='ffill')
    df = df.fillna(0.0)
    return df
452907
def _reshape_output(self, ts):
        """"""
        subodim = len(self.submodels[0].output_vars)
        shp = list(ts.shape)
        shp[1] = subodim
        shp.insert(2, self._n)
        ts = ts.reshape(tuple(shp))
        ts.labels[2] = self._node_labels()
        return ts