Create a :class:`~amplpy.DataFrame` from a numpy array or matrix.
774858
def fromNumpy(cls, data):
        
        assert np is not None
        if isinstance(data, np.ndarray):
            index = []
            if len(data.shape) == 1:
                columns = [('value', data.tolist())]
            elif len(data.shape) == 2:
                columns = [
                    ('c{}'.format(i), col)
                    for i, col in enumerate(zip(*data.tolist()))
                ]
            else:
                raise TypeError
        else:
            raise TypeError
        return cls(index=index, columns=columns)
488071
def df_from_groups(groups, columns=None):
    """"""
    if columns is None:
        columns = list(groups.get_group(groups.indices.keys()[0]).columns)
    df = pd.DataFrame()
    for col, group_label in product(columns, groups.indices.keys()):
        label = '{}_{}'.format(col, group_label)
        df[label] = pd.Series(groups.get_group(group_label)[col].values)
    return df
1161798
def _make_df(rows):
    """"""

    # Make DataFrame
    df = pd.DataFrame(rows).set_index('ts')

    # TimeDelta Support: https://issues.apache.org/jira/browse/ARROW-835
    for column in df.columns:
        if(df[column].dtype == 'timedelta64[ns]'):
            print('Converting timedelta column {:s}...'.format(column))
            df[column] = df[column].astype(str)
    return df
655070
def from_columns(columns):
        
        data = [
            [
                column[i]
                for i in range(len(column))
            ]
            for column in columns
        ]
        return Matrix(data)
66169
def from_dataframe(cls, name, df, indices, primary_key=None):
        """"""

        # ordered list (column_name, column_type) pairs
        column_types = []
        # which columns have nullable values
        nullable = set()

        # tag cached database by dataframe's number of rows and columns
        for column_name in df.columns:
            values = df[column_name]
            if values.isnull().any():
                nullable.add(column_name)
            column_db_type = db_type(values.dtype)
            column_types.append((column_name.replace(" ", "_"), column_db_type))

        def make_rows():
            return list(tuple(row) for row in df.values)

        return cls(
            name=name,
            column_types=column_types,
            make_rows=make_rows,
            indices=indices,
            nullable=nullable,
            primary_key=primary_key)
285501
def from_arrays_of_labels(cls, arrayOfLabels, inputCols, outputCols=None,
                              handleInvalid=None):
        
        sc = SparkContext._active_spark_context
        java_class = sc._gateway.jvm.java.lang.String
        jlabels = StringIndexerModel._new_java_array(arrayOfLabels, java_class)
        model = StringIndexerModel._create_from_java_class(
            "org.apache.spark.ml.feature.StringIndexerModel", jlabels)
        model.setInputCols(inputCols)
        if outputCols is not None:
            model.setOutputCols(outputCols)
        if handleInvalid is not None:
            model.setHandleInvalid(handleInvalid)
        return model
593040
def numpy_buffer(ndarray):
    """"""
    # Credits to: https://github.com/joblib/joblib/blob/04b001861e1dd03a857b7b419c336de64e05714c/joblib/hashing.py

    if isinstance(ndarray, (pd.Series, pd.DataFrame)):
        ndarray = ndarray.values

    if ndarray.flags.c_contiguous:
        obj_c_contiguous = ndarray
    elif ndarray.flags.f_contiguous:
        obj_c_contiguous = ndarray.T
    else:
        obj_c_contiguous = ndarray.flatten()

    obj_c_contiguous = obj_c_contiguous.view(np.uint8)

    if hasattr(np, 'getbuffer'):
        return np.getbuffer(obj_c_contiguous)
    else:
        return memoryview(obj_c_contiguous)
896113
def convert_sparse_to_dataframe(spark, context, sparse_matrix):
    
    m = sparse_matrix.tocoo()
    data = context.parallelize(numpy.array([m.row, m.col, m.data]).T,
                               numSlices=len(m.row)/1024)
    return spark.createDataFrame(data.map(lambda p: Row(row=int(p[0]),
                                                        col=int(p[1]),
                                                        data=float(p[2]))))
632063
def broadcast(df):
    """"""

    sc = SparkContext._active_spark_context
    return DataFrame(sc._jvm.functions.broadcast(df._jdf), df.sql_ctx)
109024
def from_xy(cls, x_array, y_array):
        
        if len(x_array) == 0:
            raise ValueError("data array is empty.")
        dim_x, dim_y = len(x_array[0]), len(y_array[0])
        dataset = cls(dim_x, dim_y)
        for x, y in zip(x_array, y_array):
            assert len(x) == dim_x and len(y) == dim_y
            dataset.add_xy(x, y)
        return dataset
866805
def as_dframe(cls, dataset):
        
        if issubclass(dataset.interface, PandasInterface):
            return dataset.data
        else:
            return dataset.dframe()
1143022
def _init_matrix(self, data, index, columns, dtype=None):
        
        data = prep_ndarray(data, copy=False)
        index, columns = self._prep_index(data, index, columns)
        data = {idx: data[:, i] for i, idx in enumerate(columns)}
        return self._init_dict(data, index, columns, dtype)
1026674
def unit_vector(vector, **kwargs):
    """"""
    # This also works for a dataframe with columns ['x', 'y', 'z']
    # However, the division operation is picky about the shapes
    # So, remember input vector shape, cast all up to 2d,
    # do the (ugly) conversion, then return unit in same shape as input
    # of course, the numpy-ized version of the input...
    vector = np.array(vector)
    out_shape = vector.shape
    vector = np.atleast_2d(vector)
    unit = vector / np.linalg.norm(vector, axis=1, **kwargs)[:, None]
    return unit.reshape(out_shape)
261033
def from_df(cls, path, df:DataFrame, dep_var:str, valid_idx:Collection[int], procs:OptTabTfms=None,
                cat_names:OptStrList=None, cont_names:OptStrList=None, classes:Collection=None, 
                test_df=None, bs:int=64, val_bs:int=None, num_workers:int=defaults.cpus, dl_tfms:Optional[Collection[Callable]]=None, 
                device:torch.device=None, collate_fn:Callable=data_collate, no_check:bool=False)->DataBunch:
        ""
        cat_names = ifnone(cat_names, []).copy()
        cont_names = ifnone(cont_names, list(set(df)-set(cat_names)-{dep_var}))
        procs = listify(procs)
        src = (TabularList.from_df(df, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)
                           .split_by_idx(valid_idx))
        src = src.label_from_df(cols=dep_var) if classes is None else src.label_from_df(cols=dep_var, classes=classes)
        if test_df is not None: src.add_test(TabularList.from_df(test_df, cat_names=cat_names, cont_names=cont_names,
                                                                 processor = src.train.x.processor))
        return src.databunch(path=path, bs=bs, val_bs=val_bs, num_workers=num_workers, device=device, 
                             collate_fn=collate_fn, no_check=no_check)
182905
def _coerce_to_ndarray(cls, data):
        

        if not isinstance(data, (np.ndarray, Index)):
            if data is None or is_scalar(data):
                cls._scalar_data_error(data)

            # other iterable of some kind
            if not isinstance(data, (ABCSeries, list, tuple)):
                data = list(data)
            data = np.asarray(data)
        return data
1176573
def _vector(x, type='row'):
    """"""
    if isinstance(x, (list, tuple)):
        x = np.array(x, dtype=np.float32)
    elif not isinstance(x, np.ndarray):
        x = np.array([x], dtype=np.float32)
    assert x.ndim == 1
    if type == 'column':
        x = x[:, None]
    return x
1158900
def _create_dataframe(cls, value):
        
        try:

            ordered_jo = json.loads(six.text_type(value), object_pairs_hook=collections.OrderedDict)

            #Pandas does not maintain the order of dicts, so we must break the dict
            #up and put it into the dataframe manually to maintain the order.

            cols = list(ordered_jo.keys())

            if len(cols) == 0:
                raise ValueError("Dataframe has no columns")

            #Assume all sub-dicts have the same set of keys
            if isinstance(ordered_jo[cols[0]], list):
                index = range(len(ordered_jo[cols[0]]))
            else:
                index = list(ordered_jo[cols[0]].keys())
            data = []
            for c in cols:
                if isinstance(ordered_jo[c], list):
                    data.append(ordered_jo[c])
                else:
                    data.append(list(ordered_jo[c].values()))

            # This goes in 'sideways' (cols=index, index=cols), so it needs to be transposed after to keep
            # the correct structure
            # We also try to coerce the data to a regular numpy array first. If the shape is correct
            # this is a much faster way of creating the DataFrame instance.
            try:
                np_data = np.array(data)
            except ValueError:
                np_data = None

            if np_data is not None and np_data.shape == (len(cols), len(index)):
                df = pd.DataFrame(np_data, columns=index, index=cols).transpose()
            else:
                # TODO should these heterogenous structure be supported?
                # See https://github.com/hydraplatform/hydra-base/issues/72
                df = pd.DataFrame(data, columns=index, index=cols).transpose()


        except ValueError as e:
            """ Raised on scalar types used as pd.DataFrame values
                in absence of index arg
            """
            raise HydraError(str(e))

        except AssertionError as e:
            log.warning("An error occurred creating the new data frame: %s. Defaulting to a simple read_json"%(e))
            df = pd.read_json(value).fillna(0)

        return df