Construct a DataFrame from an RDD of DataFrames.
        No checking or validation occurs.
1743
def fromDataFrameRDD(cls, rdd, sql_ctx):
        """"""
        result = DataFrame(None, sql_ctx)
        return result.from_rdd_of_dataframes(rdd)
515923
def fromPandas(cls, df):
        
        assert pd is not None
        if isinstance(df, pd.Series):
            df = pd.DataFrame(df)
        else:
            assert isinstance(df, pd.DataFrame)
        keys = [
            key if isinstance(key, tuple) else (key,)
            for key in df.index.tolist()
        ]
        index = [
            ('index{}'.format(i), cindex)
            for i, cindex in enumerate(zip(*keys))
        ]
        columns = [
            (str(cname), df[cname].tolist())
            for cname in df.columns.tolist()
        ]
        return cls(index=index, columns=columns)
711624
def build_conversion_table(self, dataframes):
        
        self.data = pd.DataFrame(dataframes)
        tmp_pairs = [s.split("/") for s in self.data.columns]
        self.data.columns = pd.MultiIndex.from_tuples(tmp_pairs)
1310136
def df(objs, labels=None):
    
    import pandas as pd
    from .objects import Object, DynamicObject
    if objs:
        objs = list(objs)
        obj = objs[0]
        if isinstance(obj, Object):
            df = pd.DataFrame.from_records(o.tuple() for o in objs)
            df.columns = obj.__class__.defaults
        elif isinstance(obj, DynamicObject):
            df = pd.DataFrame.from_records(o.__dict__ for o in objs)
        else:
            df = pd.DataFrame.from_records(objs)
        if isinstance(obj, tuple) and hasattr(obj, '_fields'):
            # assume it's a namedtuple
            df.columns = obj.__class__._fields
    else:
        df = None
    if labels:
        exclude = [label for label in df if label not in labels]
        df = df.drop(exclude, axis=1)
    return df
986840
def _evil_apply_with_dataframes(self, func, preserves_cols=False):
        """"""
        source_rdd = self._rdd()
        result_rdd = func(source_rdd)
        # By default we don't know what the columns & indexes are so we let
        # from_rdd_of_dataframes look at the first partition to determine them.
        column_idxs = None
        if preserves_cols:
            index_names = self._index_names
            # Remove indexes from the columns
            columns = self._schema_rdd.columns[len(self._index_names):]
            column_idxs = (columns, index_names)
        return self.from_rdd_of_dataframes(
            result_rdd, column_idxs=column_idxs)
604647
def collect(self):
        """"""
        local_df = self._schema_rdd.toPandas()
        correct_idx_df = _update_index_on_df(local_df, self._index_names)
        return correct_idx_df
315258
def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,
                tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,
                label_cols:IntsOrStrs=0, label_delim:str=None, chunksize:int=10000, max_vocab:int=60000,
                min_freq:int=2, mark_fields:bool=False, include_bos:bool=True, include_eos:bool=False, **kwargs) -> DataBunch:
        ""
        processor = _get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab,
                                   min_freq=min_freq, mark_fields=mark_fields, 
                                   include_bos=include_bos, include_eos=include_eos)
        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
                        TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
        if cls==TextLMDataBunch: src = src.label_for_lm()
        else: 
            if label_delim is not None: src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
            else: src = src.label_from_df(cols=label_cols, classes=classes)
        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))
        return src.databunch(**kwargs)
444253
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
66169
def from_dataframe(cls, name, df, indices, primary_key=None):
        """"""

        # ordered list (column_name, column_type) pairs
        column_types = []
        # which columns have nullable values
        nullable = set()

        # tag cached database by dataframe's number of rows and columns
        for column_name in df.columns:
            values = df[column_name]
            if values.isnull().any():
                nullable.add(column_name)
            column_db_type = db_type(values.dtype)
            column_types.append((column_name.replace(" ", "_"), column_db_type))

        def make_rows():
            return list(tuple(row) for row in df.values)

        return cls(
            name=name,
            column_types=column_types,
            make_rows=make_rows,
            indices=indices,
            nullable=nullable,
            primary_key=primary_key)
38772
def toDF(self, *cols):
        
        jdf = self._jdf.toDF(self._jseq(cols))
        return DataFrame(jdf, self.sql_ctx)
335389
def locus_reads_dataframe(*args, **kwargs):
    
    df_builder = DataFrameBuilder(
        LocusRead,
        variant_columns=False,
        converters={
            "reference_positions": list_to_string,
            "quality_scores": list_to_string,
        })
    for locus_read in locus_read_generator(*args, **kwargs):
        df_builder.add(variant=None, element=locus_read)
    return df_builder.to_dataframe()
234556
def build(self):
        '''
        '''

        X_factory, mX_factory, category_idx_store, term_idx_store, metadata_idx_store, y \
            = self._init_term_doc_matrix_variables()
        parse_pipeline = ParsePipelineFactory(self.get_nlp(),
                                              X_factory,
                                              mX_factory,
                                              category_idx_store,
                                              term_idx_store,
                                              metadata_idx_store,
                                              y,
                                              self)
        df = self._clean_and_filter_nulls_and_empties_from_dataframe()
        tdm = self._apply_pipeline_and_get_build_instance(X_factory,
                                                          mX_factory,
                                                          category_idx_store,
                                                          df,
                                                          parse_pipeline,
                                                          term_idx_store,
                                                          metadata_idx_store,
                                                          y)
        return tdm
287593
def _use_aggregation(self, agg, columns=None):
        """"""
        if not columns:
            columns = self._columns
        from pyspark.sql import functions as F
        aggs = map(lambda column: agg(column).alias(column), self._columns)
        aggRdd = self._grouped_spark_sql.agg(*aggs)
        df = DataFrame.from_schema_rdd(aggRdd, self._by)
        return df
577683
def as_dataframe(self, fillna=True, subjects=None):
        
        entries = []
        selected_subjects = self.subjects
        if subjects is not None:
            selected_subjects = subjects
            
        for s in selected_subjects:
            vmap = {}
            for c in self.inferred_types(s):
                vmap[c] = 1
            entries.append(vmap)
        logging.debug("Creating DataFrame")
        df = pd.DataFrame(entries, index=selected_subjects)
        if fillna:
            logging.debug("Performing fillna...")
            df = df.fillna(0)
        return df
552273
def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, schema=None):
  
  if schema is None:
    schema = google.datalab.bigquery.Schema.from_data(source)
  fields = get_field_list(fields, schema)
  rows = []
  if count < 0:
    count = len(source.index)
  df_slice = source.reset_index(drop=True)[first_row:first_row + count]
  for index, data_frame_row in df_slice.iterrows():
    row = data_frame_row.to_dict()
    for key in list(row.keys()):
      val = row[key]
      if isinstance(val, pandas.Timestamp):
        row[key] = val.to_pydatetime()

    rows.append({'c': [{'v': row[c]} if c in row else {} for c in fields]})
  cols = _get_cols(fields, schema)
  return {'cols': cols, 'rows': rows}, len(source)
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))
661761
def time_series_rdd_from_pandas_series_rdd(series_rdd):
    
    first = series_rdd.first()
    dt_index = irregular(first[1].index, series_rdd.ctx)
    return TimeSeriesRDD(dt_index, series_rdd.mapValues(lambda x: x.values))