Return a file-like object of CSV-encoded rows.

    Args:
      dataframe (pandas.DataFrame): A chunk of a dataframe to encode
343075
def encode_chunk(dataframe):
    
    csv_buffer = six.StringIO()
    dataframe.to_csv(
        csv_buffer,
        index=False,
        header=False,
        encoding="utf-8",
        float_format="%.15g",
        date_format="%Y-%m-%d %H:%M:%S.%f",
    )

    # Convert to a BytesIO buffer so that unicode text is properly handled.
    # See: https://github.com/pydata/pandas-gbq/issues/106
    body = csv_buffer.getvalue()
    if isinstance(body, bytes):
        body = body.decode("utf-8")
    body = body.encode("utf-8")
    return six.BytesIO(body)
344887
def get_dataframe():
        
        names = []
        [names.extend(line.split()) for line in CONTROL_VARIABLE_LINES]

        defaults = []
        [defaults.extend(line.split()) for line in CONTROL_DEFAULT_LINES]

        types, required,cast_defaults,formats = [],[],[],[]
        for name,default in zip(names,defaults):
            if '[' in name or ']' in name:
                required.append(False)
            else:
                required.append(True)
            v,t,f = ControlData._parse_value(default)
            types.append(t)
            formats.append(f)
            cast_defaults.append(v)
        return pandas.DataFrame({"name":names,"type":types,
                                     "value":cast_defaults,"required":required,
                                    "format":formats})
223468
def transform_dataframe(self, dataframe):
        
        dataframe.columns.name = ""

        for i in range(len(self.get_header_fields())):
            dataframe = dataframe.unstack()

        # Remove blank rows / columns
        dataframe = dataframe.dropna(
            axis=0, how='all'
        ).dropna(
            axis=1, how='all'
        )
        return dataframe
529417
def df_to_parquet(df, filename, compression='SNAPPY'):
    

    # Right now there are two open Parquet issues
    # Timestamps in Spark: https://issues.apache.org/jira/browse/ARROW-1499
    # TimeDelta Support: https://issues.apache.org/jira/browse/ARROW-835
    for column in df.columns:
        if(df[column].dtype == 'timedelta64[ns]'):
            print('Converting timedelta column {:s}...'.format(column))
            df[column] = df[column].astype(str)

    arrow_table = pa.Table.from_pandas(df)
    if compression == 'UNCOMPRESSED':
        compression = None
    pq.write_table(arrow_table, filename, compression=compression, use_deprecated_int96_timestamps=True)
177424
def file_rows(self, fo):
        """"""

        rows = []
        for i in range(NUMROWS):
            line = fo.readline()
            if not line:
                break
            rows += [line]

        return rows
1161798
def _make_df(rows):
    """"""

    # Make DataFrame
    df = pd.DataFrame(rows).set_index('ts')

    # TimeDelta Support: https://issues.apache.org/jira/browse/ARROW-835
    for column in df.columns:
        if(df[column].dtype == 'timedelta64[ns]'):
            print('Converting timedelta column {:s}...'.format(column))
            df[column] = df[column].astype(str)
    return df
616290
def get_rows(self):
        
        possible_dataframes = ['F', 'FY', 'M', 'S',
                               'D_cba', 'D_pba',  'D_imp', 'D_exp',
                               'D_cba_reg', 'D_pba_reg',
                               'D_imp_reg', 'D_exp_reg',
                               'D_cba_cap', 'D_pba_cap',
                               'D_imp_cap', 'D_exp_cap', ]
        for df in possible_dataframes:
            if (df in self.__dict__) and (getattr(self, df) is not None):
                return getattr(self, df).index.get_values()
        else:
            logging.warn("No attributes available to get row names")
            return None
526057
def csv_to_dict(csv_filepath, **kwargs):
    
    callbacks = {'to_list': csv_tolist,
                 'row_csv_limiter': row_csv_limiter,
                 'csv_row_cleaner': csv_row_cleaner,
                 'row_headers_count': row_headers_count,
                 'get_col_header': get_csv_col_headers,
                 'get_row_headers': get_row_headers,
                 'populate_headers': populate_headers,
                 'csv_column_header_cleaner': csv_column_header_cleaner,
                 'csv_column_cleaner': csv_column_cleaner,
                 'retrieve_csv_data': retrieve_csv_data}

    callbacks.update(kwargs.get('alt_callbacks', {}))
    rows = kwargs.get('rows', [])

    if not rows:
        # csv_tolist of rows
        rows = callbacks.get('to_list')(csv_filepath, **kwargs)

        if not rows:
            msg = 'Empty rows obtained from {}'.format(csv_filepath)
            logger.warning(msg)
            raise ValueError(msg)

    # apply limits
    rows = callbacks.get('row_csv_limiter')(
        rows, kwargs.get('limits', [None, None]))

    # apply row cleaner
    rows = callbacks.get('csv_row_cleaner')(rows)

    # apply column cleaner
    rows = callbacks.get('csv_column_cleaner')(rows)

    # count raw headers
    num_row_headers = callbacks.get('row_headers_count')(rows)

    # take colum_headers
    c_headers_raw = callbacks.get('get_col_header')(rows, num_row_headers)

    # get row_headers
    r_headers = callbacks.get('get_row_headers')(
        rows, num_row_headers, len(c_headers_raw))

    # format colum_headers
    c_headers_dirty = callbacks.get('populate_headers')(
        c_headers_raw) if len(c_headers_raw) > 1 else c_headers_raw[0]

    # Clean csv column headers of empty values.
    c_headers = callbacks.get('csv_column_header_cleaner')(c_headers_dirty)

    # take data
    csv_data = callbacks.get('retrieve_csv_data')(
        rows,
        column_header=len(c_headers_raw),
        row_header=num_row_headers,
        limit_column=len(c_headers) - len(c_headers_dirty) or None)

    # Check column headers validation
    if csv_data:
        assert len(c_headers) == len(csv_data[0])

    # Check row headers validation
    if r_headers:
        assert len(r_headers) == len(csv_data)

    # Transform rows into dict zipping the headers.
    kwargs.pop('rows', None)
    result = csv_format(csv_data, c_headers, r_headers, rows, **kwargs)

    return result
204183
def dataframe(self):
        
        frames = []
        for game in self.__iter__():
            # If both the runs scored and allowed are None, the game hasn't
            # been played yet, and the data should not be included in the
            # DataFrame.
            if game._runs_scored is None and game._runs_allowed is None:
                continue
            frames.append(game.dataframe)
        if frames == []:
            return None
        return pd.concat(frames)
66169
def from_dataframe(cls, name, df, indices, primary_key=None):
        """"""

        # ordered list (column_name, column_type) pairs
        column_types = []
        # which columns have nullable values
        nullable = set()

        # tag cached database by dataframe's number of rows and columns
        for column_name in df.columns:
            values = df[column_name]
            if values.isnull().any():
                nullable.add(column_name)
            column_db_type = db_type(values.dtype)
            column_types.append((column_name.replace(" ", "_"), column_db_type))

        def make_rows():
            return list(tuple(row) for row in df.values)

        return cls(
            name=name,
            column_types=column_types,
            make_rows=make_rows,
            indices=indices,
            nullable=nullable,
            primary_key=primary_key)
81911
def prepare_csv_to_dataframe(data_file, config, use_target=True):
    
    names, dtypes = [], []
    model = config.get_data_model()
    for feature in model:
        assert feature.get_name() not in names, "Two features can't have the same name."
        if not use_target and feature.is_target():
            continue
        names.append(feature.get_name())
    data = pd.read_csv(data_file, names=names)

    transform_categorical_features(config, data, use_target)

    return data
793840
def load_data(filename):
    
    data = pandas.read_csv(filename, header=None, delimiter='\t', skiprows=9)
    return data.as_matrix()
402264
def loadcsv(filename):
    
    dataframe = _pd.read_csv(filename)
    data = {}
    for key, value in dataframe.items():
        data[key] = value.values
    return data
808060
def to_dataframe(self, columns=None):
    """"""
    if not self.is_value_set:
      raise ValueError('Value must be set before converting to a DataFrame.')
    if not pandas:
      raise RuntimeError('Install pandas to convert to pandas.DataFrame')
    return pandas.DataFrame.from_records(self.value, columns=columns)
688925
def _get_processed_dataframe(self, dataframe):
        
        dataframe.index = pd.to_datetime(dataframe['epoch'], unit='s', utc=True)
        del dataframe['epoch']
        summary = dataframe.describe(percentiles=[.80, .90, .95]).transpose().loc['scriptrun_time']
        df_grp = dataframe.groupby(pd.TimeGrouper('{}S'.format(self.interval)))
        df_final = df_grp.apply(lambda x: x.describe(percentiles=[.80, .90, .95])['scriptrun_time'])

        return {
            "raw": dataframe.round(2),
            "compiled": df_final.round(2),
            "summary": summary.round(2)
        }
183933
def _render_dataframe(dataframe):
  """"""
  data = dataframe.to_dict(orient='records')
  fields = dataframe.columns.tolist()
  return IPython.core.display.HTML(
      datalab.utils.commands.HtmlBuilder.render_table(data, fields))
482126
def objify(self, doc, columns=None):
        
        cols = columns or doc[METADATA][COLUMNS]
        data = {}

        for col in cols:
            # if there is missing data in a chunk, we can default to NaN
            # and pandas will autofill the missing values to the correct length
            if col not in doc[METADATA][LENGTHS]:
                d = [np.nan]
            else:
                d = decompress(doc[DATA][doc[METADATA][LENGTHS][col][0]: doc[METADATA][LENGTHS][col][1] + 1])
                # d is ready-only but that's not an issue since DataFrame will copy the data anyway.
                d = np.frombuffer(d, doc[METADATA][DTYPE][col])

                if MASK in doc[METADATA] and col in doc[METADATA][MASK]:
                    mask_data = decompress(doc[METADATA][MASK][col])
                    mask = np.frombuffer(mask_data, 'bool')
                    d = ma.masked_array(d, mask)
            data[col] = d

        # Copy into
        return pd.DataFrame(data, columns=cols, copy=True)[cols]