https://github.com/frictionlessdata/tableschema-pandas-py#storage
353804
def write(self, bucket, rows):
        

        # Prepare
        descriptor = self.describe(bucket)
        new_data_frame = self.__mapper.convert_descriptor_and_rows(descriptor, rows)

        # Just set new DataFrame if current is empty
        if self.__dataframes[bucket].size == 0:
            self.__dataframes[bucket] = new_data_frame

        # Append new data frame to the old one setting new data frame
        # containing data from both old and new data frames
        else:
            self.__dataframes[bucket] = pd.concat([
                self.__dataframes[bucket],
                new_data_frame,
            ])
405462
def save_q_df(self, state_key, action_key, q_value):
        '''
        

        '''
        if isinstance(q_value, float) is False:
            raise TypeError("The type of q_value must be float.")

        new_q_df = pd.DataFrame([(state_key, action_key, q_value)], columns=["state_key", "action_key", "q_value"])
        if self.q_df is not None:
            self.q_df = pd.concat([new_q_df, self.q_df])
            self.q_df = self.q_df.drop_duplicates(["state_key", "action_key"])
        else:
            self.q_df = new_q_df
1339394
function createDBIfNotExists() {
  const tmpClient = knexLib({
    client: 'pg',
    connection: {
      host: POSTGRES_HOST,
      user: POSTGRES_USER,
      password: POSTGRES_PASSWORD,
      database: 'postgres',
      port: POSTGRES_PORT
    },
    pool: { min: CONNECTION_POOL_MIN, max: CONNECTION_POOL_MAX }
  });

  // https://github.com/clay/amphora-storage-postgres/pull/7/files/16d3429767943a593ad9667b0d471fefc15088d3#diff-6a1e11a6146d3a5a01f955a44a2ac07a
  return tmpClient.raw('CREATE DATABASE ??', [POSTGRES_DB])
    .then(() => tmpClient.destroy())
    .then(connect);
}
404758
def _storage_create(args, _):
  
  buckets = datalab.storage.Buckets(project_id=args['project'])
  errs = []
  for name in args['bucket']:
    try:
      bucket, key = datalab.storage._bucket.parse_name(name)
      if bucket and not key:
        buckets.create(bucket)
      else:
        raise Exception("Invalid bucket name %s" % name)
    except Exception as e:
      errs.append("Couldn't create %s: %s" %
                  (name, _extract_storage_api_response_error(str(e))))
  if errs:
    raise Exception('\n'.join(errs))
754716
def write_table_to_file(self, dtype, custom_name=None, append=False, dir_path=None):
        
        if custom_name:
            fname = custom_name
        else:
            fname = self.filenames[dtype]
        if not dir_path:
            dir_path=self.directory
        if dtype in self.tables:
            write_df = self.remove_names(dtype)
            outfile = self.tables[dtype].write_magic_file(custom_name=fname,
                                                          dir_path=dir_path,
                                                          append=append, df=write_df)
        return outfile
762025
def take_table(tbl, indices, axis=None, out=None, mode='raise', blen=None, storage=None,
               create='table', **kwargs):
    """"""

    # setup
    if axis is not None and axis != 0:
        raise NotImplementedError('only axis 0 is supported')
    if out is not None:
        # argument is only there for numpy API compatibility
        raise NotImplementedError('out argument is not supported')
    if mode is not None and mode != 'raise':
        raise NotImplementedError('only mode=raise is supported')
    names, columns = _util.check_table_like(tbl)
    length = len(columns[0])

    # check that indices are strictly increasing
    indices = np.asanyarray(indices)
    if np.any(indices[1:] <= indices[:-1]):
        raise NotImplementedError(
            'indices must be strictly increasing'
        )

    # implement via compress()
    condition = np.zeros((length,), dtype=bool)
    condition[indices] = True
    return compress_table(condition, tbl, blen=blen, storage=storage,
                          create=create, **kwargs)
1113407
def get_table(self, schema_type):
        """"""
        table_name = schema_type.lower()
        if not self.has_table(table_name):
            raise exceptions.GraphQLCompilationError(
                'No Table found in SQLAlchemy metadata for table name "{}"'.format(table_name)
            )
        return self.table_name_to_table[table_name]
18366
def _create_storer(self, group, format=None, value=None, append=False,
                       **kwargs):
        

        def error(t):
            raise TypeError(
                "cannot properly create the storer for: [{t}] [group->"
                "{group},value->{value},format->{format},append->{append},"
                "kwargs->{kwargs}]".format(t=t, group=group,
                                           value=type(value), format=format,
                                           append=append, kwargs=kwargs))

        pt = _ensure_decoded(getattr(group._v_attrs, 'pandas_type', None))
        tt = _ensure_decoded(getattr(group._v_attrs, 'table_type', None))

        # infer the pt from the passed value
        if pt is None:
            if value is None:

                _tables()
                if (getattr(group, 'table', None) or
                        isinstance(group, _table_mod.table.Table)):
                    pt = 'frame_table'
                    tt = 'generic_table'
                else:
                    raise TypeError(
                        "cannot create a storer if the object is not existing "
                        "nor a value are passed")
            else:

                try:
                    pt = _TYPE_MAP[type(value)]
                except KeyError:
                    error('_TYPE_MAP')

                # we are actually a table
                if format == 'table':
                    pt += '_table'

        # a storer node
        if 'table' not in pt:
            try:
                return globals()[_STORER_MAP[pt]](self, group, **kwargs)
            except KeyError:
                error('_STORER_MAP')

        # existing node (and must be a table)
        if tt is None:

            # if we are a writer, determine the tt
            if value is not None:

                if pt == 'series_table':
                    index = getattr(value, 'index', None)
                    if index is not None:
                        if index.nlevels == 1:
                            tt = 'appendable_series'
                        elif index.nlevels > 1:
                            tt = 'appendable_multiseries'
                elif pt == 'frame_table':
                    index = getattr(value, 'index', None)
                    if index is not None:
                        if index.nlevels == 1:
                            tt = 'appendable_frame'
                        elif index.nlevels > 1:
                            tt = 'appendable_multiframe'
                elif pt == 'wide_table':
                    tt = 'appendable_panel'
                elif pt == 'ndim_table':
                    tt = 'appendable_ndim'

            else:

                # distiguish between a frame/table
                tt = 'legacy_panel'
                try:
                    fields = group.table._v_attrs.fields
                    if len(fields) == 1 and fields[0] == 'value':
                        tt = 'legacy_frame'
                except IndexError:
                    pass

        try:
            return globals()[_TABLE_MAP[tt]](self, group, **kwargs)
        except KeyError:
            error('_TABLE_MAP')
197580
def from_dataframe(dataframe, table_name=""):
        

        return TableData(table_name, list(dataframe.columns.values), dataframe.values.tolist())
173030
def to_prettytable(df):
    
    pt = PrettyTable()
    pt.field_names = df.columns
    for tp in zip(*(l for col, l in df.iteritems())):
        pt.add_row(tp)
    return pt
815427
def create_table_from_csv(form, table):
        """"""
        def convert_to_hive_type(col_type):
            """maps tableschema's types to hive types"""
            tableschema_to_hive_types = {
                'boolean': 'BOOLEAN',
                'integer': 'INT',
                'number': 'DOUBLE',
                'string': 'STRING',
            }
            return tableschema_to_hive_types.get(col_type, 'STRING')

        bucket_path = config['CSV_TO_HIVE_UPLOAD_S3_BUCKET']

        if not bucket_path:
            logging.info('No upload bucket specified')
            raise Exception(
                'No upload bucket specified. You can specify one in the config file.')

        table_name = form.name.data
        schema_name = form.schema.data

        if config.get('UPLOADED_CSV_HIVE_NAMESPACE'):
            if '.' in table_name or schema_name:
                raise Exception(
                    "You can't specify a namespace. "
                    'All tables will be uploaded to the `{}` namespace'.format(
                        config.get('HIVE_NAMESPACE')))
            full_table_name = '{}.{}'.format(
                config.get('UPLOADED_CSV_HIVE_NAMESPACE'), table_name)
        else:
            if '.' in table_name and schema_name:
                raise Exception(
                    "You can't specify a namespace both in the name of the table "
                    'and in the schema field. Please remove one')

            full_table_name = '{}.{}'.format(
                schema_name, table_name) if schema_name else table_name

        filename = form.csv_file.data.filename

        upload_prefix = config['CSV_TO_HIVE_UPLOAD_DIRECTORY']
        upload_path = config['UPLOAD_FOLDER'] + \
            secure_filename(filename)

        # Optional dependency
        from tableschema import Table  # pylint: disable=import-error
        hive_table_schema = Table(upload_path).infer()
        column_name_and_type = []
        for column_info in hive_table_schema['fields']:
            column_name_and_type.append(
                '`{}` {}'.format(
                    column_info['name'],
                    convert_to_hive_type(column_info['type'])))
        schema_definition = ', '.join(column_name_and_type)

        # Optional dependency
        import boto3  # pylint: disable=import-error

        s3 = boto3.client('s3')
        location = os.path.join('s3a://', bucket_path, upload_prefix, table_name)
        s3.upload_file(
            upload_path, bucket_path,
            os.path.join(upload_prefix, table_name, filename))
        sql = f"""CREATE TABLE {full_table_name} ( {schema_definition} )
            ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS
            TEXTFILE LOCATION '{location}'
            tblproperties ('skip.header.line.count'='1')"""
        logging.info(form.con.data)
        engine = create_engine(form.con.data.sqlalchemy_uri_decrypted)
        engine.execute(sql)
711624
def build_conversion_table(self, dataframes):
        
        self.data = pd.DataFrame(dataframes)
        tmp_pairs = [s.split("/") for s in self.data.columns]
        self.data.columns = pd.MultiIndex.from_tuples(tmp_pairs)
552273
def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, schema=None):
  
  if schema is None:
    schema = google.datalab.bigquery.Schema.from_data(source)
  fields = get_field_list(fields, schema)
  rows = []
  if count < 0:
    count = len(source.index)
  df_slice = source.reset_index(drop=True)[first_row:first_row + count]
  for index, data_frame_row in df_slice.iterrows():
    row = data_frame_row.to_dict()
    for key in list(row.keys()):
      val = row[key]
      if isinstance(val, pandas.Timestamp):
        row[key] = val.to_pydatetime()

    rows.append({'c': [{'v': row[c]} if c in row else {} for c in fields]})
  cols = _get_cols(fields, schema)
  return {'cols': cols, 'rows': rows}, len(source)
382810
def _construct_df(self, mol2_lines, col_names, col_types):
        """"""
        return self._atomsection_to_pandas(self._get_atomsection(mol2_lines),
                                           col_names=col_names,
                                           col_types=col_types)
75551
def create_tablefn_map(fns, pqdb, poolnames):
    """"""
    poolmap = {name: pid for (name, pid) in pqdb.get_all_poolnames()}
    pqdb.store_table_files([(poolmap[pool], os.path.basename(fn))
                            for fn, pool in zip(fns, poolnames)])
    return pqdb.get_tablefn_map()
941531
def storage(self):
        
        if self.backend == 'redis':
            return RedisBackend(self.prefix, self.secondary_indexes)
        if self.backend == 'dynamodb':
            return DynamoDBBackend(self.prefix, self.key, self.sort_key,
                                   self.secondary_indexes)
        return DictBackend(self.prefix, self.secondary_indexes)
537588
def save(self, path=None, complevel=1, complib='zlib'):
        
        if path is None:
            path = self.hexuid + '.hdf5'
        elif os.path.isdir(path):
            path += os.sep + self.hexuid + '.hdf5'
        elif not (path.endswith('.hdf5') or path.endswith('.hdf')):
            raise ValueError('File path must have a ".hdf5" or ".hdf" extension.')
        with pd.HDFStore(path, 'w', complevel=complevel, complib=complib) as store:
            store['kwargs'] = pd.Series()
            store.get_storer('kwargs').attrs.metadata = self._rel()
            fc = 0    # Field counter (see special handling of fields below)
            for name, data in self._data().items():
                if hasattr(data, '_revert_categories'):
                    data._revert_categories()
                name = name[1:] if name.startswith('_') else name
                if isinstance(data, Field):    # Fields are handled separately
                    fname = 'FIELD{}_'.format(fc) + name + '/'
                    store[fname + 'data'] = pd.DataFrame(data)
                    for i, field in enumerate(data.field_values):
                        ffname = fname + 'values' + str(i)
                        if isinstance(field, pd.Series):
                            store[ffname] = pd.Series(field)
                        else:
                            store[ffname] = pd.DataFrame(field)
                    fc += 1
                elif isinstance(data, Series):
                    s = pd.Series(data)
                    if isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        s = s.astype('O')
                    store[name] = s
                elif isinstance(data, DataFrame):
                    store[name] = pd.DataFrame(data)
                elif isinstance(data, SparseSeries):
                    s = pd.SparseSeries(data)
                    if isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        s = s.astype('O')
                    store[name] = s
                elif isinstance(data, SparseDataFrame):
                    store[name] = pd.SparseDataFrame(data)
                else:
                    if hasattr(data, 'dtype') and isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        data = data.astype('O')
                    else:
                        for col in data:
                            if isinstance(data[col].dtype, pd.types.dtypes.CategoricalDtype):
                                data[col] = data[col].astype('O')
                    store[name] = data
                if hasattr(data, '_set_categories'):
                    data._set_categories()