Import lines from f as new table in db with cursor c.
769390
def LoadFile(f, table_name, conn):
  """"""
  reader = csv.reader(f)
  header = next(reader)

  columns = []
  for n in header:
    n = n.replace(' ', '')
    n = n.replace('-', '_')
    columns.append(n)

  create_columns = []
  column_types = {}
  for n in columns:
    if n in column_types:
      create_columns.append("%s %s" % (n, column_types[n]))
    else:
      create_columns.append("%s INTEGER" % (n))

  c = conn.cursor()
  try:
    c.execute("CREATE TABLE %s (%s)" % (table_name, ",".join(create_columns)))
  except sqlite.OperationalError:
    # Likely table exists
    print("table %s already exists?" % (table_name))
    for create_column in create_columns:
      try:
        c.execute("ALTER TABLE %s ADD COLUMN %s" % (table_name, create_column))
      except sqlite.OperationalError:
        # Likely it already exists
        print("column %s already exists in %s?" % (create_column, table_name))

  placeholders = ",".join(["?"] * len(columns))
  insert_values = "INSERT INTO %s (%s) VALUES (%s)" % (table_name, ",".join(columns), placeholders)

  #c.execute("BEGIN TRANSACTION;")
  for row in reader:
    if row:
      if len(row) < len(columns):
        row.extend([None] * (len(columns) - len(row)))
      c.execute(insert_values, row)
  #c.execute("END TRANSACTION;")
  conn.commit()
77639
def create_from_row(cls, table_row):
        
        kwargs = {}
        for key in table_row.colnames:
            kwargs[key] = table_row[key]

        infile_refs = kwargs.pop('infile_refs')
        outfile_refs = kwargs.pop('outfile_refs')
        rmfile_refs = kwargs.pop('rmfile_refs')
        intfile_refs = kwargs.pop('intfile_refs')

        kwargs['infile_ids'] = np.arange(infile_refs[0], infile_refs[1])
        kwargs['outfile_ids'] = np.arange(outfile_refs[0], outfile_refs[1])
        kwargs['rmfile_ids'] = np.arange(rmfile_refs[0], rmfile_refs[1])
        kwargs['intfile_ids'] = np.arange(intfile_refs[0], intfile_refs[1])
        return cls(**kwargs)
976664
def cli(
    paths,
    dbname,
    separator,
    quoting,
    skip_errors,
    replace_tables,
    table,
    extract_column,
    date,
    datetime,
    datetime_format,
    primary_key,
    fts,
    index,
    shape,
    filename_column,
    no_index_fks,
    no_fulltext_fks,
):
    
    # make plural for more readable code:
    extract_columns = extract_column
    del extract_column

    if extract_columns:
        click.echo("extract_columns={}".format(extract_columns))
    if dbname.endswith(".csv"):
        raise click.BadParameter("dbname must not end with .csv")
    if "." not in dbname:
        dbname += ".db"

    db_existed = os.path.exists(dbname)

    conn = sqlite3.connect(dbname)

    dataframes = []
    csvs = csvs_from_paths(paths)
    sql_type_overrides = None
    for name, path in csvs.items():
        try:
            df = load_csv(path, separator, skip_errors, quoting, shape)
            df.table_name = table or name
            if filename_column:
                df[filename_column] = name
                if shape:
                    shape += ",{}".format(filename_column)
            sql_type_overrides = apply_shape(df, shape)
            apply_dates_and_datetimes(df, date, datetime, datetime_format)
            dataframes.append(df)
        except LoadCsvError as e:
            click.echo("Could not load {}: {}".format(path, e), err=True)

    click.echo("Loaded {} dataframes".format(len(dataframes)))

    # Use extract_columns to build a column:(table,label) dictionary
    foreign_keys = {}
    for col in extract_columns:
        bits = col.split(":")
        if len(bits) == 3:
            foreign_keys[bits[0]] = (bits[1], bits[2])
        elif len(bits) == 2:
            foreign_keys[bits[0]] = (bits[1], "value")
        else:
            foreign_keys[bits[0]] = (bits[0], "value")

    # Now we have loaded the dataframes, we can refactor them
    created_tables = {}
    refactored = refactor_dataframes(
        conn, dataframes, foreign_keys, not no_fulltext_fks
    )
    for df in refactored:
        # This is a bit trickier because we need to
        # create the table with extra SQL for foreign keys
        if replace_tables and table_exists(conn, df.table_name):
            drop_table(conn, df.table_name)
        if table_exists(conn, df.table_name):
            df.to_sql(df.table_name, conn, if_exists="append", index=False)
        else:
            to_sql_with_foreign_keys(
                conn,
                df,
                df.table_name,
                foreign_keys,
                sql_type_overrides,
                primary_keys=primary_key,
                index_fks=not no_index_fks,
            )
            created_tables[df.table_name] = df
        if index:
            for index_defn in index:
                add_index(conn, df.table_name, index_defn)

    # Create FTS tables
    if fts:
        fts_version = best_fts_version()
        if not fts_version:
            conn.close()
            raise click.BadParameter(
                "Your SQLite version does not support any variant of FTS"
            )
        # Check that columns make sense
        for table, df in created_tables.items():
            for fts_column in fts:
                if fts_column not in df.columns:
                    raise click.BadParameter(
                        'FTS column "{}" does not exist'.format(fts_column)
                    )

        generate_and_populate_fts(conn, created_tables.keys(), fts, foreign_keys)

    conn.close()

    if db_existed:
        click.echo(
            "Added {} CSV file{} to {}".format(
                len(csvs), "" if len(csvs) == 1 else "s", dbname
            )
        )
    else:
        click.echo(
            "Created {} from {} CSV file{}".format(
                dbname, len(csvs), "" if len(csvs) == 1 else "s"
            )
        )
42067
def DumpCurrentSchema(cursor):
  """"""
  cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES "
                 "WHERE table_schema = (SELECT DATABASE())")
  defs = []
  for table, in sorted(cursor.fetchall()):
    cursor.execute("SHOW CREATE TABLE `{}`".format(table))
    rows = cursor.fetchall()
    defs.append(rows[0][1])

  return "\n\n".join(defs)
1089766
def insert_statement(table, columns, values):
    """"""
    if not all(isinstance(r, (list, set, tuple)) for r in values):
        values = [[r] for r in values]
    rows = []
    for row in values:
        new_row = []
        for col in row:
            if col is None:
                new_col = 'NULL'
            elif isinstance(col, (int, float, Decimal)):
                new_col = str(MySQLConverterBase().to_mysql(col))
            else:
                string = str(MySQLConverterBase().to_mysql(col))
                if "'" in string:
                    new_col = '"' + string + '"'
                else:
                    new_col = "'" + string + "'"
            new_row.append(new_col)
        rows.append(', '.join(new_row))
    vals = '(' + '),\n\t('.join(rows) + ')'
    statement = "INSERT INTO\n\t{0} ({1}) \nVALUES\n\t{2}".format(wrap(table), cols_str(columns), vals)
    return statement
877248
def dump_table(data: List[dict], fieldnames: Sequence[str]) -> str:
    

    def min3(num: int) -> int:
        return 3 if num < 4 else num

    width_by_col: Dict[str, int] = {
        f: min3(max([string_width(str(d.get(f))) for d in data] + [string_width(f)])) for f in fieldnames
    }

    def fill_spaces(word: str, width: int, center=False):
        """ aaa, 4 => ' aaa  ' """
        to_fills: int = width - string_width(word)
        return f" {' ' * floor(to_fills / 2)}{word}{' ' * ceil(to_fills / 2)} " if center \
            else f" {word}{' ' * to_fills} "

    def to_record(r: dict) -> str:
        return f"|{'|'.join([fill_spaces(str(r.get(f)), width_by_col.get(f)) for f in fieldnames])}|"

    return f"""
|{'|'.join([fill_spaces(x, width_by_col.get(x), center=True) for x in fieldnames])}|
|{'|'.join([fill_spaces(width_by_col.get(x) * "-", width_by_col.get(x)) for x in fieldnames])}|
{os.linesep.join([to_record(x) for x in data])}
""".lstrip()
1338088
def insert(self, table, records, create_cols=False, dtypes=None):
        
        if self._check_db() is False:
            return
        try:
            table = self.db[table]
        except Exception as e:
            self.err(e, "Can not find table " + table)
        t = type(records)
        if t == dict:
            func = table.insert
        elif t == list:
            func = table.insert_many
        else:
            msg = "Rows datatype " + \
                str(t) + " not valid: use a list or a dictionary"
            self.err(msg)
        if create_cols is True:
            try:
                func(records, ensure=True, types=dtypes)
            except Exception as e:
                self.err(e, "Can not insert create columns and insert data")
            return
        else:
            try:
                func(records, types=dtypes)
            except Exception as e:
                self.err(e, "Can not insert data")
            return
        self.ok("Rows inserted in the database")
375262
def import_from_parquet(filename_or_fobj, *args, **kwargs):
    """"""
    source = Source.from_file(filename_or_fobj, plugin_name="parquet", mode="rb")

    # TODO: should look into `schema.converted_type` also
    types = OrderedDict(
        [
            (schema.name, PARQUET_TO_ROWS[schema.type])
            for schema in parquet._read_footer(source.fobj).schema
            if schema.type is not None
        ]
    )
    header = list(types.keys())
    table_rows = list(parquet.reader(source.fobj))  # TODO: be lazy

    meta = {"imported_from": "parquet", "source": source}
    return create_table(
        [header] + table_rows, meta=meta, force_types=types, *args, **kwargs
    )
1261722
def create_tables(self, tables):
        """"""
        cursor = self.get_cursor()
        for table in tables:
            columns = mslookup_tables[table]
            try:
                cursor.execute('CREATE TABLE {0}({1})'.format(
                    table, ', '.join(columns)))
            except sqlite3.OperationalError as error:
                print(error)
                print('Warning: Table {} already exists in database, will '
                      'add to existing tables instead of creating '
                      'new.'.format(table))
            else:
                self.conn.commit()
960988
def init_functions(connection):
    """"""
    if settings.CFG["db"]["create_functions"]:
        print("Refreshing SQL functions...")
        for file in path.template_files("../sql/", exts=[".sql"]):
            func = sa.DDL(path.template_str(file))
            LOG.info("Loading: '%s' into database", file)
            connection.execute(func)
            connection.commit()
455221
def insert_function(self, fname, ftype):
        ""
        index = self.insert_id(fname, SharedData.KINDS.FUNCTION, [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.FUNCTION], ftype)
        self.table[index].set_attribute("Params",0)
        return index
639768
def insert_df(self, table_name, df):
        """"""
        df.to_sql(table_name, con=self.own_connection)
514810
def create_table_from_object(self, obj):
        '''
        
        '''
        get_type = lambda item: str(type(item)).split("'")[1]
        if not os.path.exists(os.path.join(self.db_path, obj._TABLE)):
            with gzip.open(os.path.join(self.db_path, obj._TABLE), 'wb') as table_file:
                csv.writer(table_file).writerow(['{col}:{type}'.format(col=elm[0], type=get_type(elm[1]))
                                                 for elm in tuple(obj.__dict__.items())])
            self._tables[obj._TABLE] = self._load_table(obj._TABLE)
307827
def import_from_html(
    filename_or_fobj,
    encoding="utf-8",
    index=0,
    ignore_colspan=True,
    preserve_html=False,
    properties=False,
    table_tag="table",
    row_tag="tr",
    column_tag="td|th",
    *args,
    **kwargs
):
    """"""

    source = Source.from_file(
        filename_or_fobj, plugin_name="html", mode="rb", encoding=encoding
    )

    html = source.fobj.read().decode(source.encoding)
    html_tree = document_fromstring(html)
    tables = html_tree.xpath("//{}".format(table_tag))
    table = tables[index]
    # TODO: set meta's "name" from @id or @name (if available)

    strip_tags(table, "thead")
    strip_tags(table, "tbody")
    row_elements = table.xpath(row_tag)

    table_rows = [
        _get_row(
            row,
            column_tag=column_tag,
            preserve_html=preserve_html,
            properties=properties,
        )
        for row in row_elements
    ]

    if properties:
        table_rows[0][-1] = "properties"

    if preserve_html and kwargs.get("fields", None) is None:
        # The field names will be the first table row, so we need to strip HTML
        # from it even if `preserve_html` is `True` (it's `True` only for rows,
        # not for the header).
        table_rows[0] = list(map(_extract_node_text, row_elements[0]))

    if ignore_colspan:
        max_columns = max(map(len, table_rows))
        table_rows = [row for row in table_rows if len(row) == max_columns]

    meta = {"imported_from": "html", "source": source}
    return create_table(table_rows, meta=meta, *args, **kwargs)
1122858
def load_table(self, table, rows, colnames=None, limit=None):
        

        conn = self.engine.raw_connection()
        cur = conn.cursor()

        colnames = colnames or next(rows)

        cmd = 'INSERT INTO {table} ({colnames}) VALUES ({placeholders})'.format(
            table=self.tables[table],
            colnames=', '.join(colnames),
            placeholders=', '.join([self.placeholder] * len(colnames)))

        cur.executemany(cmd, itertools.islice(rows, limit))
        conn.commit()
272393
def insert(conn, qualified_name: str, column_names, records):
    """"""

    query = create_insert_statement(qualified_name, column_names)

    with conn:
        with conn.cursor(cursor_factory=NamedTupleCursor) as cursor:
            for record in records:
                cursor.execute(query, record)
1090159
def _init_data_line(self, fnc, lnum, line):
        """"""
        fld = re.split(self.sep, line)
        # Lines may contain different numbers of items.
        # The line should have all columns requested by the user.
        if self.usr_max_idx < len(fld):
            self.convert_ints_floats(fld)
            fnc(fld)
        else:
            for fld in enumerate(zip(self.hdr2idx.keys(), fld)):
                print(fld)
            for hdr in self.hdrs_usr:
                print(hdr)
            print('# ITEMS ON A LINE:', len(fld))
            print('MAX USR IDX:', self.usr_max_idx)
            raise Exception("ERROR ON LINE {} IN {}".format(lnum+1, self.fin))