Perform batch import into database.
1951
private void batchImport() {
        m_jdbcTemplate.batchUpdate(createInsertQuery(),
                                   new BatchPreparedStatementSetter() {
                                       public void setValues(
                                               PreparedStatement ps, int i)
                                               throws SQLException {
                                           int j = 1;
                                           for (String property : m_propertyNames) {
                                               Map params = (Map) m_batch.get(
                                                       i);
                                               ps.setObject(j++, params.get(
                                                       property));
                                           }
                                       }

                                       public int getBatchSize() {
                                           return m_batch.size();
                                       }
                                   });
    }
353759
def db(ctx):
    """"""

    from hfos import database
    database.initialize(ctx.obj['dbhost'], ctx.obj['dbname'])
    ctx.obj['db'] = database
1269659
function Batch(db, options) {
  this.options = options || {};
  this.ops = [];
  this.db = db;
  this.written = false;
}
1107678
def run_batch(args: dict) -> int:
    """"""

    batcher.run_project(
        project_directory=args.get('project_directory'),
        log_path=args.get('logging_path'),
        output_directory=args.get('output_directory'),
        shared_data=load_shared_data(args.get('shared_data_path'))
    )
    return 0
881798
def update(database, table, key):
    """"""
    field = random.randrange(10)
    value = ''.join(random.choice(string.printable) for i in range(100))
    with database.batch() as batch:
        batch.update(table=table, columns=('id', 'field%d' % field),
                     values=[(key, value)])
416719
def setup_migrate(app):
    """"""
    directory = path.join(path.dirname(__file__), 'migrations')
    migrate.init_app(app, db, directory=directory)
1110551
def import_datasources(path, sync, recursive):
    """"""
    sync_array = sync.split(',')
    p = Path(path)
    files = []
    if p.is_file():
        files.append(p)
    elif p.exists() and not recursive:
        files.extend(p.glob('*.yaml'))
        files.extend(p.glob('*.yml'))
    elif p.exists() and recursive:
        files.extend(p.rglob('*.yaml'))
        files.extend(p.rglob('*.yml'))
    for f in files:
        logging.info('Importing datasources from file %s', f)
        try:
            with f.open() as data_stream:
                dict_import_export.import_from_dict(
                    db.session,
                    yaml.safe_load(data_stream),
                    sync=sync_array)
        except Exception as e:
            logging.error('Error when importing datasources from file %s', f)
            logging.error(e)
703676
def bulk_csv_import_mongo(csvfile, database_name, collection_name,
                          delete_collection_before_import=False):
    """method can be insert or update"""

    l = []
    response_dict = {}
    try:
        mongodb_client_url = getattr(settings, 'MONGODB_CLIENT',
                                 'mongodb://localhost:27017/')
        mc = MongoClient(mongodb_client_url,document_class=OrderedDict)
        db = mconnection[database_name]
        collection = db[collection_name]

        if delete_collection_before_import:
            myobjectid = collection.remove({})

        # open the csv file.
        csvhandle = csv.reader(open(csvfile._get_path(), 'rb'), delimiter=',')

        rowindex = 0
        errors = 0
        error_list = []
        success = 0
        for row in csvhandle:

            if rowindex == 0:
                column_headers = row
                cleaned_headers = []
                for c in column_headers:
                    c = c.replace(".", "")
                    c = c.replace("$", "-")
                    c = c.replace(" ", "_")
                    cleaned_headers.append(c)
            else:

                record = OrderedDict(zip(cleaned_headers, row))
                # if there is no values, skip the key value pair
                kwargs = OrderedDict()

                # Only populate fields that are not blank.
                for k, v in record.items():
                    if v:
                        if v.isdigit():
                            kwargs[k] = int(v)
                        else:
                            kwargs[k] = v
                try:

                    myobjectid = collection.insert(kwargs)
                    success += 1
                except:
                    error_message = "Error on row " + \
                        rowindex + ". " + str(sys.exc_info())
                    error_list.append(str(sys.exc_info()))

            rowindex += 1

        if error_list:
            response_dict = {}
            response_dict['num_rows_imported'] = rowindex
            response_dict['num_rows_errors'] = len(error_list)
            response_dict['errors'] = error_list
            response_dict['code'] = 400
            response_dict['message'] = "Completed with errors"
        else:

            response_dict = {}
            response_dict['num_rows_imported'] = success
            response_dict['code'] = 200
            response_dict['message'] = "Completed."
        return response_dict

    except:
        # print "Error reading from Mongo"
        # print str(sys.exc_info())
        response_dict['num_results'] = 0
        response_dict['code'] = 400
        response_dict['type'] = "Error"
        response_dict['results'] = []
        response_dict['message'] = str(sys.exc_info())
    return response_dict
1213952
def init_db(self, app, entry_point_group='invenio_db.models', **kwargs):
        """"""
        # Setup SQLAlchemy
        app.config.setdefault(
            'SQLALCHEMY_DATABASE_URI',
            'sqlite:///' + os.path.join(app.instance_path, app.name + '.db')
        )
        app.config.setdefault('SQLALCHEMY_ECHO', False)

        # 
        database = kwargs.get('db', db)
        database.init_app(app)

        # Initialize versioning support.
        self.init_versioning(app, database, kwargs.get('versioning_manager'))

        # Initialize model bases
        if entry_point_group:
            for base_entry in pkg_resources.iter_entry_points(
                    entry_point_group):
                base_entry.load()

        # All models should be loaded by now.
        sa.orm.configure_mappers()
        # Ensure that versioning classes have been built.
        if app.config['DB_VERSIONING']:
            manager = self.versioning_manager
            if manager.pending_classes:
                if not versioning_models_registered(manager, database.Model):
                    manager.builder.configure_versioned_classes()
            elif 'transaction' not in database.metadata.tables:
                manager.declarative_base = database.Model
                manager.create_transaction_model()
                manager.plugins.after_build_tx_class(manager)
1373903
def db_insert(self):
        
        assert self.has_db
        # Connect to MongoDb and get the collection.
        coll = self.manager.db_connector.get_collection()
        print("Mongodb collection %s with count %d", coll, coll.count())

        start = time.time()
        for work in self:
            for task in work:
                results = task.get_results()
                pprint(results)
                results.update_collection(coll)
            results = work.get_results()
            pprint(results)
            results.update_collection(coll)
        print("MongoDb update done in %s [s]" % time.time() - start)

        results = self.get_results()
        pprint(results)
        results.update_collection(coll)

        # Update the pickle file to save the mongo ids.
        self.pickle_dump()

        for d in coll.find():
            pprint(d)
1332247
public static int importData(final DataSet dataset, final int offset, final int count, final PreparedStatement stmt,
            final Try.BiConsumer<? super PreparedStatement, ? super Object[], SQLException> stmtSetter) throws UncheckedSQLException {
        return importData(dataset, offset, count, stmt, 200, 0, stmtSetter);
    }
147806
def import_(self, data):
        """"""
        return self.__import(json.loads(data, **self.kwargs))
1254037
def _select_batched(self, table, cols, num_rows, limit, queries_per_batch=3, execute=True):
        """"""
        # Execute select queries in small batches to avoid connection timeout
        commands, offset = [], 0
        while num_rows > 0:
            # Use number of rows as limit if num_rows < limit
            _limit = min(limit, num_rows)

            # Execute select_limit query
            commands.append(self._select_limit_statement(table, cols=cols, offset=offset, limit=limit))
            offset += _limit
            num_rows += -_limit

        # Execute commands
        if execute:
            rows = []
            til_reconnect = queries_per_batch
            for c in commands:
                if til_reconnect == 0:
                    self.disconnect()
                    self.reconnect()
                    til_reconnect = queries_per_batch
                rows.extend(self.fetch(c, False))
                til_reconnect += -1
            del commands
            return rows
        # Return commands
        else:
            return commands
988633
public function import_database() {
        $parser = $this->get_parser();
        if (!xml_parse($parser, $this->data, true)) {
            throw new dbtransfer_exception('malformedxmlexception');
        }
        xml_parser_free($parser);
    }
869581
def build_if_needed(db):
    """"""
    if len(db.engine.table_names()) == 0:
        # import all classes here
        from my_site.models.tables.user import User

        db.create_all()
788708
protected void initializeBatch(String databaseName, String table)
      throws SQLException {
    this.insertStmtPrefix = createInsertStatementStr(databaseName, table);
    this.insertPstmtForFixedBatch =
        this.conn.prepareStatement(createPrepareStatementStr(this.batchSize));
    LOG.info(String.format("Initialized for %s insert " + this, (this.batchSize > 1) ? "batch" : ""));
  }
881422
def batch_update(self, command, rows):
        
        request = {
            "database": {
                "alias": self.__options['dbAlias']
            },
            "batchUpdate": {
                "command": command,
                "rows": rows,
                "shardKey": self.__options.get('shardKey'),
            }
        }
        dr = self.__app.native_api_call('db', 'batch-update', request, self.__options, False)
        return json.loads(dr.text)