Runs the batch upload

        :param records: an iterable containing queue entries
358208
def run(self, records):
        
        self_name = type(self).__name__
        for i, batch in enumerate(grouper(records, self.BATCH_SIZE, skip_missing=True), 1):
            self.logger.info('%s processing batch %d', self_name, i)
            try:
                for j, proc_batch in enumerate(grouper(
                        process_records(batch).iteritems(), self.BATCH_SIZE, skip_missing=True), 1):
                    self.logger.info('%s uploading chunk #%d (batch %d)', self_name, j, i)
                    self.upload_records({k: v for k, v in proc_batch}, from_queue=True)
            except Exception:
                self.logger.exception('%s could not upload batch', self_name)
                return
            self.logger.info('%s finished batch %d', self_name, i)
            self.processed_records(batch)
        self.logger.info('%s finished', self_name)
978594
private void processRecordsWithRetries(List<Record> records) {
        for (Record record : records) {
            boolean processedSuccessfully = false;
            for (int i = 0; i < NUM_RETRIES; i++) {
                try {
                    //
                    // Logic to process record goes here.
                    //
                    processSingleRecord(record);

                    processedSuccessfully = true;
                    break;
                } catch (Throwable t) {
                    LOG.warn("Caught throwable while processing record " + record, t);
                }

                // backoff if we encounter an exception.
                try {
                    Thread.sleep(BACKOFF_TIME_IN_MILLIS);
                } catch (InterruptedException e) {
                    LOG.debug("Interrupted sleep", e);
                }
            }

            if (!processedSuccessfully) {
                LOG.error("Couldn't process record " + record + ". Skipping the record.");
            }
        }
    }
497210
def documents_upload(ctx, max_threads, files):
    """"""
    if len(files) == 0:
        click.echo(ctx.get_help())
        return

    files = list(files)

    bar = click.progressbar(length=sum([_file_size(x) for x in files]), label="Uploading... ")
    run_via_threadpool(
        ctx.obj["API"].Documents.upload,
        files,
        {"progressbar": bar},
        max_threads=max_threads,
        graceful_exit=False,
    )
609450
def run(self):
        ''''''
        for line in iter(self._fd.readline, ''):
            self._queue.put(line)
284355
@Override
	public void writeRecord(Record record) throws IOException {
		try {
			for (int x = 0; x < record.getNumFields(); x++) {
				Value temp = record.getField(x, fieldClasses[x]);
				addValue(x + 1, temp);
			}
			upload.addBatch();
			batchCount++;
			if(batchCount >= batchInterval) {
				upload.executeBatch();
				batchCount = 0;
			}
		} catch (SQLException sqe) {
			throw new IllegalArgumentException("writeRecord() failed:\t", sqe);
		} catch (IllegalArgumentException iae) {
			throw new IllegalArgumentException("writeRecord() failed:\t", iae);
		}
	}
66209
def extend(self, records):
        
        fields = self.fields
        for record in records:
            record = _cast_record_to_str_tuple(record, fields)
            self._records.append(record)
630802
def iterRecords(self):
        """"""
        if self.numRecords is None:
            self.__dbfHeader()
        f = self.__getFileObj(self.dbf)
        f.seek(self.__dbfHdrLength)
        for i in xrange(self.numRecords):
            r = self.__record()
            if r:
                yield r
592254
def flush_queue(self):
        """"""
        records = []

        while not self.queue.empty() and len(records) < self.batch_size:
            records.append(self.queue.get())

        if records:
            self.send_records(records)
            self.last_flush = time.time()
299852
def run_to_queue(self, queue, conn, options=None):
    """"""
    if options is None:
      # Default options.
      offset = None
      limit = None
      keys_only = None
    else:
      # Capture options we need to simulate.
      offset = options.offset
      limit = options.limit
      keys_only = options.keys_only

      # Cursors are supported for certain orders only.
      if (options.start_cursor or options.end_cursor or
          options.produce_cursors):
        names = set()
        if self.__orders is not None:
          names = self.__orders._get_prop_names()
        if '__key__' not in names:
          raise datastore_errors.BadArgumentError(
              '_MultiQuery with cursors requires __key__ order')

    # Decide if we need to modify the options passed to subqueries.
    # NOTE: It would seem we can sometimes let Cloud Datastore handle
    # the offset natively, but this would thwart the duplicate key
    # detection, so we always have to emulate the offset here.
    # We can set the limit we pass along to offset + limit though,
    # since that is the maximum number of results from a single
    # subquery we will ever have to consider.
    modifiers = {}
    if offset:
      modifiers['offset'] = None
      if limit is not None:
        modifiers['limit'] = min(_MAX_LIMIT, offset + limit)
    if keys_only and self.__orders is not None:
      modifiers['keys_only'] = None
    if modifiers:
      options = QueryOptions(config=options, **modifiers)

    if offset is None:
      offset = 0

    if limit is None:
      limit = _MAX_LIMIT

    if self.__orders is None:
      # Run the subqueries sequentially; there is no order to keep.
      keys_seen = set()
      for subq in self.__subqueries:
        if limit <= 0:
          break
        subit = tasklets.SerialQueueFuture('_MultiQuery.run_to_queue[ser]')
        subq.run_to_queue(subit, conn, options=options)
        while limit > 0:
          try:
            batch, index, result = yield subit.getq()
          except EOFError:
            break
          if keys_only:
            key = result
          else:
            key = result._key
          if key not in keys_seen:
            keys_seen.add(key)
            if offset > 0:
              offset -= 1
            else:
              limit -= 1
              queue.putq((None, None, result))
      queue.complete()
      return

    # This with-statement causes the adapter to set _orig_pb on all
    # entities it converts from protobuf.
    # TODO: Does this interact properly with the cache?
    with conn.adapter:
      # Start running all the sub-queries.
      todo = []  # List of (subit, dsquery) tuples.
      for subq in self.__subqueries:
        dsquery = subq._get_query(conn)
        subit = tasklets.SerialQueueFuture('_MultiQuery.run_to_queue[par]')
        subq.run_to_queue(subit, conn, options=options, dsquery=dsquery)
        todo.append((subit, dsquery))

      # Create a list of (first-entity, subquery-iterator) tuples.
      state = []  # List of _SubQueryIteratorState instances.
      for subit, dsquery in todo:
        try:
          thing = yield subit.getq()
        except EOFError:
          continue
        else:
          state.append(_SubQueryIteratorState(thing, subit, dsquery,
                                              self.__orders))

      # Now turn it into a sorted heap.  The heapq module claims that
      # calling heapify() is more efficient than calling heappush() for
      # each item.
      heapq.heapify(state)

      # Repeatedly yield the lowest entity from the state vector,
      # filtering duplicates.  This is essentially a multi-way merge
      # sort.  One would think it should be possible to filter
      # duplicates simply by dropping other entities already in the
      # state vector that are equal to the lowest entity, but because of
      # the weird sorting of repeated properties, we have to explicitly
      # keep a set of all keys, so we can remove later occurrences.
      # Note that entities will still be sorted correctly, within the
      # constraints given by the sort order.
      keys_seen = set()
      while state and limit > 0:
        item = heapq.heappop(state)
        batch = item.batch
        index = item.index
        entity = item.entity
        key = entity._key
        if key not in keys_seen:
          keys_seen.add(key)
          if offset > 0:
            offset -= 1
          else:
            limit -= 1
            if keys_only:
              queue.putq((batch, index, key))
            else:
              queue.putq((batch, index, entity))
        subit = item.iterator
        try:
          batch, index, entity = yield subit.getq()
        except EOFError:
          pass
        else:
          item.batch = batch
          item.index = index
          item.entity = entity
          heapq.heappush(state, item)
      queue.complete()
121887
public void flush() {
    try {
      ArrayList<Batch> batches = this.incomplete.all();
      int numOutstandingRecords = 0;
      for (Batch batch: batches) {
        numOutstandingRecords += batch.getRecords().size();
      }
      LOG.debug ("Flush called on {} batches with {} records total", batches.size(), numOutstandingRecords);
      for (Batch batch: batches) {
        batch.await();
      }
    } catch (Exception e) {
      LOG.error ("Error happened while flushing batches");
    }
  }
296018
def start_recording(recorded_events_queue=None):
    
    recorded_events_queue = recorded_events_queue or _queue.Queue()
    global _recording
    _recording = (recorded_events_queue, hook(recorded_events_queue.put))
    return _recording
1026073
def run(self):
        
        processes = []
        progress_queue = ProgressQueue(Queue())
        num_chunks = ParallelChunkProcessor.determine_num_chunks(self.config.upload_bytes_per_chunk,
                                                                 self.local_file.size)
        work_parcels = ParallelChunkProcessor.make_work_parcels(self.config.upload_workers, num_chunks)
        for (index, num_items) in work_parcels:
            processes.append(self.make_and_start_process(index, num_items, progress_queue))
        wait_for_processes(processes, num_chunks, progress_queue, self.watcher, self.local_file)
858015
def records_for_rebuild(batch_size = 1000)
      transaction do
        if use_fast_batches?
          offset = 0
          while (rows = where([ "#{table_name}.id > ?", offset ]).limit(batch_size).all).any?
            offset = rows.last.id
            yield rows, offset
          end
        else
          order = "#{primary_key} ASC" # fixes #212
          0.step(self.count, batch_size) do |offset|
            yield scoped.limit(batch_size).offset(offset).order(order).all, offset
          end
        end
      end
    end
852773
def push_all(self, record_shard_pairs):
        
        # Faster than inserting one at a time; the heap is sorted once after all inserts.
        for record, shard in record_shard_pairs:
            item = heap_item(self.clock, record, shard)
            self.heap.append(item)
        heapq.heapify(self.heap)
760923
def create_document_batches(jsonlines, id_field, max_batch_size=CLOUDSEARCH_MAX_BATCH_SIZE):
    
    batch = []
    fixed_initial_size = 2

    def create_entry(line):
        try:
            record = json.loads(line)
        except:
            raise ValueError('Could not parse JSON from: %s' % line)
        key = record[id_field]
        return '{"type":"add","id":%s,"fields":%s}' % (json.dumps(key), line)

    current_size = fixed_initial_size
    for line in jsonlines:
        entry = create_entry(line)
        entry_size = len(entry) + 1

        if max_batch_size > (current_size + entry_size):
            current_size += entry_size
            batch.append(entry)
        else:
            yield '[' + ','.join(batch) + ']'
            batch = [entry]
            current_size = fixed_initial_size + entry_size

    if batch:
        yield '[' + ','.join(batch) + ']'
1355389
def run(self):
        
        progress_printer = ProgressPrinter(self.different_items.total_items(), msg_verb='sending')
        upload_settings = UploadSettings(self.config, self.remote_store.data_service, progress_printer,
                                         self.project_name_or_id, self.file_upload_post_processor)
        project_uploader = ProjectUploader(upload_settings)
        project_uploader.run(self.local_project)
        progress_printer.finished()
851157
def _bulk_size_generator(num_records, bulk_size, active):
    
    while active and num_records > 0:
        req_size = min(num_records, bulk_size)
        num_records -= req_size
        yield req_size