Iterate over train batches.

        Arguments:
            max_yield (int): Max number of batches to iterate over in this
                cycle. Setting this avoids iter_train_batches returning too
                much data at once.
18649
def iter_train_batches(self, max_yield=999):
        

        for ev, sample_batch in self._augment_with_replay(
                self.sample_tasks.completed_prefetch(
                    blocking_wait=True, max_yield=max_yield)):
            sample_batch.decompress_if_needed()
            self.batch_buffer.append(sample_batch)
            if sum(b.count
                   for b in self.batch_buffer) >= self.train_batch_size:
                train_batch = self.batch_buffer[0].concat_samples(
                    self.batch_buffer)
                yield train_batch
                self.batch_buffer = []

            # If the batch was replayed, skip the update below.
            if ev is None:
                continue

            # Put in replay buffer if enabled
            if self.replay_buffer_num_slots > 0:
                if len(self.replay_batches) < self.replay_buffer_num_slots:
                    self.replay_batches.append(sample_batch)
                else:
                    self.replay_batches[self.replay_index] = sample_batch
                    self.replay_index += 1
                    self.replay_index %= self.replay_buffer_num_slots

            ev.set_weights.remote(self.broadcasted_weights)
            self.num_weight_syncs += 1
            self.num_sent_since_broadcast += 1

            # Kick off another sample request
            self.sample_tasks.add(ev, ev.sample.remote())
1123104
def grouped_impl(wrap, size, sequence):
    
    iterator = iter(sequence)
    try:
        while True:
            batch = islice(iterator, size)
            yield list(chain((wrap(next(batch)),), batch))
    except StopIteration:
        return
342722
def shuffled_batches(self, batch_size):
        
        if batch_size >= self.size:
            yield self
        else:
            batch_splits = math_util.divide_ceiling(self.size, batch_size)
            indices = list(range(self.size))
            np.random.shuffle(indices)

            for sub_indices in np.array_split(indices, batch_splits):
                yield Transitions(
                    size=len(sub_indices),
                    environment_information=None,
                    # Dont use it in batches for a moment, can be uncommented later if needed
                    # environment_information=[info[sub_indices.tolist()] for info in self.environment_information]
                    transition_tensors={k: v[sub_indices] for k, v in self.transition_tensors.items()}
                    # extra_data does not go into batches
                )
50241
def _iter(self):
        
        keys = self.keys()
        while keys:
            key = keys[0]
            yield key
            keys = keys[1:]
198005
def iterate(self, max_iter=None):
        
        context = zmq.Context()

        if six.PY2:
            warnings.warn('zmq_stream cannot preserve numpy array alignment '
                          'in Python 2', RuntimeWarning)

        try:
            socket = context.socket(zmq.PAIR)

            port = socket.bind_to_random_port('tcp://*',
                                              min_port=self.min_port,
                                              max_port=self.max_port,
                                              max_tries=self.max_tries)
            terminate = mp.Event()

            worker = mp.Process(target=SafeFunction(zmq_worker),
                                args=[port, self.streamer, terminate],
                                kwargs=dict(copy=self.copy,
                                            max_iter=max_iter))

            worker.daemon = True
            worker.start()

            # Yield from the queue as long as it's open
            while True:
                yield zmq_recv_data(socket)

        except StopIteration:
            pass

        except:
            # pylint: disable-msg=W0702
            six.reraise(*sys.exc_info())

        finally:
            terminate.set()
            worker.join(self.timeout)
            if worker.is_alive():
                worker.terminate()
            context.destroy()
156944
def batch(self, requests):
        

        for request in requests:
            if 'body' in request:
                request['body'] = urlencode(request['body'])

        def _grouper(complete_list, n=1):
            """
            Batches a list into constant size chunks.

            :param complete_list: A input list (not a generator).
            :param n: The size of the chunk.

            Adapted from <http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python>
            """

            for i in range(0, len(complete_list), n):
                yield complete_list[i:i + n]

        responses = []

        # Maximum batch size for Facebook is 50 so split up requests
        # https://developers.facebook.com/docs/graph-api/making-multiple-requests/#limits
        for group in _grouper(requests, 50):
            responses += self.post(
                batch=json.dumps(group)
            )

        for response, request in zip(responses, requests):

            # Facilitate for empty Graph API responses.
            #
            # https://github.com/jgorset/facepy/pull/30
            if not response:
                yield None
                continue

            try:
                yield self._parse(response['body'])
            except FacepyError as exception:
                exception.request = request
                yield exception
664416
def get_next_batch(self):
        
        if self.collection_scanner.is_enabled:
            batch = self.collection_scanner.get_new_batch()
            for item in batch:
                base_item = BaseRecord(item)
                self.increase_read()
                self.last_position['last_key'] = item['_key']
                yield base_item
            self.logger.debug('Done reading batch')
        else:
            self.logger.debug('No more batches')
            self.finished = True
794728
def data_collector(iterable, def_buf_size=5242880):
    
    buf = b''
    for data in iterable:
        buf += data
        if len(buf) >= def_buf_size:
            output = buf[:def_buf_size]
            buf = buf[def_buf_size:]
            yield output
    if len(buf) > 0:
        yield buf
1226453
def on_batch_end(self, last_loss:Tensor, iteration:int, **kwargs)->None:
        ""
        if iteration == 0: return
        self._update_batches_if_needed()
        if iteration % self.loss_iters == 0: self._write_training_loss(iteration=iteration, last_loss=last_loss)
        if iteration % self.hist_iters == 0: self._write_weight_histograms(iteration=iteration)
876287
def max_cycles(self, num):
        
        kwargs = {kw: (arg[num] if isinstance(arg, Palette) else arg)
                  for kw, arg in self.kwargs.items()}
        return self(max_cycles=num, **kwargs)
281675
def fasta_iterator(first_line, next_lines)
      current_fasta_header = first_line.chomp
      next_lines.each_slice(batch_size).with_index do |slice, i|
        fasta_mapper = []
        input_set = Set.new

        slice.each do |line|
          line.chomp!
          if fasta? line
            current_fasta_header = line
          else
            fasta_mapper << [current_fasta_header, line]
            input_set << line
          end
        end

        yield(input_set.to_a, i, fasta_mapper)
      end
    end
399190
def randiter(iterable):
    '''
'''

    iter_len = len(iterable)

    for index in xrandrange(iter_len):
        yield iterable[index]
346050
def chunks(iterable, n):
    
    for i in range(0, len(iterable), n):
        yield iterable[i:i + n]
465714
def next(self) -> mx.io.DataBatch:
        
        if self.iter_next():
            return self.next_batch
        raise StopIteration
140822
def iter(self):
        """"""
        if self.patterns:
            patterns = list(self.patterns)
            for pattern in patterns:
                yield pattern
942069
def i_batch(items, size):
    
    for items_batch in iter(lambda: tuple(islice(items, size)),
                            tuple()):
        yield items_batch
1293814
def cub200_iterator(data_path, batch_k, batch_size, data_shape):
    """"""
    return (CUB200Iter(data_path, batch_k, batch_size, data_shape, is_train=True),
            CUB200Iter(data_path, batch_k, batch_size, data_shape, is_train=False))