Get batches data and shuffle.
2827
def get_batches(qp_pairs, batch_size, need_sort=True):
    '''
    
    '''
    if need_sort:
        qp_pairs = sorted(qp_pairs, key=lambda qp: (
            len(qp['passage_tokens']), qp['id']), reverse=True)
    batches = [{'qp_pairs': qp_pairs[i:(i + batch_size)]}
               for i in range(0, len(qp_pairs), batch_size)]
    shuffle(batches)
    return batches
798682
def _get_rows(self, options):
        """"""

        if options["oldsortslice"]:
            rows = copy.deepcopy(self._rows[options["start"]:options["end"]])
        else:
            rows = copy.deepcopy(self._rows)

        # Sort
        if options["sortby"]:
            sortindex = self._field_names.index(options["sortby"])
            # Decorate
            rows = [[row[sortindex]] + row for row in rows]
            # Sort
            rows.sort(reverse=options["reversesort"], key=options["sort_key"])
            # Undecorate
            rows = [row[1:] for row in rows]

        # Slice if necessary
        if not options["oldsortslice"]:
            rows = rows[options["start"]:options["end"]]

        return rows
925001
def get_pretrain_dataset(data, batch_size, num_ctxes, shuffle, use_avg_len,
                         num_buckets, num_parts=1, part_idx=0, prefetch=True):
    """"""
    num_files = len(glob.glob(os.path.expanduser(data)))
    logging.debug('%d files found.', num_files)
    assert num_files >= num_parts, \
        'Number of training files must be greater than the number of partitions'
    split_sampler = nlp.data.SplitSampler(num_files, num_parts=num_parts, part_index=part_idx)
    stream = nlp.data.SimpleDatasetStream(nlp.data.NumpyDataset, data, split_sampler)
    if prefetch:
        stream = nlp.data.PrefetchingStream(stream)

    def get_dataloader(dataset):
        """create data loader based on the dataset chunk"""
        lengths = dataset.get_field('valid_lengths')
        # A batch includes: input_id, masked_id, masked_position, masked_weight,
        #                   next_sentence_label, segment_id, valid_length
        batchify_fn = Tuple(Pad(), Pad(), Pad(), Pad(), Stack(), Pad(), Stack())
        if use_avg_len:
            # sharded data loader
            sampler = nlp.data.FixedBucketSampler(lengths=lengths,
                                                  # batch_size per shard
                                                  batch_size=batch_size,
                                                  num_buckets=num_buckets,
                                                  shuffle=shuffle,
                                                  use_average_length=True,
                                                  num_shards=num_ctxes)
            dataloader = nlp.data.ShardedDataLoader(dataset,
                                                    batch_sampler=sampler,
                                                    batchify_fn=batchify_fn,
                                                    num_workers=num_ctxes)
        else:
            sampler = nlp.data.FixedBucketSampler(lengths,
                                                  batch_size=batch_size * num_ctxes,
                                                  num_buckets=num_buckets,
                                                  ratio=0,
                                                  shuffle=shuffle)
            dataloader = DataLoader(dataset=dataset,
                                    batch_sampler=sampler,
                                    batchify_fn=batchify_fn,
                                    num_workers=1)
        logging.debug('Sampler created for a new dataset:\n%s', sampler.stats())
        return dataloader

    stream = stream.transform(get_dataloader)
    return stream
967881
def batching(size):
    """"""

    if size < 1:
        raise ValueError("batching() size must be at least 1")

    def batching_transducer(reducer):
        return Batching(reducer, size)

    return batching_transducer
810203
def get_input_tensors(batch_size, tf_records, num_repeats=1,
                      shuffle_records=True, shuffle_examples=True,
                      shuffle_buffer_size=None,
                      filter_amount=0.05, random_rotation=True):
    
    print("Reading tf_records from {} inputs".format(len(tf_records)))
    dataset = read_tf_records(
        batch_size,
        tf_records,
        num_repeats=num_repeats,
        shuffle_records=shuffle_records,
        shuffle_examples=shuffle_examples,
        shuffle_buffer_size=shuffle_buffer_size,
        filter_amount=filter_amount,
        interleave=True)
    dataset = dataset.filter(lambda t: tf.equal(tf.shape(t)[0], batch_size))
    dataset = dataset.map(
        functools.partial(batch_parse_tf_example, batch_size))
    if random_rotation:
        dataset = dataset.map(_random_rotation_pyfunc)

    return dataset.make_one_shot_iterator().get_next()
769801
def _batchify(self, data_source):
        """"""
        assert self.cursor < self.num_data, 'DataIter needs reset.'
        # first batch of next epoch with 'roll_over'
        if self.last_batch_handle == 'roll_over' and \
            -self.batch_size < self.cursor < 0:
            assert self._cache_data is not None or self._cache_label is not None, \
                'next epoch should have cached data'
            cache_data = self._cache_data if self._cache_data is not None else self._cache_label
            second_data = self._getdata(
                data_source, end=self.cursor + self.batch_size)
            if self._cache_data is not None:
                self._cache_data = None
            else:
                self._cache_label = None
            return self._concat(cache_data, second_data)
        # last batch with 'pad'
        elif self.last_batch_handle == 'pad' and \
            self.cursor + self.batch_size > self.num_data:
            pad = self.batch_size - self.num_data + self.cursor
            first_data = self._getdata(data_source, start=self.cursor)
            second_data = self._getdata(data_source, end=pad)
            return self._concat(first_data, second_data)
        # normal case
        else:
            if self.cursor + self.batch_size < self.num_data:
                end_idx = self.cursor + self.batch_size
            # get incomplete last batch
            else:
                end_idx = self.num_data
            return self._getdata(data_source, self.cursor, end_idx)
1038366
def batch(self):
        """"""
        _batch = []
        yield _batch
        if _batch:
            new_part = super(type(self), type(self)).empty()
            new_part._from_entities(_batch, _batch[0].__dimension__)
            self.concat(new_part, inplace=True)
476504
def shuffle(self):
        
        order = range(len(self))
        random.shuffle(order)
        for name, data in self.fields.items():
            reindexed = []
            for _, i in enumerate(order):
                reindexed.append(data[i])
            self.fields[name] = reindexed
        return self
911678
def batch(iterable, chunk_size):
    
    i = iter(iterable)
    while True:
        r = list(itertools.islice(i, chunk_size))
        if not r:
            break
        yield r
151157
def batch(samples):
    
    samples = [utils.to_single_data(x) for x in samples]
    sample_order = [dd.get_sample_name(x) for x in samples]
    batch_groups = collections.defaultdict(list)
    for data in samples:
        batch_samples = tuple(data.get("batch_samples", [dd.get_sample_name(data)]))
        batch_groups[(batch_samples, dd.get_phenotype(data))].append(data)

    out = []
    for (batch_samples, phenotype), gsamples in batch_groups.items():
        if len(gsamples) > 1:
            batches = set([])
            for d in gsamples:
                batches |= set(dd.get_batches(d))
            gsamples.sort(key=dd.get_variantcaller_order)
            cur = copy.deepcopy(gsamples[0])
            cur.update({"batch_id": sorted(list(batches))[0] if batches else "_".join(batch_samples),
                        "batch_samples": batch_samples,
                        "variants": {"variantcallers": [dd.get_variantcaller(d) for d in gsamples],
                                     "calls": [d.get("vrn_file") for d in gsamples]}})
            out.append(cur)

    def by_original_order(d):
        return min([sample_order.index(s) for s in d["batch_samples"] if s in sample_order])
    return sorted(out, key=by_original_order)
664416
def get_next_batch(self):
        
        if self.collection_scanner.is_enabled:
            batch = self.collection_scanner.get_new_batch()
            for item in batch:
                base_item = BaseRecord(item)
                self.increase_read()
                self.last_position['last_key'] = item['_key']
                yield base_item
            self.logger.debug('Done reading batch')
        else:
            self.logger.debug('No more batches')
            self.finished = True
939086
def reset(self):
        """"""
        if self.seq is not None and self.shuffle:
            random.shuffle(self.seq)
        if self.last_batch_handle != 'roll_over' or \
            self._cache_data is None:
            if self.imgrec is not None:
                self.imgrec.reset()
            self.cur = 0
            if self._allow_read is False:
                self._allow_read = True
352798
def make_data(n,m):
    """"""
    p = {}
    for i in range(1,m+1):
        for j in range(1,n+1):
            p[i,j] = random.randint(1,10)
    return p
1322487
def shuffle_records(fname):
  """"""
  tf.logging.info("Shuffling records in file %s" % fname)

  # Rename file prior to shuffling
  tmp_fname = fname + ".unshuffled"
  tf.gfile.Rename(fname, tmp_fname)

  reader = tf.python_io.tf_record_iterator(tmp_fname)
  records = []
  for record in reader:
    records.append(record)
    if len(records) % 100000 == 0:
      tf.logging.info("\tRead: %d", len(records))

  random.shuffle(records)

  # Write shuffled records to original file name
  with tf.python_io.TFRecordWriter(fname) as w:
    for count, record in enumerate(records):
      w.write(record)
      if count > 0 and count % 100000 == 0:
        tf.logging.info("\tWriting record: %d" % count)

  tf.gfile.Remove(tmp_fname)
791973
def update_batches() -> None:
    ""
    batches_data  = requests.get(BATCHES_API_URL).json()
    batches_url   = {i["name"]: i["download_url"] for i in batches_data
                     if i["type"] == "file"}
    order_batches = sorted(batches_url, key=int)

    try:
        existing = set(os.listdir(BATCHES_DIR))
    except FileNotFoundError:
        BATCHES_DIR.mkdir(parents=True)
        existing = set()

    def get_batch(name: str) -> None:
        if name in existing and name != order_batches[-1]:
            return

        answer = requests.get(batches_url[name])
        try:
            answer.raise_for_status()
        except requests.RequestException:
            return

        with AtomicFile(BATCHES_DIR / name, "w") as file:
            file.write(answer.text)

    ThreadPool(8).map(get_batch, order_batches)
335214
def shuffled_batches(self, batch_size):
        
        if batch_size >= self.num_envs * self.num_steps:
            yield self
        else:
            rollouts_in_batch = batch_size // self.num_steps

            batch_splits = math_util.divide_ceiling(self.num_envs, rollouts_in_batch)

            indices = list(range(self.num_envs))
            np.random.shuffle(indices)

            for sub_indices in np.array_split(indices, batch_splits):
                yield Trajectories(
                    num_steps=self.num_steps,
                    num_envs=len(sub_indices),
                    # Dont use it in batches for a moment, can be uncommented later if needed
                    # environment_information=[x[sub_indices.tolist()] for x in self.environment_information],
                    environment_information=None,
                    transition_tensors={k: x[:, sub_indices] for k, x in self.transition_tensors.items()},
                    rollout_tensors={k: x[sub_indices] for k, x in self.rollout_tensors.items()},
                    # extra_data does not go into batches
                )
342722
def shuffled_batches(self, batch_size):
        
        if batch_size >= self.size:
            yield self
        else:
            batch_splits = math_util.divide_ceiling(self.size, batch_size)
            indices = list(range(self.size))
            np.random.shuffle(indices)

            for sub_indices in np.array_split(indices, batch_splits):
                yield Transitions(
                    size=len(sub_indices),
                    environment_information=None,
                    # Dont use it in batches for a moment, can be uncommented later if needed
                    # environment_information=[info[sub_indices.tolist()] for info in self.environment_information]
                    transition_tensors={k: v[sub_indices] for k, v in self.transition_tensors.items()}
                    # extra_data does not go into batches
                )