obtain stats payload in batches of pickle format
516705
def pickle_payload(self):
        """"""
        tstamp = int(time.time())
        payload = []

        for item in self.counters:
            payload.append(("%s.%s%s" % (self.rate_prefix, item,
                                         self.rate_suffix),
                            (tstamp,
                             self.counters[item] / self.flush_interval)))
            payload.append(("%s.%s%s" % (self.count_prefix, item,
                                         self.count_suffix),
                            (tstamp, self.counters[item])))
            self.counters[item] = 0

        for key in self.timers:
            if len(self.timers[key]) > 0:
                self.process_timer_key(key, tstamp, payload, pickled=True)
                self.timers[key] = []

        for key in self.gauges:
            payload.append(("%s.%s" % (self.gauge_prefix, key),
                            (tstamp, self.gauges[key])))
            self.gauges[key] = 0

        if payload:
            batched_payload = []
            for batch in self._get_batches(payload):
                if self.debug:
                    print "pickling batch: %r" % batch
                serialized_data = pickle.dumps(batch, protocol=-1)
                length_prefix = struct.pack("!L", len(serialized_data))
                batched_payload.append(length_prefix + serialized_data)
            return batched_payload
        return None
964689
def load_batch(server_context, assay_id, batch_id):
    # type: (ServerContext, int, int) -> Union[Batch, None]
    
    load_batch_url = server_context.build_url('assay', 'getAssayBatch.api')
    loaded_batch = None

    payload = {
        'assayId': assay_id,
        'batchId': batch_id
    }

    headers = {
        'Content-type': 'application/json',
        'Accept': 'text/plain'
    }

    json_body = server_context.make_request(load_batch_url, json_dumps(payload, sort_keys=True), headers=headers)
    if json_body is not None:
        loaded_batch = Batch.from_data(json_body['batch'])

    return loaded_batch
1223448
def stats_per_chunk(chunk):
    """"""

    for block_id in chunk.iter_block():
        try:
            block_counts[block_id] += 1
        except KeyError:
            block_counts[block_id] = 1
1076093
def _get_data_from_rawfile(path_to_data, raw_data_id):
    """"""
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    for raw_dataset in raw_datasets:
        if raw_dataset['handwriting'].raw_data_id == raw_data_id:
            return raw_dataset['handwriting']
    return None
280882
def get_dataloader(train_dataset, train_data_lengths,
                   test_dataset, batch_size):
    
    bucket_num, bucket_ratio = 20, 0.2
    batchify_fn = gluonnlp.data.batchify.Tuple(
        gluonnlp.data.batchify.Pad(axis=0, ret_length=True),
        gluonnlp.data.batchify.Stack(dtype='float32'))
    batch_sampler = gluonnlp.data.sampler.FixedBucketSampler(
        train_data_lengths,
        batch_size=batch_size,
        num_buckets=bucket_num,
        ratio=bucket_ratio,
        shuffle=True)
    train_dataloader = gluon.data.DataLoader(
        dataset=train_dataset,
        batch_sampler=batch_sampler,
        batchify_fn=batchify_fn)
    test_dataloader = gluon.data.DataLoader(
        dataset=test_dataset,
        batch_size=batch_size,
        shuffle=False,
        batchify_fn=batchify_fn)
    return train_dataloader, test_dataloader
413213
def batch(byte_array, funcs):
    
    result = []
    length = bytes_to_int(byte_array[0:4])
    item_size = bytes_to_int(byte_array[4:8])
    for i in range(0, length):
        chunk = byte_array[8+i*item_size:8+(i+1)*item_size]
        for f in funcs:
            f(chunk)
    return result
180542
def packet_get_samples_per_frame(data, fs):
    """"""

    data_pointer = ctypes.c_char_p(data)

    result = _packet_get_nb_frames(data_pointer, ctypes.c_int(fs))
    if result < 0:
        raise OpusError(result)

    return result
1247155
def _get_next_obj(self, length):
        """"""
        data = b''
        while len(data) < length:
            data += self.sock.recv(length - len(data))

        return pickle.loads(data)
647068
def pickle_load(name, extension='.pkl'):
    
    filename = name + extension
    infile = open(filename, 'rb')
    data = pickle.load(infile)
    infile.close()
    return data
669471
def get_statistics(prefix=''):
    
    stats = []
    keys = options.messages.keys()
    keys.sort()
    for key in keys:
        if key.startswith(prefix):
            stats.append('%-7s %s %s' %
                         (options.counters[key], key, options.messages[key]))
    return stats
763359
def _get_batch(self):
    
    batch_indices = np.mod(
        np.array([
            np.arange(head_index, head_index + self._num_steps + 1) for
            head_index in self._head_indices]),
        self._n_flat_elements)

    obs = np.array([
        self._flat_data[indices[:self._num_steps]]
        for indices in batch_indices]).T
    target = np.array([
        self._flat_data[indices[1:self._num_steps + 1]]
        for indices in batch_indices]).T

    if self._random_sampling:
      self._reset_head_indices()
    else:
      self._head_indices = np.mod(
          self._head_indices + self._num_steps, self._n_flat_elements)
    return obs, target
924975
def batch_iter(data, batch_size, num_epochs):
    """"""
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]
213044
def get_cur_batch(items):
    
    batches = []
    for data in items:
        batch = tz.get_in(["metadata", "batch"], data, [])
        batches.append(set(batch) if isinstance(batch, (list, tuple)) else set([batch]))
    combo_batches = reduce(lambda b1, b2: b1.intersection(b2), batches)
    if len(combo_batches) == 1:
        return combo_batches.pop()
    elif len(combo_batches) == 0:
        return None
    else:
        raise ValueError("Found multiple overlapping batches: %s -- %s" % (combo_batches, batches))
460587
def csv_to_pickle(path=ROOT / "raw", clean=False):
    """"""

    for file in os.listdir(path):
        base, ext = os.path.splitext(file)
        if ext != ".csv":
            continue

        LOG(f"converting {file} to pickle")
        df = pd.read_csv(path / file, low_memory=True)
        WRITE_DF(df, path / (base + "." + FILE_EXTENSION), **WRITE_DF_OPTS)
        if clean:
            os.remove(path / file)
            LOG(f"removed {file}")
910833
def stats_flush(self):
        
        while True:
            try:
                eventlet.sleep(self.flush_interval)
                if self.debug:
                    print "seen %d stats so far." % self.stats_seen
                    print "current counters: %s" % self.counters
                if self.pickle_proto:
                    payload = self.pickle_payload()
                    if payload:
                        for batch in payload:
                            self.report_stats(batch)
                else:
                    payload = self.plain_payload()
                    if payload:
                        self.report_stats(payload)
            except: # safety net
                self.logger.critical('Encountered error in stats_flush loop')
1267331
def _get_batches(self, items):
        """"""
        for i in xrange(0, len(items), self.max_batch_size):
            yield items[i:i + self.max_batch_size]
247653
def load_data_batch(self, data_batch):
        """"""
        _load_data(data_batch, self.data_arrays)
        _load_label(data_batch, self.label_arrays)