Convert numpy/dask arrays from characters to fixed width bytes.
771520
def char_to_bytes(arr):
    """"""
    if arr.dtype != 'S1':
        raise ValueError("argument must have dtype='S1'")

    if not arr.ndim:
        # no dimension to concatenate along
        return arr

    size = arr.shape[-1]

    if not size:
        # can't make an S0 dtype
        return np.zeros(arr.shape[:-1], dtype=np.string_)

    if isinstance(arr, dask_array_type):
        import dask.array as da

        if len(arr.chunks[-1]) > 1:
            raise ValueError('cannot stacked dask character array with '
                             'multiple chunks in the last dimension: {}'
                             .format(arr))

        dtype = np.dtype('S' + str(arr.shape[-1]))
        return da.map_blocks(_numpy_char_to_bytes, arr,
                             dtype=dtype,
                             chunks=arr.chunks[:-1],
                             drop_axis=[arr.ndim - 1])
    else:
        return StackedBytesArray(arr)
1117452
def rndbytes(size=16, alphabet=""):
    
    x = rndstr(size, alphabet)
    if isinstance(x, six.string_types):
        return x.encode('utf-8')
    return x
326730
def num2bytes(value, size):
    """"""
    res = []
    for _ in range(size):
        res.append(value & 0xFF)
        value = value >> 8
    assert value == 0
    return bytes(bytearray(list(reversed(res))))
240319
def apply_array_ufunc(func, *args, dask='forbidden'):
    """"""
    if any(isinstance(arg, dask_array_type) for arg in args):
        if dask == 'forbidden':
            raise ValueError('apply_ufunc encountered a dask array on an '
                             'argument, but handling for dask arrays has not '
                             'been enabled. Either set the ``dask`` argument '
                             'or load your data into memory first with '
                             '``.load()`` or ``.compute()``')
        elif dask == 'parallelized':
            raise ValueError("cannot use dask='parallelized' for apply_ufunc "
                             'unless at least one input is an xarray object')
        elif dask == 'allowed':
            pass
        else:
            raise ValueError('unknown setting for dask array handling: {}'
                             .format(dask))
    return func(*args)
418478
def from_byte_array(cls, bytes_):
        
        runs = cls._length_run_coding.parse(bytes_)
        pixels = (run.pixels for run in runs.data)
        data = "".join(itertools.chain.from_iterable(pixels))
        return cls(data)
1012048
private static String getDataTemplate(int[] widths) {
        return IntStream.range(0, widths.length)
                .mapToObj(i -> " %" + (i + 1) + "$" + widths[i] + "s  |")
                .reduce((left, right) -> left + " " + right)
                .orElse("");
    }
892162
def bytes_array(self):
        ''''''
        assert len(self.dimensions) == 2, \
            '{}: cannot get value as bytes array!'.format(self.name)
        l, n = self.dimensions
        return [self.bytes[i*l:(i+1)*l] for i in range(n)]
1063902
def convert_bytes(bytes_, K=1 << 10, M=1 << 20, G=1 << 30, T=1 << 40):
    """"""
    fn = float(bytes_)
    if bytes_ >= T:
        return '%.1fT' % (fn / T)
    elif bytes_ >= G:
        return '%.1fG' % (fn / G)
    elif bytes_ >= M:
        return '%.1fM' % (fn / M)
    elif bytes_ >= K:
        return '%.1fK' % (fn / K)
    else:
        return '%d' % bytes_
317361
def get_nbytes(dset):
    
    if 'nbytes' in dset.attrs:
        # look if the dataset has an attribute nbytes
        return dset.attrs['nbytes']
    elif hasattr(dset, 'dtype'):
        # else extract nbytes from the underlying array
        return dset.size * numpy.zeros(1, dset.dtype).nbytes
499999
def encode(df, encoding='utf8', verbosity=1):
    """"""
    if verbosity > 0:
        # pbar_i = 0
        pbar = progressbar.ProgressBar(maxval=df.shape[1])
        pbar.start()
    # encode strings as UTF-8 so they'll work in python2 and python3
    for colnum, col in enumerate(df.columns):
        if isinstance(df[col], pd.Series):
            if verbosity:
                pbar.update(colnum)
            if df[col].dtype in (np.dtype('object'), np.dtype('U'), np.dtype('S')) and any(isinstance(obj, basestring) for obj in df[col]):
                strmask = np.array([isinstance(obj, basestring) for obj in df[col]])
                series = df[col].copy()
                try:
                    series[strmask] = np.char.encode(series[strmask].values.astype('U'))
                except TypeError:
                    print("Unable to convert {} elements starting at position {} in column {}".format(
                          sum(strmask), [i for i, b in enumerate(strmask) if b][:1], col))
                    raise
                except (UnicodeDecodeError, UnicodeEncodeError):
                    try:
                        series[strmask] = np.array([eval(s, {}, {}) for s in series[strmask]])
                    # FIXME: do something different for unicode and decode errors
                    except (SyntaxError, UnicodeDecodeError, UnicodeEncodeError):
                        newseries = []
                        for s in series[strmask]:
                            try:
                                newseries += [s.encode('utf8')]
                            except:
                                print(u'Had trouble encoding {} so used repr to turn it into {}'.format(s, repr(transcode_unicode(s))))
                                # strip all unicode chars are convert to ASCII str
                                newseries += [transcode_unicode(s)]
                        # for dtype('U'): UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 207: ordinal not in r
                        series[strmask] = np.array(newseries).astype('O')
                df[col] = series
                # df[col] = np.array([x.encode('utf8') if isinstance(x, unicode) else x for x in df[col]])

            # WARNING: this takes DAYS for only 100k tweets!
            # series = df[col].copy()
            # for i, value in series.iteritems():   

            #     if isinstance(value, basestring):
            #         series[i] = str(value.encode(encoding))
            # df[col] = series
    if verbosity:
        pbar.finish()
    return df
145329
def to_bytes_or_none(value):
    """"""
    if value == ffi.NULL:
        return None
    elif isinstance(value, ffi.CData):
        return ffi.string(value)
    else:
        raise ValueError('Value must be char[] or NULL')
134794
def to_array(tensor):  # type: (TensorProto) -> np.ndarray[Any]
    
    if tensor.HasField("segment"):
        raise ValueError(
            "Currently not supporting loading segments.")
    if tensor.data_type == TensorProto.UNDEFINED:
        raise ValueError("The data type is not defined.")

    tensor_dtype = tensor.data_type
    np_dtype = mapping.TENSOR_TYPE_TO_NP_TYPE[tensor_dtype]
    storage_type = mapping.TENSOR_TYPE_TO_STORAGE_TENSOR_TYPE[tensor_dtype]
    storage_np_dtype = mapping.TENSOR_TYPE_TO_NP_TYPE[storage_type]
    storage_field = mapping.STORAGE_TENSOR_TYPE_TO_FIELD[storage_type]
    dims = tensor.dims

    if tensor.data_type == TensorProto.STRING:
        utf8_strings = getattr(tensor, storage_field)
        ss = list(s.decode('utf-8') for s in utf8_strings)
        return np.asarray(ss).astype(np_dtype).reshape(dims)

    if tensor.HasField("raw_data"):
        # Raw_bytes support: using frombuffer.
        return np.frombuffer(
            tensor.raw_data,
            dtype=np_dtype).reshape(dims)
    else:
        data = getattr(tensor, storage_field),  # type: Sequence[np.complex64]
        if (tensor_dtype == TensorProto.COMPLEX64
                or tensor_dtype == TensorProto.COMPLEX128):
            data = combine_pairs_to_complex(data)
        return (
            np.asarray(
                data,
                dtype=storage_np_dtype)
            .astype(np_dtype)
            .reshape(dims)
        )
1202270
def decode_array(input_array):
    """"""
    codec, length, param, input_array = parse_header(input_array)
    return codec_dict[codec].decode(input_array, param)
757682
def _reformat(p, buf):
    
    if numpy.ndim(buf) != 1:
        raise ValueError("Buffer ``buf`` must be 1-d.")
    if hasattr(p, 'keys'):
        ans = _gvar.BufferDict(p)
        if ans.size != len(buf):
            raise ValueError(       #
                "p, buf size mismatch: %d, %d"%(ans.size, len(buf)))
        ans = _gvar.BufferDict(ans, buf=buf)
    else:
        if numpy.size(p) != len(buf):
            raise ValueError(       #
                "p, buf size mismatch: %d, %d"%(numpy.size(p), len(buf)))
        ans = numpy.array(buf).reshape(numpy.shape(p))
    return ans
1121052
def codes_get_bytes_array(handle, key, size):
    # type: (cffi.FFI.CData, str, int) -> T.List[int]
    
    values = ffi.new('unsigned char[]', size)
    size_p = ffi.new('size_t *', size)
    _codes_get_bytes(handle, key.encode(ENC), values, size_p)
    return list(values)
170401
def asarray2d(a):
    """"""
    arr = np.asarray(a)
    if arr.ndim == 1:
        arr = arr.reshape(-1, 1)
    return arr
124066
def as_shared_dtype(scalars_or_arrays):
    """"""
    arrays = [asarray(x) for x in scalars_or_arrays]
    # Pass arrays directly instead of dtypes to result_type so scalars
    # get handled properly.
    # Note that result_type() safely gets the dtype from dask arrays without
    # evaluating them.
    out_type = dtypes.result_type(*arrays)
    return [x.astype(out_type, copy=False) for x in arrays]