Convert blocks to local ndarray
18403
def toarray(self):
        
        if self.mode == 'spark':
            return self.values.unchunk().toarray()

        if self.mode == 'local':
            return self.values.unchunk()
369024
def extract_locals(trcback):
    

    output = []
    stack = extract_stack(get_inner_most_frame(trcback))
    for frame, file_name, line_number, name, context, index in stack:
        args_names, nameless, keyword = extract_arguments(frame)
        arguments, nameless_args, keyword_args, locals = OrderedDict(), [], {}, {}
        for key, data in frame.f_locals.iteritems():
            if key == nameless:
                nameless_args = map(repr, frame.f_locals.get(nameless, ()))
            elif key == keyword:
                keyword_args = dict((arg, repr(value)) for arg, value in frame.f_locals.get(keyword, {}).iteritems())
            elif key in args_names:
                arguments[key] = repr(data)
            else:
                locals[key] = repr(data)
        output.append(((name, file_name, line_number), (arguments, nameless_args, keyword_args, locals)))
    return output
1257748
def _image2array(filepath):
    '''
    
    '''
    im = Pimage.open(filepath).convert('RGB')
    (width, height) = im.size
    _r = np.array(list(im.getdata(0)))/255.0
    _g = np.array(list(im.getdata(1)))/255.0
    _b = np.array(list(im.getdata(2)))/255.0
    _r = _r.reshape((height, width))
    _g = _g.reshape((height, width))
    _b = _b.reshape((height, width))
    return _r, _g, _b
1041995
def convert_parameter_shape(pb):
    """"""
    if len(pb.network) != 1:
        raise ValueError(
            "NNP with more then a single network is currently not supported")
    net = pb.network[0]
    batch_norm_constants = []
    for f in net.function:
        if f.type == "BatchNormalization":
            # BatchNormalization in ONNX requires the scale, bias, mean, and variance input to be
            # one dimensional (https://github.com/onnx/onnx/blob/master/docs/Operators.md#batchnormalization).
            # However in NNabla these input must have a specific shape that matches the input shape.
            # For example if the input shape is (1,3,3,3), the above variables must have the shape (1,3,1,1) and not (3).
            # (1,3,1,1) is actually the same as a one-dimensional tensor of size 3,
            # but NNabla's check currently does not allow this.
            # Thus, we convert the shape of the above input so we can pass NNabla's check.
            # If NNabla lightens the shape check, we should be able to remove this conversion.
            # copy all input names for scale, bias, mean, variance
            batch_norm_constants.extend(f.input[1:5])

    # This loop should be fairly slow since we loop through all variables and parameters per constant
    for c in batch_norm_constants:
        # Reshape all BatchNormalization constant inputs assuming the size is (1,size,1,1)
        for v in net.variable:
            if v.name == c:
                size = v.shape.dim[0]
                del v.shape.dim[:]
                v.shape.dim.extend([1, size, 1, 1])
                break
        for p in pb.parameter:
            if p.variable_name == c:
                size = p.shape.dim[0]
                del p.shape.dim[:]
                p.shape.dim.extend([1, size, 1, 1])
                break
239744
public static void convertTranSrc(DMatrixRMaj src , DMatrixRBlock dst )
    {
        if( src.numRows != dst.numCols || src.numCols != dst.numRows )
            throw new IllegalArgumentException("Incompatible matrix shapes.");

        for( int i = 0; i < dst.numRows; i += dst.blockLength ) {
            int blockHeight = Math.min( dst.blockLength , dst.numRows - i);

            for( int j = 0; j < dst.numCols; j += dst.blockLength ) {
                int blockWidth = Math.min( dst.blockLength , dst.numCols - j);

                int indexDst = i*dst.numCols + blockHeight*j;
                int indexSrc = j*src.numCols + i;

                for( int l = 0; l < blockWidth; l++ ) {
                    int rowSrc = indexSrc + l*src.numCols;
                    int rowDst = indexDst + l;
                    for( int k = 0; k < blockHeight; k++ , rowDst += blockWidth ) {
                        dst.data[ rowDst ] = src.data[rowSrc++];
                    }
                }
            }
        }
    }
126720
def scatter_blocks_2d(x, indices, shape):
  """"""
  x_shape = common_layers.shape_list(x)
  # [length, batch, heads, dim]
  x_t = tf.transpose(
      tf.reshape(x, [x_shape[0], x_shape[1], -1, x_shape[-1]]), [2, 0, 1, 3])
  x_t_shape = common_layers.shape_list(x_t)
  indices = tf.reshape(indices, [-1, 1])
  scattered_x = tf.scatter_nd(indices, x_t, x_t_shape)
  scattered_x = tf.transpose(scattered_x, [1, 2, 0, 3])
  return tf.reshape(scattered_x, shape)
811938
def _convert_to_ndarray(self, data):
        """"""
        if data.__class__.__name__ != "DataFrame":
            raise Exception(f"data is not a DataFrame but {data.__class__.__name__}.")
        shape_ndarray = (self._height, self._width, data.shape[1])
        data_ndarray = data.values.reshape(shape_ndarray)
        return data_ndarray
267826
def blocks(self):
        

        for block_addr, block in self._local_blocks.items():
            try:
                yield self._get_block(block_addr, size=block.size,
                                      byte_string=block.bytestr if isinstance(block, BlockNode) else None)
            except (SimEngineError, SimMemoryError):
                pass
1275062
def residual_block(x, hparams):
  """"""
  k = (hparams.kernel_height, hparams.kernel_width)
  dilations_and_kernels = [((1, 1), k) for _ in range(3)]
  y = common_layers.subseparable_conv_block(
      x,
      hparams.hidden_size,
      dilations_and_kernels,
      padding="SAME",
      separability=0,
      name="residual_block")
  x = common_layers.layer_norm(x + y, hparams.hidden_size, name="lnorm")
  return tf.nn.dropout(x, 1.0 - hparams.dropout)
1042016
def to_numpy_array(multiprocessing_array, shape, dtype):
    

    return np.frombuffer(multiprocessing_array.get_obj(),
                         dtype = dtype).reshape(shape)
879443
def conv1d_block(inputs, filters, dilation_rates_and_kernel_sizes, **kwargs):
  """"""
  return conv_block_internal(conv1d, inputs, filters,
                             dilation_rates_and_kernel_sizes, **kwargs)
1001716
def tolocal(self):
        
        from thunder.images.readers import fromarray

        if self.mode == 'local':
            logging.getLogger('thunder').warn('images already in local mode')
            pass

        return fromarray(self.toarray())
1032203
def aggregate(self, block_size):
        '''
        
        '''
        raster2 = block_reduce(self.raster, block_size, func=np.ma.sum)
        geot = self.geot
        geot = (geot[0], block_size[0] * geot[1], geot[2], geot[3], geot[4],
                block_size[1] * geot[-1])
        return GeoRaster(raster2, geot, nodata_value=self.nodata_value,\
                        projection=self.projection, datatype=self.datatype)
982172
def block_sep1(self, Y):
        """"""

        Y1 = Y[..., self.cri.M:]

        # If cri.Cd > 1 (multi-channel dictionary), we need to undo the
        # reshape performed in block_cat
        if self.cri.Cd > 1:
            shp = list(Y1.shape)
            shp[self.cri.axisM] = self.cri.dimN
            shp[self.cri.axisC] = self.cri.Cd
            Y1 = Y1.reshape(shp)

        # Axes are swapped here for similar reasons to those
        # motivating swapping in cbpdn.ConvTwoBlockCnstrnt.block_sep0
        Y1 = np.swapaxes(Y1[..., np.newaxis], self.cri.axisM, -1)

        return Y1
674047
def ensure_ndarray(A, shape=None, uniform=None, ndim=None, size=None, dtype=None, kind=None):
    

    """
    if not isinstance(A, np.ndarray):
        try:
            A = np.array(A)
        except:
            raise AssertionError('Given argument cannot be converted to an ndarray:\n'+str(A))
    assert_array(A, shape=shape, uniform=uniform, ndim=ndim, size=size, dtype=dtype, kind=kind)
    return A
199731
def get_blen_array(data, blen=None):
    """"""

    if blen is None:

        if hasattr(data, 'chunklen'):
            # bcolz carray
            return data.chunklen

        elif hasattr(data, 'chunks') and \
                hasattr(data, 'shape') and \
                hasattr(data.chunks, '__len__') and \
                hasattr(data.shape, '__len__') and \
                len(data.chunks) == len(data.shape):
            # something like h5py dataset
            return data.chunks[0]

        else:
            # fall back to something simple, ~1Mb chunks
            row = np.asarray(data[0])
            return max(1, (2**20) // row.nbytes)

    else:
        return blen
1229423
def get_frames(self):
        ""
        nb_frames = self.input_totalframes // self.output_blocksize

        if self.input_totalframes % self.output_blocksize == 0:
            nb_frames -= 1  # Last frame must send eod=True

        for index in xrange(0,
                            nb_frames * self.output_blocksize,
                            self.output_blocksize):
            yield (self.samples[index:index + self.output_blocksize], False)

        yield (self.samples[nb_frames * self.output_blocksize:], True)