Shuffles a sparse matrix and set of labels together.
  Resorts to densifying and then re-sparsifying the matrix, for
  convenience.  Still very fast.
503126
def shuffle_sparse_matrix_and_labels(matrix, labels):
  
  print "Shuffling data"
  new_matrix = matrix.toDense()
  rng_state = numpy.random.get_state()
  numpy.random.shuffle(new_matrix)
  numpy.random.set_state(rng_state)
  numpy.random.shuffle(labels)

  print "Data shuffled"
  return SM32(new_matrix), numpy.asarray(labels)
345235
public Collection<String> nearestLabels(INDArray labelVector, int topN) {
        if (labelsMatrix == null || labelsList == null || labelsList.isEmpty())
            extractLabels();

        List<BasicModelUtils.WordSimilarity> result = new ArrayList<>();

        // if list still empty - return empty collection
        if (labelsMatrix == null || labelsList == null || labelsList.isEmpty()) {
            log.warn("Labels list is empty!");
            return new ArrayList<>();
        }

        if (!normalizedLabels) {
            synchronized (this) {
                if (!normalizedLabels) {
                    labelsMatrix.diviColumnVector(labelsMatrix.norm1(1));
                    normalizedLabels = true;
                }
            }
        }

        INDArray similarity = Transforms.unitVec(labelVector).mmul(labelsMatrix.transpose());
        List<Double> highToLowSimList = getTopN(similarity, topN + 20);

        for (int i = 0; i < highToLowSimList.size(); i++) {
            String word = labelsList.get(highToLowSimList.get(i).intValue()).getLabel();
            if (word != null && !word.equals("UNK") && !word.equals("STOP")) {
                INDArray otherVec = lookupTable.vector(word);
                double sim = Transforms.cosineSim(labelVector, otherVec);

                result.add(new BasicModelUtils.WordSimilarity(word, sim));
            }
        }

        Collections.sort(result, new BasicModelUtils.SimilarityComparator());

        return BasicModelUtils.getLabels(result, topN);
    }
999879
private ValFrame slow_table(Vec v1, Vec v2, String[] colnames, boolean dense) {


    // For simplicity, repeat v1 if v2 is missing; this will end up filling in
    // only the diagonal of a 2-D array (in what is otherwise a 1-D array).
    // This should be nearly the same cost as a 1-D array, since everything is
    // sparsely filled in.

    // If this is the 1-column case (all counts on the diagonals), just build a
    // 1-d result.
    if (v2 == null) {


      // Slow-pass group counting, very sparse hashtables.  Note that Vec v2 is
      // used as the left-most arg, or OUTER dimension - which will be columns in
      // the final result.
      AstTable.SlowCnt sc = new AstTable.SlowCnt().doAll(v1, v1);

      // Get the column headers as sorted doubles
      double dcols[] = collectDomain(sc._col0s);

      Frame res = new Frame();
      Vec rowlabel = Vec.makeVec(dcols, Vec.VectorGroup.VG_LEN1.addVec());
      rowlabel.setDomain(v1.domain());
      res.add(colnames[0], rowlabel);
      long cnts[] = new long[dcols.length];
      for (int col = 0; col < dcols.length; col++) {
        long lkey = Double.doubleToRawLongBits(dcols[col]);
        NonBlockingHashMapLong<AtomicLong> colx = sc._col0s.get(lkey);
        AtomicLong al = colx.get(lkey);
        cnts[col] = al.get();
      }
      Vec vec = Vec.makeVec(cnts, null, Vec.VectorGroup.VG_LEN1.addVec());
      res.add("Counts", vec);
      return new ValFrame(res);
    }

    // 2-d table result.
    Frame res = new Frame();
    if (!dense) {

      // Slow-pass group counting, very sparse hashtables.  Note that Vec v2 is
      // used as the left-most arg, or OUTER dimension - which will be columns in
      // the final result.
      AstTable.SlowCnt sc = new AstTable.SlowCnt().doAll(v2, v1);

      // Get the column headers as sorted doubles
      double dcols[] = collectDomain(sc._col0s);

      // Need the row headers as sorted doubles also, but these are scattered
      // throughout the nested tables.  Fold 'em into 1 table.
      NonBlockingHashMapLong<AtomicLong> rows = new NonBlockingHashMapLong<>();
      for (NonBlockingHashMapLong.IteratorLong i = iter(sc._col0s); i.hasNext(); )
        rows.putAll(sc._col0s.get(i.nextLong()));
      double drows[] = collectDomain(rows);

      // Now walk the columns one by one, building a Vec per column, building a
      // Frame result.  Rowlabel for first column.

      Vec rowlabel = Vec.makeVec(drows, Vec.VectorGroup.VG_LEN1.addVec());
      rowlabel.setDomain(v1.domain());
      res.add(colnames[0], rowlabel);
      long cnts[] = new long[drows.length];
      for (int col = 0; col < dcols.length; col++) {
        NonBlockingHashMapLong<AtomicLong> colx = sc._col0s.get(Double.doubleToRawLongBits(dcols[col]));
        for (int row = 0; row < drows.length; row++) {
          AtomicLong al = colx.get(Double.doubleToRawLongBits(drows[row]));
          cnts[row] = al == null ? 0 : al.get();
        }
        Vec vec = Vec.makeVec(cnts, null, Vec.VectorGroup.VG_LEN1.addVec());
        res.add(v2.isCategorical() ? v2.domain()[col] : Double.toString(dcols[col]), vec);
      }
    } else {

      AstTable.SlowCnt sc = new AstTable.SlowCnt().doAll(v1, v2);

      double dcols[] = collectDomain(sc._col0s);

      NonBlockingHashMapLong<AtomicLong> rows = new NonBlockingHashMapLong<>();
      for (NonBlockingHashMapLong.IteratorLong i = iter(sc._col0s); i.hasNext(); )
        rows.putAll(sc._col0s.get(i.nextLong()));
      double drows[] = collectDomain(rows);

      int x = 0;
      int sz = 0;
      for (NonBlockingHashMapLong.IteratorLong i = iter(sc._col0s); i.hasNext(); ) {
        sz += sc._col0s.get(i.nextLong()).size();
      }
      long cnts[] = new long[sz];
      double[] left_categ = new double[sz];
      double[] right_categ = new double[sz];

      for (double dcol : dcols) {
        NonBlockingHashMapLong<AtomicLong> colx = sc._col0s.get(Double.doubleToRawLongBits(dcol));
        for (double drow : drows) {
          AtomicLong al = colx.get(Double.doubleToRawLongBits(drow));
          if (al != null) {
            left_categ[x] = dcol;
            right_categ[x] = drow;
            cnts[x] = al.get();
            x++;
          }
        }
      }

      Vec vec = Vec.makeVec(left_categ, Vec.VectorGroup.VG_LEN1.addVec());
      if (v1.isCategorical()) vec.setDomain(v1.domain());
      res.add(colnames[0], vec);
      vec = Vec.makeVec(right_categ, Vec.VectorGroup.VG_LEN1.addVec());
      if (v2.isCategorical()) vec.setDomain(v2.domain());
      res.add(colnames[1], vec);
      vec = Vec.makeVec(cnts, null, Vec.VectorGroup.VG_LEN1.addVec());
      res.add("Counts", vec);
    }
    return new ValFrame(res);
  }
211318
def fastaAlignmentRead(fasta, mapFn=(lambda x : x), l=None):
    
    if l is None:
        l = _getMultiFastaOffsets(fasta)
    else:
        l = l[:]
    seqNo = len(l)
    for i in xrange(0, seqNo):
        j = open(fasta, 'r')
        j.seek(l[i])
        l[i] = j
    column = [sys.maxint]*seqNo
    if seqNo != 0:
        while True:
            for j in xrange(0, seqNo):
                i = l[j].read(1)
                while i == '\n':
                    i = l[j].read(1)
                column[j] = i
            if column[0] == '>' or column[0] == '':
                for j in xrange(1, seqNo):
                    assert column[j] == '>' or column[j] == ''
                break
            for j in xrange(1, seqNo):
                 assert column[j] != '>' and column[j] != ''
                 column[j] = mapFn(column[j])
            yield column[:]
    for i in l:
        i.close()
1352783
def _zero_many(self, i, j):
    
    i, j, M, N = self._prepare_indices(i, j)

    n_samples = len(i)
    offsets = np.empty(n_samples, dtype=self.indices.dtype)
    ret = _sparsetools.csr_sample_offsets(M, N, self.indptr, self.indices,
                                          n_samples, i, j, offsets)
    if ret == 1:
        # rinse and repeat
        self.sum_duplicates()
        _sparsetools.csr_sample_offsets(M, N, self.indptr,
                                        self.indices, n_samples, i, j,
                                        offsets)

    # only assign zeros to the existing sparsity structure
    self.data[list(offsets[offsets > -1])] = 0
653720
private void append2slow( ) {
// PUBDEV-2639 - don't die for many rows, few columns -> can be long chunks
//    if( _sparseLen > FileVec.DFLT_CHUNK_SIZE )
//      throw new ArrayIndexOutOfBoundsException(_sparseLen);
    assert _ds==null;
    if(_ms != null && _sparseLen > 0){
      if(_id == null) { // check for sparseness
        int nzs = _ms._nzs + (_missing != null?_missing.cardinality():0);
        int nonnas = _sparseLen - ((_missing != null)?_missing.cardinality():0);
        if((nonnas+1)*_sparseRatio < _len) {
          set_sparse(nonnas,Compress.NA);
          assert _id.length == _ms.len():"id.len = " + _id.length + ", ms.len = " + _ms.len();
          assert _sparseLen <= _len;
          return;
        } else if((nzs+1)*_sparseRatio < _len) { // note order important here
          set_sparse(nzs,Compress.ZERO);
          assert _sparseLen <= _len;
          assert _sparseLen == nzs;
          return;
        }
      } else {
        // verify we're still sufficiently sparse
        if(2*_sparseLen > _len)  cancel_sparse();
        else _id = MemoryManager.arrayCopyOf(_id, _id.length*2);
      }
      _ms.resize(_sparseLen*2);
      _xs.resize(_sparseLen*2);
    } else {
      _ms = new Mantissas(16);
      _xs = new Exponents(16);
      if (_id != null) _id = new int[16];
    }
    assert _sparseLen <= _len;
  }
139353
def _sparse_blockify(tuples, dtype=None):
    

    new_blocks = []
    for i, names, array in tuples:
        array = _maybe_to_sparse(array)
        block = make_block(array, placement=[i])
        new_blocks.append(block)

    return new_blocks
828422
public static DenseMatrix random(int rows, int columns, Random random) {
        return Basic2DMatrix.random(rows, columns, random);
    }
368827
private Matrix loadSparseBinary(InputStream fileStream) throws IOException {
        DataInputStream dis = new DataInputStream(fileStream);
        int rows = dis.readInt();
        int cols = dis.readInt();
        // Create the sparse matrix as individual rows since we can fully
        // allocate the indices values at once, rather than pay the log(n)
        // overhead of sorting them
        CompactSparseVector[] rowVectors = new CompactSparseVector[rows];

        for (int row = 0; row < rows; ++row) {
            String word = dis.readUTF();
            termToIndex.put(word, row);
            
            int nonZero = dis.readInt();
            int[] indices = new int[nonZero];
            double[] values = new double[nonZero];
            for (int i = 0; i < nonZero; ++i) {
                int nz = dis.readInt();
                double val = dis.readDouble();
                indices[i] = nz;
                values[i] = val;
            }
            rowVectors[row] = new CompactSparseVector(indices, values, cols);
        }
        return Matrices.asSparseMatrix(Arrays.asList(rowVectors));
    }
289783
def write_spmatrix_to_sparse_tensor(file, array, labels=None):
    """"""

    if not issparse(array):
        raise TypeError("Array must be sparse")

    # Validate shape of array and labels, resolve array and label types
    if not len(array.shape) == 2:
        raise ValueError("Array must be a Matrix")
    if labels is not None:
        if not len(labels.shape) == 1:
            raise ValueError("Labels must be a Vector")
        if labels.shape[0] not in array.shape:
            raise ValueError("Label shape {} not compatible with array shape {}".format(
                             labels.shape, array.shape))
        resolved_label_type = _resolve_type(labels.dtype)
    resolved_type = _resolve_type(array.dtype)

    csr_array = array.tocsr()
    n_rows, n_cols = csr_array.shape

    record = Record()
    for row_idx in range(n_rows):
        record.Clear()
        row = csr_array.getrow(row_idx)
        # Write values
        _write_feature_tensor(resolved_type, record, row.data)
        # Write keys
        _write_keys_tensor(resolved_type, record, row.indices.astype(np.uint64))

        # Write labels
        if labels is not None:
            _write_label_tensor(resolved_label_type, record, labels[row_idx])

        # Write shape
        _write_shape(resolved_type, record, n_cols)

        _write_recordio(file, record.SerializeToString())
68520
private Matrix calculateEdgeSimMatrix(
            final List<Edge> edgeList, final SparseMatrix sm) {

        final int numEdges = edgeList.size();
        final Matrix edgeSimMatrix = 
            new SparseSymmetricMatrix(
                new SparseHashMatrix(numEdges, numEdges));

        Object key = workQueue.registerTaskGroup(numEdges);
        for (int i = 0; i < numEdges; ++i) {
            final int row = i;
            workQueue.add(key, new Runnable() {
                    public void run() {
                        for (int j = row; j < numEdges; ++j) {
                            Edge e1 = edgeList.get(row);
                            Edge e2 = edgeList.get(j);
                            
                            double sim = getEdgeSimilarity(sm, e1, e2);
                
                            if (sim > 0) {
                                // The symmetric matrix handles the (j,i) case
                                edgeSimMatrix.set(row, j, sim);
                            }
                        }
                    }
                });            
        }
        workQueue.await(key);
        return edgeSimMatrix;
    }
301848
def to_simple_rdd(sc, features, labels):
    
    pairs = [(x, y) for x, y in zip(features, labels)]
    return sc.parallelize(pairs)
914195
private static void vectorSwap(PagesIndex pagesIndex, int from, int l, int s)
    {
        for (int i = 0; i < s; i++, from++, l++) {
            pagesIndex.swap(from, l);
        }
    }
556752
def skipgram_batch(centers, contexts, num_tokens, dtype, index_dtype):
    """"""
    contexts = mx.nd.array(contexts[2], dtype=index_dtype)
    indptr = mx.nd.arange(len(centers) + 1)
    centers = mx.nd.array(centers, dtype=index_dtype)
    centers_csr = mx.nd.sparse.csr_matrix(
        (mx.nd.ones(centers.shape), centers, indptr), dtype=dtype,
        shape=(len(centers), num_tokens))
    return centers_csr, contexts, centers
285152
def _maybe_to_sparse(array):
    
    if isinstance(array, ABCSparseSeries):
        array = array.values.copy()
    return array
1238389
def skeletonize_labels(labels):
    ''''''
    #
    # The trick here is to separate touching labels by coloring the
    # labels matrix and then processing each color separately
    #
    colors = color_labels(labels)
    max_color = np.max(colors)
    if max_color == 0:
        return labels
    result = np.zeros(labels.shape, labels.dtype)
    for i in range(1,max_color+1):
        mask = skeletonize(colors==i)
        result[mask] = labels[mask]
    return result
466896
private void fastAddAll(ArrayBackedSortedColumns other)
    {
        if (other.isInsertReversed() == isInsertReversed())
        {
            cells = Arrays.copyOf(other.cells, other.cells.length);
            size = other.size;
            sortedSize = other.sortedSize;
            isSorted = other.isSorted;
        }
        else
        {
            if (cells.length < other.getColumnCount())
                cells = new Cell[Math.max(MINIMAL_CAPACITY, other.getColumnCount())];
            Iterator<Cell> iterator = reversed ? other.reverseIterator() : other.iterator();
            while (iterator.hasNext())
                cells[size++] = iterator.next();
            sortedSize = size;
            isSorted = true;
        }
    }