make_data: prepare matrix of m times n random processing times
352798
def make_data(n,m):
    """"""
    p = {}
    for i in range(1,m+1):
        for j in range(1,n+1):
            p[i,j] = random.randint(1,10)
    return p
128118
def make_data(n):
    """"""
    V = range(1,n+1)
    x = dict([(i, random.random()) for i in V])
    y = dict([(i, random.random()) for i in V])
    c, q = {}, {}
    Q = 100
    for i in V:
        q[i] = random.randint(10, 20)
        for j in V:
            if j > i:
                c[i, j] = distance(x[i], y[i], x[j], y[j])
    return V, c, q, Q
183600
def preprocess_with_pca(adata, n_pcs=None, random_state=0):
    
    if n_pcs == 0:
        logg.info('    using data matrix X directly (no PCA)')
        return adata.X
    elif n_pcs is None and 'X_pca' in adata.obsm_keys():
        logg.info('    using \'X_pca\' with n_pcs = {}'
                  .format(adata.obsm['X_pca'].shape[1]))
        return adata.obsm['X_pca']
    elif ('X_pca' in adata.obsm_keys()
          and adata.obsm['X_pca'].shape[1] >= n_pcs):
        logg.info('    using \'X_pca\' with n_pcs = {}'
                  .format(n_pcs))
        return adata.obsm['X_pca'][:, :n_pcs]
    else:
        n_pcs = N_PCS if n_pcs is None else n_pcs
        if adata.X.shape[1] > n_pcs:
            logg.info('    computing \'X_pca\' with n_pcs = {}'.format(n_pcs))
            logg.hint('avoid this by setting n_pcs = 0')
            X = pca(adata.X, n_comps=n_pcs, random_state=random_state)
            adata.obsm['X_pca'] = X
            return X
        else:
            logg.info('    using data matrix X directly (no PCA)')
            return adata.X
354354
private double[][] createMatrix() {
        double time = 0;
        double tmin = 0;
        tmin = rainData[0][0];
        time = tmin;
        double[][] matrix = new double[nTime][networkPipes.length + 1];

        for( int i = 0; i < nTime; ++i ) {
            matrix[i][0] = time;
            time += dt;
        }
        return matrix;
    }
1373770
def gen_batches(data, batch_size):
    
    data = np.array(data)

    for i in range(0, data.shape[0], batch_size):
        yield data[i:i + batch_size]
294392
def preprocess(options):

    assert options.bfile!=None, 'Please specify a bfile.'

    
    if options.compute_cov:
       assert options.bfile!=None, 'Please specify a bfile.'
       assert options.cfile is not None, 'Specify covariance matrix basename'
       print('Computing covariance matrix')
       t0 = time.time()
       computeCovarianceMatrix(options.plink_path,options.bfile,options.cfile,options.sim_type)
       t1 = time.time()
       print(('... finished in %s seconds'%(t1-t0)))
       print('Computing eigenvalue decomposition')
       t0 = time.time()
       eighCovarianceMatrix(options.cfile)
       t1 = time.time()
       print(('... finished in %s seconds'%(t1-t0)))

    """ computing principal components """
    if options.compute_PCs>0:
       assert options.ffile is not None, 'Specify fix effects basename for saving PCs'
       t0 = time.time()
       computePCs(options.plink_path,options.compute_PCs,options.bfile,options.ffile)
       t1 = time.time()
       print(('... finished in %s seconds'%(t1-t0)))

    """ fitting the null model """
    if options.fit_null:
        if options.nfile is None:
            options.nfile = os.path.split(options.bfile)[-1]
            warnings.warn('nfile not specifed, set to %s'%options.nfile)
        print('Fitting null model')
        assert options.pfile is not None, 'phenotype file needs to be specified'
        # read pheno
        Y = readPhenoFile(options.pfile,idx=options.trait_idx)
        # read covariance
        if options.cfile is None:
            cov = {'eval':None,'evec':None}
            warnings.warn('cfile not specifed, a one variance compoenent model will be considered')
        else:
            cov = readCovarianceMatrixFile(options.cfile,readCov=False)
            assert Y.shape[0]==cov['eval'].shape[0],  'dimension mismatch'
        # read covariates
        F = None
        if options.ffile is not None:
            F = readCovariatesFile(options.ffile)
            assert Y.shape[0]==F.shape[0], 'dimensions mismatch'
        t0 = time.time()
        fit_null(Y,cov['eval'],cov['evec'],options.nfile, F)
        t1 = time.time()
        print(('.. finished in %s seconds'%(t1-t0)))

    """ precomputing the windows """
    if options.precompute_windows:
        if options.wfile==None:
            options.wfile = os.path.split(options.bfile)[-1] + '.%d'%options.window_size
            warnings.warn('wfile not specifed, set to %s'%options.wfile)
        print('Precomputing windows')
        t0 = time.time()
        pos = readBimFile(options.bfile)
        nWnds,nSnps=splitGeno(pos,size=options.window_size,out_file=options.wfile+'.wnd')
        print(('Number of variants:',pos.shape[0]))
        print(('Number of windows:',nWnds))
        print(('Minimum number of snps:',nSnps.min()))
        print(('Maximum number of snps:',nSnps.max()))
        t1 = time.time()
        print(('.. finished in %s seconds'%(t1-t0)))

    # plot distribution of nSnps
    if options.plot_windows:
        print('Plotting ditribution of number of SNPs')
        plot_file = options.wfile+'.wnd.pdf'
        plt = pl.subplot(1,1,1)
        pl.hist(nSnps,30)
        pl.xlabel('Number of SNPs')
        pl.ylabel('Number of windows')
        pl.savefig(plot_file)
385308
def preprocess_data(Xs_raw):
    ''''''
    n = len(Xs_raw)
    Xs_raw_mean = sum(X for X in Xs_raw) / n

    return [X - Xs_raw_mean for X in Xs_raw], Xs_raw_mean
1374912
def pool(data, batch_size, key, batch_size_fn=lambda new, count, sofar: count,
         random_shuffler=None, shuffle=False, sort_within_batch=False):
    
    if random_shuffler is None:
        random_shuffler = random.shuffle
    for p in batch(data, batch_size * 100, batch_size_fn):
        p_batch = batch(sorted(p, key=key), batch_size, batch_size_fn) \
            if sort_within_batch \
            else batch(p, batch_size, batch_size_fn)
        if shuffle:
            for b in random_shuffler(list(p_batch)):
                yield b
        else:
            for b in list(p_batch):
                yield b
600763
def randomprune(self,n):
        """"""
        self.data = random.sample(self.data, n)
382384
def process(self, process_data):
        """"""
        if self.parallel and not process_data.options.dry_run:
            return self.run_matrix_in_parallel(process_data)
        return self.run_matrix_ordered(process_data)
884366
def ndim(n, *args, **kwargs):
    
    thunk = kwargs.get("thunk", lambda: random.random())
    if not args: 
        return [thunk() for i in range(n)]
    A = [] 
    for i in range(n):
        A.append( ndim(*args, thunk=thunk) ) 
    return A
951775
def __init_from_list_np2d(self, mats, params_str, ref_dataset):
        """"""
        ncol = mats[0].shape[1]
        nrow = np.zeros((len(mats),), np.int32)
        if mats[0].dtype == np.float64:
            ptr_data = (ctypes.POINTER(ctypes.c_double) * len(mats))()
        else:
            ptr_data = (ctypes.POINTER(ctypes.c_float) * len(mats))()

        holders = []
        type_ptr_data = None

        for i, mat in enumerate(mats):
            if len(mat.shape) != 2:
                raise ValueError('Input numpy.ndarray must be 2 dimensional')

            if mat.shape[1] != ncol:
                raise ValueError('Input arrays must have same number of columns')

            nrow[i] = mat.shape[0]

            if mat.dtype == np.float32 or mat.dtype == np.float64:
                mats[i] = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
            else:
                # change non-float data to float data, need to copy
                mats[i] = np.array(mat.reshape(mat.size), dtype=np.float32)

            chunk_ptr_data, chunk_type_ptr_data, holder = c_float_array(mats[i])
            if type_ptr_data is not None and chunk_type_ptr_data != type_ptr_data:
                raise ValueError('Input chunks must have same type')
            ptr_data[i] = chunk_ptr_data
            type_ptr_data = chunk_type_ptr_data
            holders.append(holder)

        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateFromMats(
            ctypes.c_int(len(mats)),
            ctypes.cast(ptr_data, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.c_int(type_ptr_data),
            nrow.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ctypes.c_int(ncol),
            ctypes.c_int(C_API_IS_ROW_MAJOR),
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
        return self
320708
def gen_stochastic_matrix(size, random_state=None):
    
    if not type(size) is tuple:
        size = (1, size)

    assert len(size) == 2

    n = random_state.uniform(size=(size[0], size[1] - 1))
    n = np.concatenate([np.zeros((size[0], 1)), n, np.ones((size[0], 1))], axis=1)

    A = np.diff(np.sort(n))
    return A.squeeze()
306395
def matrix_to_blockmatrix(self, blocksize):
        
        if self.get_width() % blocksize or self.get_height() % blocksize:
            raise ValueError("Number of rows and columns have to be evenly dividable by blocksize")
        selfBlocks = []
        for columnIndex in range(0, self.get_width() - 1, blocksize):
            for rowIndex in range(0, self.get_height() - 1, blocksize):
                currentBlock = []
                for blockRows in self.get_array(False)[columnIndex:columnIndex + blocksize]:
                    currentBlock += blockRows[rowIndex:rowIndex + blocksize]
                selfBlocks.append(Matrix(blocksize, blocksize, currentBlock, rowBased=False))
        return Matrix(self.get_width() / blocksize, self.get_height() / blocksize, selfBlocks, rowBased=False)
415112
def preprocess(df):
    """"""

    def process_row(idx_row):
        idx, row = idx_row

        # press-press latency
        tau = 1000 * row[4::3].astype(float).values
        tau = np.r_[np.median(tau), tau]

        # duration
        duration = 1000 * row[3::3].astype(float).values

        keyname = list('.tie5Roanl') + ['enter']

        return pd.DataFrame.from_items([
            ('user', [row['subject']] * 11),
            ('session', [row['sessionIndex'] * 100 + row['rep']] * 11),
            ('tau', tau),
            ('duration', duration),
            ('event', keyname)
        ])

    df = pd.concat(map(process_row, df.iterrows())).set_index(['user', 'session'])
    return df
199934
def _process_batch(self, param, dataframe):
        
        now = time.time()
        if param.eval_metric is not None:
            metrics = dict(param.eval_metric.get_name_value())
            param.eval_metric.reset()
        else:
            metrics = {}
        # #11504
        try:
            speed = self.frequent / (now - self.last_time)
        except ZeroDivisionError:
            speed = float('inf')
        metrics['batches_per_sec'] = speed * self.batch_size
        metrics['records_per_sec'] = speed
        metrics['elapsed'] = self.elapsed()
        metrics['minibatch_count'] = param.nbatch
        metrics['epoch'] = param.epoch
        self.append_metrics(metrics, dataframe)
        self.last_time = now
89020
def sp_rand(m,n,a):
    
    if m == 0 or n == 0: return spmatrix([], [], [], (m,n))
    nnz = min(max(0, int(round(a*m*n))), m*n)
    nz = matrix(random.sample(range(m*n), nnz), tc='i')
    return spmatrix(normal(nnz,1), nz%m, matrix([int(ii) for ii in nz/m]), (m,n))