return statistical significance for props' columns.

        *pairwise_chisq* (ndarray) Matrix of chi-squared values (bases for Wishart CDF)
343798
def _pvals_from_chi_squared(self, pairwise_chisq):
        
        return self._intersperse_insertion_rows_and_columns(
            1.0 - WishartCDF(pairwise_chisq, self._n_min, self._n_max).values
        )
569526
def corrgroups60(display=False):
    

    # set a constant seed
    old_seed = np.random.seed()
    np.random.seed(0)

    # generate dataset with known correlation
    N = 1000
    M = 60

    # set one coefficent from each group of 3 to 1
    beta = np.zeros(M)
    beta[0:30:3] = 1

    # build a correlation matrix with groups of 3 tightly correlated features
    C = np.eye(M)
    for i in range(0,30,3):
        C[i,i+1] = C[i+1,i] = 0.99
        C[i,i+2] = C[i+2,i] = 0.99
        C[i+1,i+2] = C[i+2,i+1] = 0.99
    f = lambda X: np.matmul(X, beta)

    # Make sure the sample correlation is a perfect match
    X_start = np.random.randn(N, M)
    X_centered = X_start - X_start.mean(0)
    Sigma = np.matmul(X_centered.T, X_centered) / X_centered.shape[0]
    W = np.linalg.cholesky(np.linalg.inv(Sigma)).T
    X_white = np.matmul(X_centered, W.T)
    assert np.linalg.norm(np.corrcoef(np.matmul(X_centered, W.T).T) - np.eye(M)) < 1e-6 # ensure this decorrelates the data

    # create the final data
    X_final = np.matmul(X_white, np.linalg.cholesky(C).T)
    X = X_final
    y = f(X) + np.random.randn(N) * 1e-2

    # restore the previous numpy random seed
    np.random.seed(old_seed)

    return pd.DataFrame(X), y
1023255
def ts_stats_significance(ts, ts_stat_func, null_ts_func, B=1000, permute_fast=False):
    
    stats_ts = ts_stat_func(ts)
    if permute_fast:
        # Permute it in 1 shot
        null_ts = map(np.random.permutation, np.array([ts, ] * B))
    else:
        null_ts = np.vstack([null_ts_func(ts) for i in np.arange(0, B)])
    stats_null_ts = np.vstack([ts_stat_func(nts) for nts in null_ts])
    pvals = []
    nums = []
    for i in np.arange(0, len(stats_ts)):
        num_samples = np.sum((stats_null_ts[:, i] >= stats_ts[i]))
        nums.append(num_samples)
        pval = num_samples / float(B)
        pvals.append(pval)

    return stats_ts, pvals, nums
784740
def cdf(self, x):
        
        t = 0
        N = float(self.n)

        if len(self) == 1:  # only one centroid
            return int(x >= self.C.min_key())

        for i, key in enumerate(self.C.keys()):
            c_i = self.C[key]
            if i == len(self) - 1:
                delta = (c_i.mean - self.C.prev_item(key)[1].mean) / 2.
            else:
                delta = (self.C.succ_item(key)[1].mean - c_i.mean) / 2.
            z = max(-1, (x - c_i.mean) / delta)

            if z < 1:
                return t / N + c_i.count / N * (z + 1) / 2

            t += c_i.count
        return 1
611029
def decompose_covariance(c):
    

    # make it a kickass copy of the original
    c = _n.array(c)

    # first get the error vector
    e = []
    for n in range(0, len(c[0])): e.append(_n.sqrt(c[n][n]))

    # now cycle through the matrix, dividing by e[1]*e[2]
    for n in range(0, len(c[0])):
        for m in range(0, len(c[0])):
            c[n][m] = c[n][m] / (e[n]*e[m])

    return [_n.array(e), _n.array(c)]
846810
private double getScore(int[] contingencyTable, SignificanceTest test) {
        switch (test) {
        case PMI:
            return pmi(contingencyTable);
        case CHI_SQUARED:
            return chiSq(contingencyTable);
        case LOG_LIKELIHOOD:
            return logLikelihood(contingencyTable);
        default:
            throw new Error(test + " not implemented yet");
        }
    }
1263805
def estimate_row_scales(
            self,
            X_centered,
            column_scales):
        
        n_rows, n_cols = X_centered.shape
        column_scales = np.asarray(column_scales)
        if len(column_scales) != n_cols:
            raise ValueError("Expected length %d but got shape %s" % (
                n_cols, column_scales))
        row_variances = np.nanmean(
            X_centered ** 2 / (column_scales ** 2).reshape((1, n_cols)),
            axis=1)
        row_variances[row_variances == 0] = 1.0
        assert len(row_variances) == n_rows, "%d != %d" % (
            len(row_variances),
            n_rows)
        return np.sqrt(row_variances)
887927
def calculate_rsq(self):
        
        assert hasattr(self, 'betas'), 'no betas found, please run regression before rsq'

        explained_times = self.design_matrix.sum(axis = 0) != 0

        explained_signal = self.predict_from_design_matrix(self.design_matrix)
        self.rsq = 1.0 - np.sum((explained_signal[:,explained_times] - self.resampled_signal[:,explained_times])**2, axis = -1) / np.sum(self.resampled_signal[:,explained_times].squeeze()**2, axis = -1)
        self.ssr = np.sum((explained_signal[:,explained_times] - self.resampled_signal[:,explained_times])**2, axis = -1)
        return np.squeeze(self.rsq)
1161114
public Map<Integer, Double> chi_square(BaseFeatureData stats)
    {
        Map<Integer, Double> selectedFeatures = new HashMap<Integer, Double>();

        double N1dot, N0dot, N00, N01, N10, N11;
        double chisquareScore;
        Double previousScore;
        for (int feature = 0; feature < stats.featureCategoryJointCount.length; feature++)
        {
            int[] categoryList = stats.featureCategoryJointCount[feature];

            //计算 N1. (含有该特征的文档数量)
            N1dot = 0;
            for (int count : categoryList)
            {
                N1dot += count;
            }

            //还有 N0. (不含该特征的文档数量)
            N0dot = stats.n - N1dot;

            for (int category = 0; category < categoryList.length; category++)
            {

                N11 = categoryList[category]; //N11 是含有该特征并属于该类目的文档数量
                N01 = stats.categoryCounts[category] - N11; //N01 是不含该特征却属于该类目的文档数量

                N00 = N0dot - N01; //N00 是不含该特征也不属于该类目的文档数量
                N10 = N1dot - N11; //N10 是含有该特征却不属于该类目的文档数量

                //基于上述统计数据计算卡方分值
                chisquareScore = stats.n * Math.pow(N11 * N00 - N10 * N01, 2) / ((N11 + N01) * (N11 + N10) * (N10 + N00) * (N01 + N00));

                //如果分数大于临界值则加入特征列表
                if (chisquareScore >= chisquareCriticalValue)
                {
                    previousScore = selectedFeatures.get(feature);
                    if (previousScore == null || chisquareScore > previousScore)
                    {
                        selectedFeatures.put(feature, chisquareScore);
                    }
                }
            }
        }
        if (selectedFeatures.size() == 0) // 当特征全部无法通过卡方检测时，取全集作为特征
        {
            for (int feature = 0; feature < stats.featureCategoryJointCount.length; feature++)
            {
                selectedFeatures.put(feature, 0.);
            }
        }
        if (selectedFeatures.size() > maxSize)
        {
            MaxHeap<Map.Entry<Integer, Double>> maxHeap = new MaxHeap<Map.Entry<Integer, Double>>(maxSize, new Comparator<Map.Entry<Integer, Double>>()
            {
                @Override
                public int compare(Map.Entry<Integer, Double> o1, Map.Entry<Integer, Double> o2)
                {
                    return o1.getValue().compareTo(o2.getValue());
                }
            });
            for (Map.Entry<Integer, Double> entry : selectedFeatures.entrySet())
            {
                maxHeap.add(entry);
            }
            selectedFeatures.clear();
            for (Map.Entry<Integer, Double> entry : maxHeap)
            {
                selectedFeatures.put(entry.getKey(), entry.getValue());
            }
        }

        return selectedFeatures;
    }
1223325
def correlations(self):
        
        corr_df = self._sensors_num_df.corr()
        corr_names = []
        corrs = []

        for i in range(len(corr_df.index)):
            for j in range(len(corr_df.index)):
                c_name = corr_df.index[i]
                r_name = corr_df.columns[j]
                corr_names.append("%s-%s" % (c_name, r_name))
                corrs.append(corr_df.ix[i, j])

        corrs_all = pd.DataFrame(index=corr_names)
        corrs_all["value"] = corrs
        corrs_all = corrs_all.dropna().drop(
            corrs_all[(corrs_all["value"] == float(1))].index
        )
        corrs_all = corrs_all.drop(corrs_all[corrs_all["value"] == float(-1)].index)

        corrs_all = corrs_all.sort_values("value", ascending=False)
        corrs_all = corrs_all.drop_duplicates()
        return corrs_all
469883
def leastsq_generalized(x, y, x2, y2, res_std=None, res2_std=None, fit_offset=False, fit_offset2=False, perc=None):
    
    if perc is not None:
        if not fit_offset and isinstance(perc, (list, tuple)): perc = perc[1]
        weights = csr_matrix(get_weight(x, y, perc)).astype(bool)
        x, y = weights.multiply(x).tocsr(), weights.multiply(y).tocsr()

    n_obs, n_var = x.shape
    offset, offset_ss = np.zeros(n_var, dtype="float32"), np.zeros(n_var, dtype="float32")
    gamma = np.ones(n_var, dtype="float32")

    if (res_std is None) or (res2_std is None): res_std, res2_std = np.ones(n_var), np.ones(n_var)
    ones, zeros = np.ones(n_obs), np.zeros(n_obs)

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        x, y = np.vstack((make_dense(x)/res_std, x2/res2_std)), np.vstack((make_dense(y)/res_std, y2/res2_std))

    if fit_offset and fit_offset2:
        for i in range(n_var):
            A = np.c_[np.vstack((np.c_[ones/res_std[i], zeros], np.c_[zeros, ones/res2_std[i]])), x[:, i]]
            offset[i], offset_ss[i], gamma[i] = np.linalg.pinv(A.T.dot(A)).dot(A.T.dot(y[:, i]))
    elif fit_offset:
        for i in range(n_var):
            A = np.c_[np.hstack((ones/res_std[i], zeros)), x[:, i]]
            offset[i], gamma[i] = np.linalg.pinv(A.T.dot(A)).dot(A.T.dot(y[:, i]))
    elif fit_offset2:
        for i in range(n_var):
            A = np.c_[np.hstack((zeros, ones/res2_std[i])), x[:, i]]
            offset_ss[i], gamma[i] = np.linalg.pinv(A.T.dot(A)).dot(A.T.dot(y[:, i]))
    else:
        for i in range(n_var):
            A = np.c_[x[:, i]]
            gamma[i] = np.linalg.pinv(A.T.dot(A)).dot(A.T.dot(y[:, i]))

    offset[np.isnan(offset)] = 0
    offset_ss[np.isnan(offset_ss)] = 0
    gamma[np.isnan(gamma)] = 0

    return offset, offset_ss, gamma
548666
def variance_inflation_factors(df):
    '''
    
    '''
    corr = np.corrcoef(df, rowvar=0)
    corr_inv = np.linalg.inv(corr)
    vifs = np.diagonal(corr_inv)
    return pd.Series(vifs, df.columns, name='VIF')
1012851
def chi_squared(self, p=None):
        
        chi2s = self.chi_squareds(p)
        if chi2s == None: return None
        
        return sum(self.chi_squareds(p))
1199874
def spatial_correlation_array(dmatrix, imts, correl='yes',
                              vs30clustered=True):
    
    assert correl in 'yes no full', correl
    n = len(dmatrix)
    corr = numpy.zeros((len(imts), n, n))
    for imti, im in enumerate(imts):
        if correl == 'no':
            corr[imti] = numpy.eye(n)
        if correl == 'full':
            corr[imti] = numpy.ones((n, n))
        elif correl == 'yes':
            corr[imti] = correlation.jbcorrelation(dmatrix, im, vs30clustered)
    return corr
79668
def column_correlations(self, X):
        """"""
        utils.validation.check_is_fitted(self, 's_')

        X_global = self._build_X_global(X)
        row_pc = self._row_coordinates_from_global(X_global)

        return pd.DataFrame({
            component: {
                feature: row_pc[component].corr(X_global[feature].to_dense())
                for feature in X_global.columns
            }
            for component in row_pc.columns
        })
538517
def rwishart_cov(n, C):
    
    # return rwishart(n, np.linalg.inv(C))

    p = np.shape(C)[0]
    # Need cholesky decomposition of precision matrix C^-1?
    sig = np.linalg.cholesky(C)

    if n <= (p-1):
        raise ValueError('Wishart parameter n must be greater '
                         'than size of matrix.')

    norms = np.random.normal(size=(p * (p - 1)) // 2)
    chi_sqs = np.sqrt(np.random.chisquare(df=np.arange(n, n - p, -1)))
    A = flib.expand_triangular(chi_sqs, norms)

    flib.dtrmm_wrap(sig, A, side='L', uplo='L', transa='N', alpha=1.)
    w = np.asmatrix(np.dot(A, A.T))
    flib.symmetrize(w)
    return w
816524
function calculateChiSquareProbability(x, df) {
    if (x <= 0.0 || df < 1)
        return 1.0;

    var a = 0.5 * x;

    if (df > 1)
        var y = calculateExponent(-a);

    var s = 2.0 * calculateNormalZProbability(-Math.sqrt(x));

    if (df > 2) {
        x = 0.5 * (df - 1.0);
        var z = 0.5;

        if (a > BIG_X) {
            var e = LOG_SQRT_PI;
            var c = Math.log(a);

            while (z <= x) {
                e = Math.log(z) + e;
                s += calculateExponent(c * z - a - e);
                z += 1.0;
            }

            return (s);
        }
        else {
            e = I_SQRT_PI / Math.sqrt(a);
            c = 0.0;

            while (z <= x) {
                e = e * (a / z);
                c = c + e;
                z += 1.0;
            }

            return (c * y + s);
        }
    }

    return s;
}