Lasso Regression
353721
def cric__lasso():
    
    model = sklearn.linear_model.LogisticRegression(penalty="l1", C=0.002)

    # we want to explain the raw probability outputs of the trees
    model.predict = lambda X: model.predict_proba(X)[:,1]
    
    return model
745665
def _penalized_acquisition(self, x,  model, X_batch, r_x0, s_x0):
        '''
        
        '''
        fval = -self.acq.acquisition_function(x)[:,0]

        if self.transform=='softplus':
            fval_org = fval.copy()
            fval[fval_org>=40.] = np.log(fval_org[fval_org>=40.])
            fval[fval_org<40.] = np.log(np.log1p(np.exp(fval_org[fval_org<40.])))
        elif self.transform=='none':
            fval = np.log(fval+1e-50)

        fval = -fval
        if X_batch is not None:
            h_vals = self._hammer_function(x, X_batch, r_x0, s_x0)
            fval += -h_vals.sum(axis=-1)
        return fval
261706
def fit(self, X, y):
        

        # note that GLM takes endog (y) and then exog (X):
        # this is the reverse of sklearn's methods
        self.glm_model = self.glm_partial(y, X)
        self.glm_results = self.glm_model.fit()
        return self
314955
def LOO(self, kern, X, Y, likelihood, posterior, Y_metadata=None, K=None, f_hat=None, W=None, Ki_W_i=None):
        
        Ki_f_init = np.zeros_like(Y)

        if K is None:
            K = kern.K(X)

        if f_hat is None:
            f_hat, _ = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata)

        if W is None:
            W = -likelihood.d2logpdf_df2(f_hat, Y, Y_metadata=Y_metadata)

        if Ki_W_i is None:
            _, _, _, Ki_W_i = self._compute_B_statistics(K, W, likelihood.log_concave)

        logpdf_dfhat = likelihood.dlogpdf_df(f_hat, Y, Y_metadata=Y_metadata)

        if W.shape[1] == 1:
            W = np.diagflat(W)

        #Eq 14, and 16
        var_site = 1./np.diag(W)[:, None]
        mu_site = f_hat + var_site*logpdf_dfhat
        prec_site = 1./var_site
        #Eq 19
        marginal_cov = Ki_W_i
        marginal_mu = marginal_cov.dot(np.diagflat(prec_site)).dot(mu_site)
        marginal_var = np.diag(marginal_cov)[:, None]
        #Eq 30 with using site parameters instead of Gaussian site parameters
        #(var_site instead of sigma^{2} )
        posterior_cav_var = 1./(1./marginal_var - 1./var_site)
        posterior_cav_mean = posterior_cav_var*((1./marginal_var)*marginal_mu - (1./var_site)*Y)

        flat_y = Y.flatten()
        flat_mu = posterior_cav_mean.flatten()
        flat_var = posterior_cav_var.flatten()

        if Y_metadata is not None:
            #Need to zip individual elements of Y_metadata aswell
            Y_metadata_flat = {}
            if Y_metadata is not None:
                for key, val in Y_metadata.items():
                    Y_metadata_flat[key] = np.atleast_1d(val).reshape(-1, 1)

            zipped_values = []

            for i in range(Y.shape[0]):
                y_m = {}
                for key, val in Y_metadata_flat.items():
                    if np.isscalar(val) or val.shape[0] == 1:
                        y_m[key] = val
                    else:
                        #Won't broadcast yet
                        y_m[key] = val[i]
                zipped_values.append((flat_y[i], flat_mu[i], flat_var[i], y_m))
        else:
            #Otherwise just pass along None's
            zipped_values = zip(flat_y, flat_mu, flat_var, [None]*Y.shape[0])

        def integral_generator(yi, mi, vi, yi_m):
            def f(fi_star):
                #More stable in the log space
                p_fi = np.exp(likelihood.logpdf(fi_star, yi, yi_m)
                              - 0.5*np.log(2*np.pi*vi)
                              - 0.5*np.square(mi-fi_star)/vi)
                return p_fi
            return f

        #Eq 30
        p_ystar, _ = zip(*[quad(integral_generator(y, m, v, yi_m), -np.inf, np.inf)
                           for y, m, v, yi_m in zipped_values])
        p_ystar = np.array(p_ystar).reshape(-1, 1)
        return np.log(p_ystar)
509778
def lasso_leftdown(self, event=None):
        """"""
        try:
            self.report_leftdown(event=event)
        except:
            return

        if event.inaxes:
            # set lasso color
            color='goldenrod'
            cmap = getattr(self.conf, 'cmap', None)
            if isinstance(cmap, dict):
                cmap = cmap['int']
            try:
                if cmap is not None:
                    rgb = (int(i*255)^255 for i in cmap._lut[0][:3])
                    color = '#%02x%02x%02x' % tuple(rgb)
            except:
                pass
            self.lasso = Lasso(event.inaxes, (event.xdata, event.ydata),
                               self.lassoHandler)
            self.lasso.line.set_color(color)
481524
def quic_graph_lasso_ebic_manual(X, gamma=0):
    
    print("QuicGraphicalLasso (manual EBIC) with:")
    print("   mode: path")
    print("   gamma: {}".format(gamma))
    model = QuicGraphicalLasso(
        lam=1.0,
        mode="path",
        init_method="cov",
        path=np.logspace(np.log10(0.01), np.log10(1.0), num=100, endpoint=True),
    )
    model.fit(X)
    ebic_index = model.ebic_select(gamma=gamma)
    covariance_ = model.covariance_[ebic_index]
    precision_ = model.precision_[ebic_index]
    lam_ = model.lam_at_index(ebic_index)
    print("   len(path lams): {}".format(len(model.path_)))
    print("   lam_scale_: {}".format(model.lam_scale_))
    print("   lam_: {}".format(lam_))
    print("   ebic_index: {}".format(ebic_index))
    return covariance_, precision_, lam_
912796
def _construct_X_M(self, omega, **kwargs):
        """"""
        X = self._construct_X(omega, weighted=True, **kwargs)
        M = np.dot(X.T, X)

        if getattr(self, 'regularization', None) is not None:
            diag = M.ravel(order='K')[::M.shape[0] + 1]
            if self.regularize_by_trace:
                diag += diag.sum() * np.asarray(self.regularization)
            else:
                diag += np.asarray(self.regularization)

        return X, M
417822
def regression(fname="regression.png"):
    
    _, axes = plt.subplots(ncols=2, figsize=(18, 6))
    alphas = np.logspace(-10, 1, 300)
    data = load_concrete(split=True)

    # Plot prediction error in the middle
    oz = PredictionError(LassoCV(alphas=alphas), ax=axes[0])
    oz.fit(data.X.train, data.y.train)
    oz.score(data.X.test, data.y.test)
    oz.finalize()

    # Plot residuals on the right
    oz = ResidualsPlot(RidgeCV(alphas=alphas), ax=axes[1])
    oz.fit(data.X.train, data.y.train)
    oz.score(data.X.test, data.y.test)
    oz.finalize()

    # Save figure
    path = os.path.join(FIGURES, fname)
    plt.tight_layout()
    plt.savefig(path)
879066
private void train2(Attribute[] attributes, double[][] x, int[] y) {        
        int n = x.length;

        int[] nc = new int[k];
        for (int i = 0; i < n; i++) {
            nc[y[i]]++;
        }

        int[] y2 = new int[n];
        for (int i = 0; i < n; i++) {
            if (y[i] == 1) {
                y2[i] = 1;
            } else {
                y2[i] = -1;
            }
        }
        
        double[] h = new double[n]; // current F(x_i)
        double[] response = new double[n]; // response variable for regression tree.

        double mu = Math.mean(y2);
        b = 0.5 * Math.log((1 + mu) / (1 - mu));

        for (int i = 0; i < n; i++) {
            h[i] = b;
        }

        int[][] order = SmileUtils.sort(attributes, x);
        RegressionTree.NodeOutput output = new L2NodeOutput(response);
        trees = new RegressionTree[ntrees];

        int[] perm = new int[n];
        int[] samples = new int[n];
        for (int i = 0; i < n; i++) {
            perm[i] = i;
        }

        for (int m = 0; m < ntrees; m++) {
            Arrays.fill(samples, 0);
            Math.permutate(perm);
            for (int l = 0; l < k; l++) {
                int subj = (int) Math.round(nc[l] * subsample);
                int count = 0;
                for (int i = 0; i < n && count < subj; i++) {
                    int xi = perm[i];
                    if (y[xi] == l) {
                        samples[xi] = 1;
                        count++;
                    }
                }
            }

            for (int i = 0; i < n; i++) {
                response[i] = 2.0 * y2[i] / (1 + Math.exp(2 * y2[i] * h[i]));
            }

            trees[m] = new RegressionTree(attributes, x, response, maxNodes, 5, x[0].length, order, samples, output);

            for (int i = 0; i < n; i++) {
                h[i] += shrinkage * trees[m].predict(x[i]);
            }
        }
    }
22176
def _unscramble_regressor_columns(parent_data, data):
    
    matches = ['_power[0-9]+', '_derivative[0-9]+']
    var = OrderedDict((c, deque()) for c in parent_data.columns)
    for c in data.columns:
        col = c
        for m in matches:
            col = re.sub(m, '', col)
        if col == c:
            var[col].appendleft(c)
        else:
            var[col].append(c)
    unscrambled = reduce((lambda x, y: x + y), var.values())
    return data[[*unscrambled]]
668901
def _sparse_fit(self, X, strategy, missing_values, fill_value):
        """"""
        mask_data = _get_mask(X.data, missing_values)
        n_implicit_zeros = X.shape[0] - np.diff(X.indptr)

        statistics = np.empty(X.shape[1])

        if strategy == "constant":
            # for constant strategy, self.statistcs_ is used to store
            # fill_value in each column
            statistics.fill(fill_value)

        else:
            for i in range(X.shape[1]):
                column = X.data[X.indptr[i]:X.indptr[i + 1]]
                mask_column = mask_data[X.indptr[i]:X.indptr[i + 1]]
                column = column[~mask_column]

                # combine explicit and implicit zeros
                mask_zeros = _get_mask(column, 0)
                column = column[~mask_zeros]
                n_explicit_zeros = mask_zeros.sum()
                n_zeros = n_implicit_zeros[i] + n_explicit_zeros

                if strategy == "mean":
                    s = column.size + n_zeros
                    statistics[i] = np.nan if s == 0 else column.sum() / s

                elif strategy == "median":
                    statistics[i] = _get_median(column,
                                                n_zeros)

                elif strategy == "most_frequent":
                    statistics[i] = _most_frequent(column,
                                                   0,
                                                   n_zeros)
        return statistics
1087155
def predict(self, L, P, R):
        
        phi = self.extract_one(L, P, R)
        return self.classifier.predict(phi)
254812
def obfn_reg(self):
        
        """

        rl21 = np.sum(self.wl21 * np.sqrt(np.sum(self.obfn_gvar()**2,
                                                 axis=self.cri.axisC)))
        rgr = sl.rfl2norm2(np.sqrt(self.GHGf*np.conj(fvf)*fvf), self.cri.Nv,
                           self.cri.axisN)/2.0
        return (self.lmbda*rl21 + self.mu*rgr, rl21, rgr)
1330207
def partial_fit(self, X):
        
        opt, cost = self.sess.run((self.optimizer, self.cost), 
                                  feed_dict={self.x: X})
        return cost
320546
def run_models(self):
        

        self.linear_regression()
        self.lasso_regression()
        self.ridge_regression()
        self.elastic_net_regression()
        self.random_forest()
        self.ann()

        # Index of the model with max score
        best_model_index = self.max_scores.index(max(self.max_scores))

        # Store name of the optimal model
        self.best_model_name = self.model_names[best_model_index]

        # Store optimal model
        self.best_model = self.models[best_model_index]

        return self.metrics
150046
def predict(self, X):
        """"""
        return self.__cost(self.__unroll(self.__thetas), 0, np.matrix(X))
1349474
def multiple_optima(gene_number=937, resolution=80, model_restarts=10, seed=10000, max_iters=300, optimize=True, plot=True):
    

    # Contour over a range of length scales and signal/noise ratios.
    length_scales = np.linspace(0.1, 60., resolution)
    log_SNRs = np.linspace(-3., 4., resolution)

    try:import pods
    except ImportError:
        print('pods unavailable, see https://github.com/sods/ods for example datasets')
        return
    data = pods.datasets.della_gatta_TRP63_gene_expression(data_set='della_gatta',gene_number=gene_number)
    # data['Y'] = data['Y'][0::2, :]
    # data['X'] = data['X'][0::2, :]

    data['Y'] = data['Y'] - np.mean(data['Y'])

    lls = GPy.examples.regression._contour_data(data, length_scales, log_SNRs, GPy.kern.RBF)
    if plot:
        pb.contour(length_scales, log_SNRs, np.exp(lls), 20, cmap=pb.cm.jet)
        ax = pb.gca()
        pb.xlabel('length scale')
        pb.ylabel('log_10 SNR')

        xlim = ax.get_xlim()
        ylim = ax.get_ylim()

    # Now run a few optimizations
    models = []
    optim_point_x = np.empty(2)
    optim_point_y = np.empty(2)
    np.random.seed(seed=seed)
    for i in range(0, model_restarts):
        # kern = GPy.kern.RBF(1, variance=np.random.exponential(1.), lengthscale=np.random.exponential(50.))
        kern = GPy.kern.RBF(1, variance=np.random.uniform(1e-3, 1), lengthscale=np.random.uniform(5, 50))

        m = GPy.models.GPRegression(data['X'], data['Y'], kernel=kern)
        m.likelihood.variance = np.random.uniform(1e-3, 1)
        optim_point_x[0] = m.rbf.lengthscale
        optim_point_y[0] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);

        # optimize
        if optimize:
            m.optimize('scg', xtol=1e-6, ftol=1e-6, max_iters=max_iters)

        optim_point_x[1] = m.rbf.lengthscale
        optim_point_y[1] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);

        if plot:
            pb.arrow(optim_point_x[0], optim_point_y[0], optim_point_x[1] - optim_point_x[0], optim_point_y[1] - optim_point_y[0], label=str(i), head_length=1, head_width=0.5, fc='k', ec='k')
        models.append(m)

    if plot:
        ax.set_xlim(xlim)
        ax.set_ylim(ylim)
    return m