Fit a k-means clustering model using an ordered sequence of blocks.
501367
def fit(self, blocks, y=None):
        
        self.kmeans.fit(make_weninger_features(blocks))
        # set the cluster center closest to the origin to exactly (0.0, 0.0)
        self.kmeans.cluster_centers_.sort(axis=0)
        self.kmeans.cluster_centers_[0, :] = np.zeros(2)
        return self
1264326
def _centroids(n_clusters: int, points: List[List[float]]) -> List[List[float]]:
    

    k_means = KMeans(n_clusters=n_clusters)
    k_means.fit(points)

    closest, _ = pairwise_distances_argmin_min(k_means.cluster_centers_, points)

    return list(map(list, np.array(points)[closest.tolist()]))
1102639
def _guess_lines(ys, max_lines=50, confidence_minimum=0.0):
        """"""
        ys = ys.astype(numpy.float32)
        compactness_list, means_list, diffs, deviations = [], [], [], []
        start_n = 1
        for k in range(start_n, min(len(ys), max_lines)):
            compactness, classified_points, means = cv2.kmeans(data=ys, K=k, bestLabels=None, criteria=(
            cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_MAX_ITER, 1, 10), attempts=2, flags=cv2.KMEANS_PP_CENTERS)
            means = numpy.sort(means, axis=0)
            means_list.append(means)
            compactness_list.append(compactness)
            if k < 3:
                tmp1 = [1, 2, 500, 550]  # forge data for bad clusters
            else:
                # calculate the center of each cluster. Assuming lines are equally spaced...
                tmp1 = numpy.diff(means, axis=0)  # diff will be equal or very similar
            tmp2 = numpy.std(tmp1) / numpy.mean(means)  # so variance is minimal
            tmp3 = numpy.sum((tmp1 - numpy.mean(tmp1)) ** 2)  # root mean square deviation, more sensitive than std
            diffs.append(tmp1)
            deviations.append(tmp3)

        compactness_list = numpy.diff(
            numpy.log(numpy.array(compactness_list) + 0.01))  # sum small amount to avoid log(0)
        deviations = numpy.array(deviations[1:])
        deviations[0] = numpy.mean(deviations[1:])
        compactness_list = (compactness_list - numpy.mean(compactness_list)) / numpy.std(compactness_list)
        deviations = (deviations - numpy.mean(deviations)) / numpy.std(deviations)
        aglomerated_metric = 0.1 * compactness_list + 0.9 * deviations

        i = numpy.argmin(aglomerated_metric) + 1
        lines = means_list[i]

        # calculate confidence
        betterness = numpy.sort(aglomerated_metric, axis=0)
        confidence = (betterness[1] - betterness[0]) / (betterness[2] - betterness[1])
        if confidence < confidence_minimum:
            raise Exception("low confidence")
        return lines
1035235
def fit(self, X, y, n_iter=None):
        """"""
        self.n_iter = self.n_iter if n_iter is None else n_iter
        X = getattr(X, 'values', X).reshape(len(X), 1)
        X_1 = self.homogenize(X)
        for i in range(self.n_iter):
            for i in range(0, len(X), 10):  # minibatch learning for numerical stability
                batch = slice(i, min(i + 10, len(X)))
                Xbatch, ybatch = X[batch, :], y[batch]
                X_1_batch = X_1[batch, :]
                self.W += (self.alpha / len(X) ** 1.5) * (
                    self.delta(Xbatch, ybatch).reshape((len(Xbatch), 1)).T.dot(X_1_batch))
        return self
317915
def fit(self, documents, labels, weights=None):
        
        block_groups = np.array([self.blockifier.blockify(doc) for doc in documents])
        mask = [self._has_enough_blocks(blocks) for blocks in block_groups]
        block_groups = block_groups[mask]
        labels = np.concatenate(np.array(labels)[mask])

        # TODO: This only 'fit's one doc at a time. No feature fitting actually
        # happens for now, but this might be important if the features change
        features_mat = np.concatenate([self.features.fit_transform(blocks)
                                       for blocks in block_groups])
        if weights is None:
            self.model.fit(features_mat, labels)
        else:
            weights = np.concatenate(np.array(weights)[mask])
            self.model.fit(features_mat, labels, sample_weight=weights)
        return self
940414
def init_cao(X, n_clusters, dissim):
    
    n_points, n_attrs = X.shape
    centroids = np.empty((n_clusters, n_attrs), dtype='object')
    # Method is based on determining density of points.
    dens = np.zeros(n_points)
    for iattr in range(n_attrs):
        freq = defaultdict(int)
        for val in X[:, iattr]:
            freq[val] += 1
        for ipoint in range(n_points):
            dens[ipoint] += freq[X[ipoint, iattr]] / float(n_points) / float(n_attrs)

    # Choose initial centroids based on distance and density.
    centroids[0] = X[np.argmax(dens)]
    if n_clusters > 1:
        # For the remaining centroids, choose maximum dens * dissim to the
        # (already assigned) centroid with the lowest dens * dissim.
        for ik in range(1, n_clusters):
            dd = np.empty((ik, n_points))
            for ikk in range(ik):
                dd[ikk] = dissim(X, centroids[ikk]) * dens
            centroids[ik] = X[np.argmax(np.min(dd, axis=0))]

    return centroids
73527
def corpus2clusters(corpus, index,params_clustermap={'vmin':0,'vmax':1,'figsize':[6,6]}):
    
    from sklearn.feature_extraction.text import TfidfVectorizer
    vect = TfidfVectorizer()
    tfidf = vect.fit_transform(corpus)

    df=pd.DataFrame((tfidf * tfidf.T).A,
                index=index,
                columns=index,
                )
    clustergrid=sns.clustermap(df,**params_clustermap
    #                                method='complete', metric='canberra',
                                  )
    dclusters=get_clusters(clustergrid,axis=0,criterion='maxclust',clusters_fraction=0.25)
    return dclusters
140996
def _dense_fit(self, X, strategy, missing_values, fill_value):
        """"""
        mask = _get_mask(X, missing_values)
        masked_X = ma.masked_array(X, mask=mask)

        # Mean
        if strategy == "mean":
            mean_masked = np.ma.mean(masked_X, axis=0)
            # Avoid the warning "Warning: converting a masked element to nan."
            mean = np.ma.getdata(mean_masked)
            mean[np.ma.getmask(mean_masked)] = np.nan

            return mean

        # Median
        elif strategy == "median":
            median_masked = np.ma.median(masked_X, axis=0)
            # Avoid the warning "Warning: converting a masked element to nan."
            median = np.ma.getdata(median_masked)
            median[np.ma.getmaskarray(median_masked)] = np.nan

            return median

        # Most frequent
        elif strategy == "most_frequent":
            # scipy.stats.mstats.mode cannot be used because it will no work
            # properly if the first element is masked and if its frequency
            # is equal to the frequency of the most frequent valid element
            # See https://github.com/scipy/scipy/issues/2636

            # To be able access the elements by columns
            X = X.transpose()
            mask = mask.transpose()

            if X.dtype.kind == "O":
                most_frequent = np.empty(X.shape[0], dtype=object)
            else:
                most_frequent = np.empty(X.shape[0])

            for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
                row_mask = np.logical_not(row_mask).astype(np.bool)
                row = row[row_mask]
                most_frequent[i] = _most_frequent(row, np.nan, 0)

            return most_frequent

        # Constant
        elif strategy == "constant":
            # for constant strategy, self.statistcs_ is used to store
            # fill_value in each column
            return np.full(X.shape[1], fill_value, dtype=X.dtype)
1332418
def calc_cat_clust_order(net, inst_rc):
  '''
  
  '''
  from .__init__ import Network
  from copy import deepcopy
  from . import calc_clust, run_filter

  inst_keys = list(net.dat['node_info'][inst_rc].keys())
  all_cats = [x for x in inst_keys if 'cat-' in x]

  if len(all_cats) > 0:

    for inst_name_cat in all_cats:

      tmp_name = 'dict_' + inst_name_cat.replace('-', '_')
      dict_cat = net.dat['node_info'][inst_rc][tmp_name]

      unordered_cats = dict_cat.keys()

      ordered_cats = order_categories(unordered_cats)

      # this is the ordering of the columns based on their category, not
      # including their clustering ordering within category
      all_cat_orders = []
      tmp_names_list = []
      for inst_cat in ordered_cats:

        inst_nodes = dict_cat[inst_cat]

        tmp_names_list.extend(inst_nodes)

      #   cat_net = deepcopy(Network())

      #   cat_net.dat['mat'] = deepcopy(net.dat['mat'])
      #   cat_net.dat['nodes'] = deepcopy(net.dat['nodes'])

      #   cat_df = cat_net.dat_to_df()

      #   sub_df = {}
      #   if inst_rc == 'col':
      #     sub_df['mat'] = cat_df['mat'][inst_nodes]
      #   elif inst_rc == 'row':
      #     # need to transpose df
      #     cat_df['mat'] = cat_df['mat'].transpose()
      #     sub_df['mat'] = cat_df['mat'][inst_nodes]
      #     sub_df['mat'] = sub_df['mat'].transpose()

      #   # filter matrix before clustering
      #   ###################################
      #   threshold = 0.0001
      #   sub_df = run_filter.df_filter_row_sum(sub_df, threshold)
      #   sub_df = run_filter.df_filter_col_sum(sub_df, threshold)

      #   # load back to dat
      #   cat_net.df_to_dat(sub_df)

      #   cat_mat_shape = cat_net.dat['mat'].shape

      #   print('***************')
      #   try:
      #     if cat_mat_shape[0]>1 and cat_mat_shape[1] > 1 and all_are_numbers == False:

      #       calc_clust.cluster_row_and_col(cat_net, 'cos')
      #       inst_cat_order = cat_net.dat['node_info'][inst_rc]['clust']
      #     else:
      #       inst_cat_order = list(range(len(cat_net.dat['nodes'][inst_rc])))

      #   except:
      #     inst_cat_order = list(range(len(cat_net.dat['nodes'][inst_rc])))


      #   prev_order_len = len(all_cat_orders)

      #   # add prev order length to the current order number
      #   inst_cat_order = [i + prev_order_len for i in inst_cat_order]
      #   all_cat_orders.extend(inst_cat_order)

      # # generate ordered list of row/col names, which will be used to
      # # assign the order to specific nodes
      # names_clust_list = [x for (y, x) in sorted(zip(all_cat_orders,
      #                     tmp_names_list))]

      names_clust_list = tmp_names_list

      # calc category-cluster order
      final_order = []

      for i in range(len(net.dat['nodes'][inst_rc])):

        inst_node_name = net.dat['nodes'][inst_rc][i]
        inst_node_num = names_clust_list.index(inst_node_name)

        final_order.append(inst_node_num)

      inst_index_cat = inst_name_cat.replace('-', '_') + '_index'

      net.dat['node_info'][inst_rc][inst_index_cat] = final_order
1026046
def fit(self, X, y=None):
        

        if self.max_landmarks is not None:
            if self.n_clusters > self.n_landmarks:
                self.n_landmarks = self.max_landmarks
        
        if self.n_landmarks is None:
            distances = pdist(X, self.metric)
            tree = linkage(distances, method=self.linkage)
            self.landmark_labels_ = fcluster(tree, criterion='maxclust',
                                             t=self.n_clusters) - 1
            self.cardinality_ = np.bincount(self.landmark_labels_)
            self.squared_distances_within_cluster_ = np.zeros(self.n_clusters)

            n = len(X)
            for k in range(len(distances)):
                i = int(n - 2 - np.floor(np.sqrt(-8*k + 4*n*(n-1)-7)/2.0 - 0.5))
                j = int(k + i + 1 - n*(n-1)/2 + (n-i)*((n-i)-1)/2)
                if self.landmark_labels_[i] == self.landmark_labels_[j]:
                    self.squared_distances_within_cluster_[
                        self.landmark_labels_[i]] += distances[k] ** 2

            self.landmarks_ = X

        else:
            if self.landmark_strategy == 'random':
                land_indices = check_random_state(self.random_state).randint(
                    len(X), size=self.n_landmarks)
            else:
                land_indices = np.arange(len(X))[::(len(X) //
                                        self.n_landmarks)][:self.n_landmarks]

            distances = pdist(X[land_indices], self.metric)
            tree = linkage(distances, method=self.linkage)
            self.landmark_labels_ = fcluster(tree, criterion='maxclust',
                                             t=self.n_clusters) - 1
            self.cardinality_ = np.bincount(self.landmark_labels_)
            self.squared_distances_within_cluster_ = np.zeros(self.n_clusters)

            n = len(X[land_indices])
            for k in range(len(distances)):
                i = int(n - 2 - np.floor(np.sqrt(-8*k + 4*n*(n-1)-7)/2.0 - 0.5))
                j = int(k + i + 1 - n*(n-1)/2 + (n-i)*((n-i)-1)/2)
                if self.landmark_labels_[i] == self.landmark_labels_[j]:
                    self.squared_distances_within_cluster_[
                        self.landmark_labels_[i]] += distances[k] ** 2

            self.landmarks_ = X[land_indices]

        if self.metric != 'rmsd':
            cluster_centers_ = []
            for i in range(self.n_clusters):
                temp = list(np.mean(self.landmarks_[self.landmark_labels_==i], axis=0))
                cluster_centers_.append(temp)
            self.cluster_centers_ = np.array(cluster_centers_)

        return self
1234250
def make_clust(net, dist_type='cosine', run_clustering=True, dendro=True,
                          requested_views=['pct_row_sum', 'N_row_sum'],
                          linkage_type='average', sim_mat=False, filter_sim=0.1,
                          calc_cat_pval=False, sim_mat_views=['N_row_sum'],
                          run_enrichr=None, enrichrgram=None):
  '''
  
  '''
  from copy import deepcopy
  import scipy
  from . import calc_clust, run_filter, make_views, make_sim_mat, cat_pval
  from . import enrichr_functions as enr_fun

  df = net.dat_to_df()

  threshold = 0.0001
  df = run_filter.df_filter_row_sum(df, threshold)
  df = run_filter.df_filter_col_sum(df, threshold)

  # default setting
  define_cat_colors = False

  if run_enrichr is not None:
    df = enr_fun.add_enrichr_cats(df, 'row', run_enrichr)

    define_cat_colors = True

  # calculate initial view with no row filtering
  net.df_to_dat(df, define_cat_colors=True)


  inst_dm = calc_clust.cluster_row_and_col(net, dist_type=dist_type,
                                linkage_type=linkage_type,
                                run_clustering=run_clustering,
                                dendro=dendro, ignore_cat=False,
                                calc_cat_pval=calc_cat_pval)

  all_views = []
  send_df = deepcopy(df)

  if 'N_row_sum' in requested_views:
    all_views = make_views.N_rows(net, send_df, all_views,
                                  dist_type=dist_type, rank_type='sum')

  if 'N_row_var' in requested_views:
    all_views = make_views.N_rows(net, send_df, all_views,
                                  dist_type=dist_type, rank_type='var')

  if 'pct_row_sum' in requested_views:
    all_views = make_views.pct_rows(net, send_df, all_views,
                                    dist_type=dist_type, rank_type='sum')

  if 'pct_row_var' in requested_views:
    all_views = make_views.pct_rows(net, send_df, all_views,
                                    dist_type=dist_type, rank_type='var')

  which_sim = []

  if sim_mat == True:
    which_sim = ['row', 'col']
  elif sim_mat == 'row':
    which_sim = ['row']
  elif sim_mat == 'col':
    which_sim = ['col']

  if sim_mat is not False:
    sim_net = make_sim_mat.main(net, inst_dm, which_sim, filter_sim, sim_mat_views)

    net.sim = {}

    for inst_rc in which_sim:
      net.sim[inst_rc] = sim_net[inst_rc].viz

      if inst_rc == 'row':
        other_rc = 'col'
      elif inst_rc == 'col':
        other_rc = 'row'

      # keep track of cat_colors
      net.sim[inst_rc]['cat_colors'][inst_rc] = net.viz['cat_colors'][inst_rc]
      net.sim[inst_rc]['cat_colors'][other_rc] = net.viz['cat_colors'][inst_rc]

  else:
    net.sim = {}

  net.viz['views'] = all_views

  if enrichrgram != None:
    # toggle enrichrgram functionality from back-end
    net.viz['enrichrgram'] = enrichrgram

  if 'enrichrgram_lib' in net.dat:
    net.viz['enrichrgram'] = True
    net.viz['enrichrgram_lib'] = net.dat['enrichrgram_lib']

  if 'row_cat_bars' in net.dat:
    net.viz['row_cat_bars'] = net.dat['row_cat_bars']
1240247
def classify_clusters(points, n=10):
    
    arr = [[p.x, p.y] for p in points.values]
    clf = KMeans(n_clusters=n)
    clf.fit(arr)
    classes = clf.predict(arr)
    return classes
5027
def init_scalable(
    X, n_clusters, random_state=None, max_iter=None, oversampling_factor=2
):
    

    logger.info("Initializing with k-means||")
    # Step 1: Initialize Centers
    idx = 0
    centers = da.compute(X[idx, np.newaxis])[0]
    c_idx = {idx}

    # Step 2: Initialize cost
    cost, = compute(evaluate_cost(X, centers))

    if cost == 0:
        n_iter = 0
    else:
        n_iter = int(np.round(np.log(cost)))

    if max_iter is not None:
        n_iter = min(max_iter, n_iter)

    # Steps 3 - 6: update candidate Centers
    for i in range(n_iter):
        with _timer(
            "init iteration %2d/%2d , %2d centers" % (i + 1, n_iter, len(c_idx)),
            _logger=logger,
        ):
            new_idxs = _sample_points(X, centers, oversampling_factor, random_state)
            new_idxs = set(*compute(new_idxs))
            c_idx |= new_idxs

        # Sort before slicing, for better performance / memory
        # usage with the scheduler.
        # See https://github.com/dask/dask-ml/issues/39
        centers = X[sorted(c_idx)].compute()

    # XXX: scikit-learn doesn't have weighted k-means.
    # The paper weights each center by the number of points closest to it.
    # https://stackoverflow.com/a/37198799/1889400 claims you can scale the
    # features before clustering, but that doesn't seem right.
    # I think that replicating the *points*, proportional to the number of
    # original points closest to the candidate centers, would be a better way
    # to do that.

    if len(centers) < n_clusters:
        logger.warning("Found fewer than %d clusters in init.", n_clusters)
        # supplement with random
        need = n_clusters - len(centers)
        locs = sorted(
            random_state.choice(
                np.arange(0, len(X)), size=need, replace=False, chunks=len(X)
            )
        )
        extra = X[locs].compute()
        return np.vstack([centers, extra])
    else:
        # Step 7, 8 without weights
        # dask RandomState objects aren't valid for scikit-learn
        rng2 = (
            random_state.randint(0, np.iinfo("i4").max - 1, chunks=())
            .compute(scheduler="single-threaded")
            .item()
        )
        km = sk_k_means.KMeans(n_clusters, random_state=rng2)
        km.fit(centers)

    return km.cluster_centers_
1180216
function clusterByNeighbor(points) {
    const sorted = points.slice().sort((a, b) => {
        if (a.y < b.y) {
            return -1;
        } else if (a.y > b.y) {
            return 1;
        } else {
            if (a.x < b.x) {
                return -1;
            } else if (a.x > b.x) {
                return 1;
            } else {
                return 0;
            }
        }
    });

    const cluster = sorted.reduce((sets, point) => {
        const setContainingNeighbors = sets.find(set => set.some((setPoint) => {
            return getNeighbors(setPoint).some(spn => areEqual(spn, point));
        }));

        if (setContainingNeighbors) {
            setContainingNeighbors.push(point);
        } else {
            sets.push([point]);
        }

        return sets;
    }, []);

    return cluster;
}
190371
def _k_prototypes_iter(Xnum, Xcat, centroids, cl_attr_sum, cl_memb_sum, cl_attr_freq,
                       membship, num_dissim, cat_dissim, gamma, random_state):
    """"""
    moves = 0
    for ipoint in range(Xnum.shape[0]):
        clust = np.argmin(
            num_dissim(centroids[0], Xnum[ipoint]) +
            gamma * cat_dissim(centroids[1], Xcat[ipoint], X=Xcat, membship=membship)
        )
        if membship[clust, ipoint]:
            # Point is already in its right place.
            continue

        # Move point, and update old/new cluster frequencies and centroids.
        moves += 1
        old_clust = np.argwhere(membship[:, ipoint])[0][0]

        # Note that membship gets updated by kmodes.move_point_cat.
        # move_point_num only updates things specific to the k-means part.
        cl_attr_sum, cl_memb_sum = move_point_num(
            Xnum[ipoint], clust, old_clust, cl_attr_sum, cl_memb_sum
        )
        cl_attr_freq, membship, centroids[1] = kmodes.move_point_cat(
            Xcat[ipoint], ipoint, clust, old_clust,
            cl_attr_freq, membship, centroids[1]
        )

        # Update old and new centroids for numerical attributes using
        # the means and sums of all values
        for iattr in range(len(Xnum[ipoint])):
            for curc in (clust, old_clust):
                if cl_memb_sum[curc]:
                    centroids[0][curc, iattr] = cl_attr_sum[curc, iattr] / cl_memb_sum[curc]
                else:
                    centroids[0][curc, iattr] = 0.

        # In case of an empty cluster, reinitialize with a random point
        # from largest cluster.
        if not cl_memb_sum[old_clust]:
            from_clust = membship.sum(axis=1).argmax()
            choices = [ii for ii, ch in enumerate(membship[from_clust, :]) if ch]
            rindx = random_state.choice(choices)

            cl_attr_sum, cl_memb_sum = move_point_num(
                Xnum[rindx], old_clust, from_clust, cl_attr_sum, cl_memb_sum
            )
            cl_attr_freq, membship, centroids[1] = kmodes.move_point_cat(
                Xcat[rindx], rindx, old_clust, from_clust,
                cl_attr_freq, membship, centroids[1]
            )

    return centroids, moves
920503
def _fit(self, index, width, height):
        """"""
        node = self._atlas_nodes[index]
        x, y = node[0], node[1]
        width_left = width
        if x+width > self._shape[1]:
            return -1
        i = index
        while width_left > 0:
            node = self._atlas_nodes[i]
            y = max(y, node[1])
            if y+height > self._shape[0]:
                return -1
            width_left -= node[2]
            i += 1
        return y
127092
def k_modes(X, n_clusters, max_iter, dissim, init, n_init, verbose, random_state, n_jobs):
    """"""
    random_state = check_random_state(random_state)
    if sparse.issparse(X):
        raise TypeError("k-modes does not support sparse data.")

    X = check_array(X, dtype=None)

    # Convert the categorical values in X to integers for speed.
    # Based on the unique values in X, we can make a mapping to achieve this.
    X, enc_map = encode_features(X)

    n_points, n_attrs = X.shape
    assert n_clusters <= n_points, "Cannot have more clusters ({}) " \
                                   "than data points ({}).".format(n_clusters, n_points)

    # Are there more n_clusters than unique rows? Then set the unique
    # rows as initial values and skip iteration.
    unique = get_unique_rows(X)
    n_unique = unique.shape[0]
    if n_unique <= n_clusters:
        max_iter = 0
        n_init = 1
        n_clusters = n_unique
        init = unique

    results = []
    seeds = random_state.randint(np.iinfo(np.int32).max, size=n_init)
    if n_jobs == 1:
        for init_no in range(n_init):
            results.append(k_modes_single(X, n_clusters, n_points, n_attrs, max_iter,
                                          dissim, init, init_no, verbose, seeds[init_no]))
    else:
        results = Parallel(n_jobs=n_jobs, verbose=0)(
            delayed(k_modes_single)(X, n_clusters, n_points, n_attrs, max_iter,
                                    dissim, init, init_no, verbose, seed)
            for init_no, seed in enumerate(seeds))
    all_centroids, all_labels, all_costs, all_n_iters = zip(*results)

    best = np.argmin(all_costs)
    if n_init > 1 and verbose:
        print("Best run was number {}".format(best + 1))

    return all_centroids[best], enc_map, all_labels[best], \
        all_costs[best], all_n_iters[best]