Compute allowed distance for clustering based on end confidence intervals.
3265
def _calculate_cluster_distance(end_iter):
    
    out = []
    sizes = []
    for x in end_iter:
        out.append(x)
        sizes.append(x.end1 - x.start1)
        sizes.append(x.end2 - x.start2)
    distance = sum(sizes) // len(sizes)
    return distance, out
1337167
def median_kneighbour_distance(X, k=5):
    
    N_all = X.shape[0]
    k = min(k, N_all)
    N_subset = min(N_all, 2000)
    sample_idx_train = np.random.permutation(N_all)[:N_subset]
    nn = neighbors.NearestNeighbors(k)
    nn.fit(X[sample_idx_train, :])
    d, idx = nn.kneighbors(X[sample_idx_train, :])
    return np.median(d[:, -1])
38686
def _assign_clusters(self):
        
        self.clusters = np.array([self._closest_centroid(x) for x in self._X])
1264326
def _centroids(n_clusters: int, points: List[List[float]]) -> List[List[float]]:
    

    k_means = KMeans(n_clusters=n_clusters)
    k_means.fit(points)

    closest, _ = pairwise_distances_argmin_min(k_means.cluster_centers_, points)

    return list(map(list, np.array(points)[closest.tolist()]))
400051
def cluster_number(self, data, maxgap): 
        '''

        '''

        data.sort()
        groups = [[data[0]]]
        for x in data[1:]:
            if abs(x - groups[-1][-1]) <= maxgap:
                groups[-1].append(x)
            else:
                groups.append([x])
        return groups
997108
def k_closest(points, k, origin=(0, 0)):
    # Time: O(k+(n-k)logk)
    # Space: O(k)
    
    heap = [(-distance(p, origin), p) for p in points[:k]]
    heapify(heap)

    """
    For every point p in points[k:],
    check if p is smaller than the root of the max heap;
    if it is, add p to heap and remove root. Reheapify.
    """
    for p in points[k:]:
        d = distance(p, origin)

        heappushpop(heap, (-d, p))  # heappushpop does conditional check
        """Same as:
            if d < -heap[0][0]:
                heappush(heap, (-d,p))
                heappop(heap)

        Note: heappushpop is more efficient than separate push and pop calls.
        Each heappushpop call takes O(logk) time.
        """

    return [p for nd, p in heap]
1001606
def _compute_distance(self, rup, dists, C):
        
        rref = 1.0
        c31 = -1.7
        return (c31 * np.log10(dists.rhypo) + C['c32'] * (dists.rhypo - rref))
592952
def _regroup(self):
        
        # clean up old maps
        for j in range(self.nout):
            self.inv_map[j] = []

        # find smallest divergence between input component i
        # and output component j of the cluster mixture density
        for i in range(self.nin):
            self.min_kl[i] = np.inf
            j_min = None
            for j in range(self.nout):
                kl = kullback_leibler(self.f.components[i], self.g.components[j])
                if kl < self.min_kl[i]:
                    self.min_kl[i] = kl
                    j_min = j
            assert j_min is not None
            self.inv_map[j_min].append(i)
834651
def fisher(yhat,y,samples=False):
    """"""
    classes = np.unique(y)
    mu = np.zeros(len(classes))
    v = np.zeros(len(classes))
    # pdb.set_trace()
    for c in classes.astype(int):
        mu[c] = np.mean(yhat[y==c])
        v[c] = np.var(yhat[y==c])

    if not samples:
        fisher = 0
        for c1,c2 in pairwise(classes.astype(int)):
            fisher += np.abs(mu[c1] - mu[c2])/np.sqrt(v[c1]+v[c2])
    else:
        # lexicase version
        fisher = np.zeros(len(yhat))
        # get closests classes to each class (min mu distance)
        mu_d = pairwise_distances(mu.reshape(-1,1))
        min_mu=np.zeros(len(classes),dtype=int)
        for i in np.arange(len(min_mu)):
            min_mu[i] = np.argsort(mu_d[i])[1]
        # for c1, pairwise(classes.astype(int)):
        #     min_mu[c1] = np.argmin()
        for i,l in enumerate(yhat.astype(int)):
            fisher[i] = np.abs(l - mu[min_mu[y[i]]])/np.sqrt(v[y[i]]+v[min_mu[y[i]]])

    # pdb.set_trace()
    return fisher
254207
def _bisect_kmeans(X, n_clusters, n_trials, max_iter, tol):
    
    membs = np.empty(shape=X.shape[0], dtype=int)
    centers = dict() #np.empty(shape=(n_clusters,X.shape[1]), dtype=float)
    sse_arr = dict() #-1.0*np.ones(shape=n_clusters, dtype=float)

    ## data structure to store cluster hierarchies
    tree = treelib.Tree()
    tree = _add_tree_node(tree, 0, ilev=0, X=X) 

    km = _kmeans.KMeans(n_clusters=2, n_trials=n_trials, max_iter=max_iter, tol=tol)
    for i in range(1,n_clusters):
        sel_clust_id,sel_memb_ids = _select_cluster_2_split(membs, tree)
        X_sub = X[sel_memb_ids,:]
        km.fit(X_sub)

        #print("Bisecting Step %d    :"%i, sel_clust_id, km.sse_arr_, km.centers_)
        ## Updating the clusters & properties
        #sse_arr[[sel_clust_id,i]] = km.sse_arr_
        #centers[[sel_clust_id,i]] = km.centers_
        tree = _add_tree_node(tree, 2*i-1, i, \
                              size=np.sum(km.labels_ == 0), center=km.centers_[0], \
                              sse=km.sse_arr_[0], parent= sel_clust_id)
        tree = _add_tree_node(tree, 2*i,   i, \
                             size=np.sum(km.labels_ == 1), center=km.centers_[1], \
                             sse=km.sse_arr_[1], parent= sel_clust_id)

        pred_labels = km.labels_
        pred_labels[np.where(pred_labels == 1)[0]] = 2*i
        pred_labels[np.where(pred_labels == 0)[0]] = 2*i - 1
        #if sel_clust_id == 1:
        #    pred_labels[np.where(pred_labels == 0)[0]] = sel_clust_id
        #    pred_labels[np.where(pred_labels == 1)[0]] = i
        #else:
        #    pred_labels[np.where(pred_labels == 1)[0]] = i
        #    pred_labels[np.where(pred_labels == 0)[0]] = sel_clust_id

        membs[sel_memb_ids] = pred_labels


    for n in tree.leaves():
        label = n.data['label']
        centers[label] = n.data['center']
        sse_arr[label] = n.data['sse']

    return(centers, membs, sse_arr, tree)
265640
def _find_neighbors(self, inst, avg_dist):
        
        NN = []
        min_indicies = []

        for i in range(self._datalen):
            if inst != i:
                locator = [inst, i]
                if i > inst:
                    locator.reverse()
                d = self._distance_array[locator[0]][locator[1]]
                if d < avg_dist:  # Defining the neighborhood with an average distance radius.
                    min_indicies.append(i)
        for i in range(len(min_indicies)):
            NN.append(min_indicies[i])
        return np.array(NN, dtype=np.int32)
528739
def estimate(self, points):
        """"""
        points = list(points)
        densities = callMLlibFunc(
            "estimateKernelDensity", self._sample, self._bandwidth, points)
        return np.asarray(densities)
390616
private List<Pair<Subspace, ModifiableDBIDs>> determineClusters(List<CLIQUESubspace> denseSubspaces) {
    List<Pair<Subspace, ModifiableDBIDs>> clusters = new ArrayList<>();

    for(CLIQUESubspace subspace : denseSubspaces) {
      List<Pair<Subspace, ModifiableDBIDs>> clustersInSubspace = subspace.determineClusters();
      if(LOG.isDebugging()) {
        LOG.debugFine("Subspace " + subspace + " clusters " + clustersInSubspace.size());
      }
      clusters.addAll(clustersInSubspace);
    }
    return clusters;
  }
982125
def _make_clusters(self, matrix, num_clusters_per_roi, metric):
        """"""

        from scipy.cluster.hierarchy import fclusterdata

        # maxclust needed to ensure t is interpreted as # clusters in heirarchical clustering
        group_ids = fclusterdata(matrix, metric=metric, t=num_clusters_per_roi,
                                 criterion='maxclust')
        group_set = np.unique(group_ids)
        clusters = [
            self._summary_func(matrix[group_ids == group, :], axis=0, keepdims=True)
            for group in group_set]

        return np.vstack(clusters).squeeze()
11969
def epsilon_net(points, close_distance):
  ''''''
  num_points = points.shape[0]
  indices = set(range(num_points))
  selected = []
  while indices:
    idx = indices.pop()
    nn_inds, = nearest_neighbors(points[idx], points, epsilon=close_distance)
    indices.difference_update(nn_inds)
    selected.append(idx)
  return selected
145036
def _compute_distance(cls, dists, coeffs):
        
        return coeffs['b3']*np.log10(np.sqrt(dists.rjb**2. + coeffs['b4']**2.))
1125515
def fit(self, X):
        
        self.centers_, self.labels_, self.sse_arr_, self.n_iter_ = \
              _kmedoids(X, self.n_clusters, self.distance, self.max_iter, self.n_trials, self.tol, self.rng)