<jupyter_start><jupyter_text>Dimensionality Reduction  Term Clustering Cluster columns instead of rows in order to find term clustering and use those cluster centroids as features to reduce dimensions.<jupyter_code>csrL2Normalized.T
rowLabels = train_model.bisecting_kmeans(csrL2Normalized.T.toarray(), 3, 3)<jupyter_output><empty_output><jupyter_text>LDA (Latent Dirichlet Allocation)<jupyter_code>from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=1000, random_state=0)
csrL2Normalized_lda = lda.fit_transform(csrL2Normalized)
csrL2Normalized_lda.shape
type(csrL2Normalized_lda)
pd.DataFrame(csrL2Normalized_lda).head()
csrL2Normalized_lda[:,1:2].shape
calinski_harabaz_score(denseMatrix, csrL2Normalized_lda[:,1:2])
labels = train_model.bisecting_kmeans(csrL2Normalized_lda, 3, 3)
calinski_harabaz_score(denseMatrix, labels)<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>from sklearn.decomposition import TruncatedSVD
pca = TruncatedSVD(10000)
csrL2Normalized_pca = pca.fit_transform(csrL2Normalized)<jupyter_output><empty_output><jupyter_text>Outlier Detection<jupyter_code>dfDense = pd.DataFrame(denseMatrix)
dfDense[2].head()
sns.distplot(dfDense[2])
from sklearn.ensemble import IsolationForest
clf = IsolationForest(behaviour='new', random_state=10, contamination='auto', n_jobs=-1)
outl = clf.fit_predict(csrL2Normalized.T)
pd.DataFrame(outl)[0].value_counts()
dfDense.shape
colsToBeRemoved = list()
for idx,out in enumerate(outl):
    if out == -1:
        colsToBeRemoved.append(idx)
denseMatrixWithoutOutliers = np.delete(denseMatrix, colsToBeRemoved, 1)
denseMatrixWithoutOutliers.shape<jupyter_output><empty_output><jupyter_text>Scikit KMeans<jupyter_code>from sklearn.cluster import KMeans
def calcSSE(csr, cluster):
    matrix = csr.toarray()
    members = matrix[cluster,:]
    return np.sum(np.square(members - np.mean(members)))
def bisect(csr, k=7, max_iter=10, random_state=10, n_jobs=-1):
    clusters = list()
    initial_clusters = list()
    
    for i in range(csr.shape[0]):
        initial_clusters.append(i)
    
    clusters.append(initial_clusters)
    
    km = KMeans(n_clusters=2, init='k-means++', max_iter=max_iter, random_state=random_state, n_jobs=n_jobs)
    
    parentSSE = list()
    parentSSE.append(calcSSE(csr, initial_clusters))
    
    while len(clusters) <= k :
        minClusterASSE = float("inf")
        minClusterBSSE = float("inf")
        minClusterA = list()
        minClusterB = list()
        minIndex = -1
        for idx, cluster in enumerate(clusters):
            totalParentSSE = parentSSE[idx]
            km.fit(csr)
            clusterA = list()
            clusterB = list()
            for index,clu in enumerate(km.labels_):
                if clu==0:
                    clusterA.append(index)
                else:
                    clusterB.append(index)
            clusterASSE = calcSSE(csr,clusterA)
            clusterBSSE = calcSSE(csr,clusterB)
            
            if clusterASSE + clusterBSSE < minClusterASSE + minClusterBSSE:
                minClusterASSE = clusterASSE
                minClusterBSSE = clusterBSSE
                minIndex = idx
                minClusterA = clusterA
                minClusterB = clusterB
            
        del clusters[minIndex]
        del parentSSE[minIndex]
        
        clusters.append(minClusterA)
        clusters.append(minClusterB)
        parentSSE.append(minClusterASSE)
        parentSSE.append(minClusterBSSE)
    
    labels = np.zeros(csr.shape[0], dtype=np.int)
    for index,cluster in enumerate(clusters):
        for idx in cluster:
            labels[idx] = index+1
    return labels
            
labels = bisect(csrL2Normalized, 3, 3)
calinski_harabaz_score(csrL2Normalized.toarray(), labels)
labels = bisect(csrL2Normalized, max_iter=10)
calinski_harabaz_score(csrL2Normalized.toarray(), labels)
read_transform.write_predictions(labels, '../models/predictions/1.1-am-my-bisect.dat')<jupyter_output><empty_output><jupyter_text>Raw KMeans<jupyter_code>km = KMeans(n_clusters=7, init='k-means++', max_iter=100, random_state=10, n_jobs=-1)
km.fit(csrL2Normalized)
pd.DataFrame(km.labels_)[0].value_counts()
km.inertia_
calinski_harabaz_score(csrL2Normalized.toarray(), km.labels_)
read_transform.write_predictions(km.labels_ + 1, '../models/predictions/1.1-am-kmeans-sci.dat')<jupyter_output><empty_output><jupyter_text>Raw kmeans without outliers<jupyter_code>csr_without_outliers = csr_matrix(denseMatrixWithoutOutliers)
csr_without_outliers.shape
km.fit(csr_without_outliers)
pd.DataFrame(km.labels_)[0].value_counts()
km.inertia_
calinski_harabaz_score(csrL2Normalized.toarray(), km.labels_)<jupyter_output><empty_output><jupyter_text>Raw Kmeans with LDA<jupyter_code>from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=10, random_state=0)
csrL2Normalized_lda = lda.fit_transform(csrL2Normalized)
csrL2Normalized_lda.shape
km.fit(csrL2Normalized_lda)
calinski_harabaz_score(csrL2Normalized_lda, km.labels_)
pd.DataFrame(km.labels_)[0].value_counts()<jupyter_output><empty_output><jupyter_text>Raw Kmeans with PCA<jupyter_code>from sklearn.decomposition import TruncatedSVD
pca = TruncatedSVD(10)
csrL2Normalized_pca = pca.fit_transform(csrL2Normalized)
csrL2Normalized_pca.shape
km.fit(csrL2Normalized_pca)
calinski_harabaz_score(csrL2Normalized_pca, km.labels_)
pd.DataFrame(km.labels_)[0].value_counts()
read_transform.write_predictions(km.labels_ + 1, '../models/predictions/1.1-am-raw-kmeans-pca-10.dat')<jupyter_output><empty_output><jupyter_text>Bisect KMeans without outliers<jupyter_code>labels = train_model.bisecting_kmeans(denseMatrixWithoutOutliers, 7, 10)
calinski_harabaz_score(denseMatrixWithoutOutliers, labels)
read_transform.write_predictions(labels, '../models/predictions/1.1-am-bisect-without-outliers.dat')<jupyter_output><empty_output><jupyter_text>Bisect Kmeans with PCA<jupyter_code>pca = TruncatedSVD(n_components=100, n_iter=10, random_state=10, algorithm='arpack')
csrL2Normalized_pca = pca.fit_transform(csrL2Normalized)
labels = train_model.bisecting_kmeans(csrL2Normalized_pca, 7, 10)
calinski_harabaz_score(csrL2Normalized_pca, labels)
read_transform.write_predictions(labels, '../models/predictions/1.1-am-bisect-with-pca.dat')
pd.DataFrame(labels)[0].value_counts()
chscores = list()
ks = list()

for k in range(3, 21, 2):
    labels = train_model.bisecting_kmeans(csrL2Normalized_pca, k, 10)
    ks.append(k)
    chscore = calinski_harabaz_score(csrL2Normalized_pca, labels)
    chscores.append(chscore)
    print ("K= %d CH Score = %f" %(k, chscore))
plt.plot(ks, chscores)<jupyter_output><empty_output><jupyter_text>Bisect Kmeans with LDA<jupyter_code>csrL2Normalized_lda = hstack((csrL2Normalized, csrL2Normalized_lda)).tocsr()
csrL2Normalized_lda
pca = TruncatedSVD(100)
csrL2Normalized_lda_pca = pca.fit_transform(csrL2Normalized_lda)
csrL2Normalized_lda_pca.shape
labels = train_model.bisecting_kmeans(csrL2Normalized_lda_pca, 7, 10)
pd.DataFrame(labels)[0].value_counts()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup