<jupyter_start><jupyter_text>1. We try K-means method<jupyter_code>n_clusters=2
kmeans = KMeans(n_clusters=n_clusters, n_init=100, n_jobs=4)
class_pred = kmeans.fit_predict(desc_clean_data)
class_pred
unique, counts = np.unique(class_pred, return_counts=True)
dict(zip(unique, counts))<jupyter_output><empty_output><jupyter_text>2. We try similarity matrix and spectral clustering<jupyter_code>print(len(desc_clean_data))
print(desc_clean_data.shape)
desc_clean_data
from tqdm import tqdm, tqdm_notebook
from scipy import sparse
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import SpectralClustering
desc_sparse = sparse.csr_matrix(desc_clean_data)
%time
sim_matrix = cosine_similarity(desc_sparse, desc_sparse)
#sim_matrix<jupyter_output><empty_output><jupyter_text>Explore sim_matrix<jupyter_code>np.median(sim_matrix)
np.argwhere(sim_matrix < 0.98)[0:10,:]
n_clusters=2
model = SpectralClustering(n_clusters, affinity='precomputed')
class_pred = model.fit_predict(sim_matrix)
unique, counts = np.unique(class_pred, return_counts=True)
dict(zip(unique, counts))<jupyter_output><empty_output><jupyter_text>Analyze<jupyter_code># jpprieto 26/06/2019
# Plot the principal components
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
X_pre = desc_clean_data
scaler = StandardScaler()
X = scaler.fit_transform(X_pre)
pca = PCA()
scaler = StandardScaler()
X=scaler.fit_transform(X)
pca.fit_transform(X)
features = range(pca.n_components_)
print(pca.n_components_)
print(pca.components_)
plt.bar(features, pca.explained_variance_ratio_)
plt.xticks(features)
plt.xlabel("PCA features")
plt.ylabel("variance")
plt.yscale('log')
plt.ylim(1e-4,2e0)
plt.xlim(-0.5,15.5)
plt.grid()
plt.show()
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
plt.xlim(-0.5,15.5)
plt.grid()
plt.show()
pca = PCA(7)  # project from 97 to 7 dimensions
X_pca = pca.fit_transform(X)
print(X.shape)
print(X_pca.shape)
import numpy as np
import sklearn.datasets, sklearn.decomposition

mu = np.mean(X, axis=0)

pca = sklearn.decomposition.PCA()
pca.fit(X)

nComp = 7
Xhat = np.dot(pca.transform(X)[:,:nComp], pca.components_[:nComp,:])
Xhat += mu
print(Xhat.shape)
print(Xhat)
pca = PCA()
scaler = StandardScaler()
Xhat=scaler.fit_transform(Xhat)
pca.fit_transform(Xhat)
features = range(pca.n_components_)
print(pca.n_components_)
print(pca.components_)
plt.bar(features, pca.explained_variance_ratio_)
plt.xticks(features)
plt.xlabel("PCA features")
plt.ylabel("variance")
plt.yscale('log')
plt.ylim(1e-4,2e0)
plt.xlim(-0.5,15.5)
plt.grid()
plt.show()
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
plt.xlim(-0.5,15.5)
plt.grid()
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup